diff --git a/.agent/local-agent.example.json b/.agent/local-agent.example.json new file mode 100644 index 00000000..e96bd8dd --- /dev/null +++ b/.agent/local-agent.example.json @@ -0,0 +1,10 @@ +{ + "repo": "ktalpay/CarbonOps-Parser", + "source_root": "/Users/oxygen/dev/CarbonOps/Parser/Source/CarbonOps-Parser", + "agents_root": "/Users/oxygen/FutureOps/Agents/CarbonOps-Parser", + "base_branch": "develop", + "validation_mode": "minimal", + "python_bin": "python", + "runner_script_path": "/Users/oxygen/dev/CarbonOps/Parser/Source/CarbonOps-Parser/scripts/local_codex_task_runner.py", + "log_directory": "/Users/oxygen/FutureOps/Agents/CarbonOps-Parser/.logs" +} diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..56635f5a --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @ktalpay diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7b06e380..e9a78c46 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -10,6 +10,10 @@ assignees: "" Describe the issue briefly. +## Production Scope + +- [ ] This report does not claim broader production, compliance, legal, source-owner, or carbon-accounting correctness. + ## Checklist - [ ] I have not included confidential data, credentials, or private source files. @@ -40,3 +44,6 @@ Describe the issue briefly. ## Actual Result + + +## Validation Tried diff --git a/.github/ISSUE_TEMPLATE/documentation_request.md b/.github/ISSUE_TEMPLATE/documentation_request.md new file mode 100644 index 00000000..ec9f38f9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation_request.md @@ -0,0 +1,33 @@ +--- +name: Documentation request +about: Request a documentation correction, clarification, or new docs page +title: "[Docs]: " +labels: documentation +assignees: "" +--- + +## Summary + +Describe the documentation change. + +## Affected Page + +Link to the page or section. + +## Suggested Change + +Describe the requested wording, example, or explanation. + +## Implementation Target + +- [ ] Python +- [ ] .NET +- [ ] Docs +- [ ] Database +- [ ] Shared + +## Checklist + +- [ ] I have not included confidential data, credentials, private source files, or private connection strings. +- [ ] This request preserves the documented production-ready scope. +- [ ] This request does not add package publishing, compliance, legal, source-owner, or carbon-accounting correctness claims. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index ab8da304..12a35253 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -15,6 +15,7 @@ Describe the proposed improvement. - [ ] I have not included confidential data, credentials, or private source files. - [ ] I included the source name and version or file name when relevant. - [ ] I selected the implementation target below. +- [ ] I understand source-specific ingestion, parser, database, scheduler, downloader, package publishing, or production promotion work must be explicitly scoped. ## Implementation Target @@ -31,3 +32,8 @@ Describe the proposed improvement. ## Out Of Scope + + +## Production-Ready Scope + +Describe how this feature preserves the current narrow production-ready scope. diff --git a/.github/ISSUE_TEMPLATE/production_readiness_question.md b/.github/ISSUE_TEMPLATE/production_readiness_question.md new file mode 100644 index 00000000..357d5c90 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/production_readiness_question.md @@ -0,0 +1,34 @@ +--- +name: Production-readiness question +about: Ask whether a use case is inside or outside the documented production-ready scope +title: "[Production Readiness]: " +labels: question +assignees: "" +--- + +## Question + +Describe the scope question. + +## Intended Use + +Describe the runtime, workflow, source family, or PostgreSQL operator path involved. + +## Implementation Target + +- [ ] Python +- [ ] .NET +- [ ] Docs +- [ ] Database +- [ ] Shared + +## Source Context + +- Source family: +- Source version or file name: + +## Checklist + +- [ ] I have not included credentials, private connection strings, private source files, or production infrastructure secrets. +- [ ] I understand maintainers may classify this as outside the current production-ready scope. +- [ ] I am not requesting a compliance, legal, source-owner, factor-correctness, or carbon-accounting correctness guarantee. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f2d98efe..3cd896a8 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -2,28 +2,49 @@ Briefly describe the task and outcome. -## Changes +## Scope - -## Validation +## Runtime Impact + +- [ ] No Python runtime behavior changed +- [ ] No .NET runtime behavior changed +- [ ] Runtime behavior changed and is described here: + +## Validation Performed - [ ] Tests/checks run and listed: - [ ] `git diff --check` passed -## Scope Guard Checklist +## PostgreSQL Impact + +- [ ] No PostgreSQL behavior, schema, migration, or operator workflow changed +- [ ] PostgreSQL docs/checklists changed and are described here: +- [ ] PostgreSQL runtime behavior changed and is described here: + +## Docs Impact + +- [ ] No docs changed +- [ ] Docs changed and are described here: + +## Secrets And Artifacts Checklist - [ ] No confidential data -- [ ] No production/compliance/legal claims +- [ ] No credentials, private connection strings, private source files, or production secrets +- [ ] No generated artifacts, build outputs, caches, database dumps, downloaded source files, or local test output + +## Production-Ready Claim Checklist + - [ ] No source-specific ingestion unless requested - [ ] No parser/database/scheduler/downloader coupling unless requested - [ ] No external dependencies unless requested -- [ ] One commit only -- [ ] Tests/checks run and listed +- [ ] No broader production, compliance, legal, source-owner, package publishing, or carbon-accounting correctness claims +- [ ] Production-ready wording stays within the documented narrow supported scope -## Deferred Items +## Maintainer Merge Reminder -- +Only maintainers merge pull requests. Contributors should not self-merge or imply merge authority. ## Reviewer Notes diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 941a5bee..ea5576e7 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,7 +1,23 @@ # GitHub Workflows -CI workflows are intentionally not configured yet. +CI includes a Phase 1 release validation gate in +`release-validation.yml`. -Workflows should be added after implementation and test structure exist for the Python and .NET paths. Until then, the repository uses documentation and review checklists instead of workflow YAML files. +The release gate validates a focused Phase 1 Python release-validation test set, +local static safety boundaries, local source acquisition dry-run behavior, local +parser fixture dry-run behavior, focused stable .NET production-safety contract +tests, parity fixture presence, sample config safety, runbook consistency, and +whitespace cleanliness. -Do not add placeholder workflow YAML files without a task that defines the checks they should run. +The full Python test suite, full .NET contract suite, and full repository +public-safety validation are intentionally outside the default release gate. The +full .NET contract suite is outside the default release gate until known +deterministic parser assertion failures are resolved. They remain separate +tracked hardening items until baseline/noise cleanup, allowlist support, and +parser fixture determinism cleanup are available. + +The default workflow path is local-only. PostgreSQL integration validation is +manual and opt-in through the workflow input plus the +`CARBONOPS_RELEASE_GATE_RUN_INTEGRATION`, +`CARBONOPS_RUN_POSTGRESQL_INTEGRATION`, and externally supplied +`CARBONOPS_POSTGRESQL_TEST_DSN` controls. diff --git a/.github/workflows/agent-task-watcher.yml b/.github/workflows/agent-task-watcher.yml index b56e4d2f..7b6944bc 100644 --- a/.github/workflows/agent-task-watcher.yml +++ b/.github/workflows/agent-task-watcher.yml @@ -23,304 +23,44 @@ jobs: runs-on: ubuntu-latest steps: - - name: "Resolve PR metadata" + - name: "Check out repository" + uses: actions/checkout@v4 + + - name: "Resolve PR number" id: pr env: GH_TOKEN: ${{ github.token }} REPOSITORY: ${{ github.repository }} EVENT_NAME: ${{ github.event_name }} EVENT_PR_NUMBER: ${{ github.event.pull_request.number }} - EVENT_PR_MERGED: ${{ github.event.pull_request.merged }} INPUT_PR_NUMBER: ${{ inputs.pr_number }} run: | set -euo pipefail if [ "$EVENT_NAME" = "pull_request" ]; then PR_NUMBER="$EVENT_PR_NUMBER" - PR_MERGED="$EVENT_PR_MERGED" else PR_NUMBER="$INPUT_PR_NUMBER" - PR_MERGED="$(gh pr view "$PR_NUMBER" --repo "$REPOSITORY" --json mergedAt --jq '(.mergedAt // "") != ""')" fi echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT" - echo "pr_merged=$PR_MERGED" >> "$GITHUB_OUTPUT" - - if [ "$PR_MERGED" != "true" ]; then - echo "PR #$PR_NUMBER is not merged. No action taken." - echo "should_process=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - PR_BODY="$(gh pr view "$PR_NUMBER" --repo "$REPOSITORY" --json body --jq '.body // ""')" - TASK_ID="$(printf '%s\n' "$PR_BODY" | sed -nE 's/^[[:space:]]*Task-ID:[[:space:]]*([A-Za-z]+-[0-9A-Za-z-]+)[[:space:]]*$/\1/p' | tail -n 1)" - ISSUE_NUMBER="$(printf '%s\n' "$PR_BODY" | sed -nE 's/^[[:space:]]*Task-Issue:[[:space:]]*#?([0-9]+)[[:space:]]*$/\1/ip' | tail -n 1)" - - if [ -z "$TASK_ID" ]; then - echo "Missing Task-ID in PR #$PR_NUMBER body. No label changes applied." - echo "should_process=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - if [ -z "$ISSUE_NUMBER" ]; then - echo "Missing Task-Issue metadata such as Task-Issue: #123 in PR #$PR_NUMBER body. No label changes applied." - echo "should_process=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "task_id=$TASK_ID" >> "$GITHUB_OUTPUT" - echo "issue_number=$ISSUE_NUMBER" >> "$GITHUB_OUTPUT" - echo "should_process=true" >> "$GITHUB_OUTPUT" - - - name: "Mark issue as status merged" + - name: "Process merged task PR" id: mark_issue - if: steps.pr.outputs.should_process == 'true' env: GH_TOKEN: ${{ github.token }} REPOSITORY: ${{ github.repository }} PR_NUMBER: ${{ steps.pr.outputs.pr_number }} - TASK_ID: ${{ steps.pr.outputs.task_id }} - ISSUE_NUMBER: ${{ steps.pr.outputs.issue_number }} run: | set -euo pipefail - STATUS_LABELS=( - "status:blocked" - "status:ready" - "status:in-progress" - "status:in-review" - "status:needs-fix" - "status:merged" - ) - - ACTIVE_STATUS_LABELS=( - "status:blocked" - "status:ready" - "status:in-progress" - "status:in-review" - "status:needs-fix" - ) - - extract_field() { - local field_name="$1" - local body="$2" - - printf '%s\n' "$body" | awk -v field="$field_name" ' - { - line = $0 - sub(/^[[:space:]]+/, "", line) - needle = field ":" - - if (tolower(substr(line, 1, length(needle))) == tolower(needle)) { - print substr(line, length(needle) + 1) - exit - } - } - ' - } - - has_field() { - local field_name="$1" - local body="$2" - - printf '%s\n' "$body" | awk -v field="$field_name" ' - BEGIN { - found = 1 - } - - { - line = $0 - sub(/^[[:space:]]+/, "", line) - needle = field ":" - - if (tolower(substr(line, 1, length(needle))) == tolower(needle)) { - found = 0 - exit - } - } - - END { - exit found - } - ' - } - - parse_task_list() { - local raw_value="$1" - - printf '%s\n' "$raw_value" | - tr ',' '\n' | - sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//' | - awk ' - { - normalized = tolower($0) - } - - $0 == "" { - next - } - - normalized == "none" || normalized == "n/a" || $0 == "-" { - next - } - - { - print - } - ' - } - - find_issue_for_task() { - local task_id="$1" - - gh issue list \ - --repo "$REPOSITORY" \ - --state all \ - --search "$task_id in:title" \ - --json number,title \ - --limit 100 | - jq -r --arg prefix "[$task_id]" ' - [.[] | select(.title | startswith($prefix))] | - if length == 1 then .[0].number else empty end - ' - } - - issue_has_label() { - local issue_number="$1" - local label="$2" - - gh issue view "$issue_number" \ - --repo "$REPOSITORY" \ - --json labels | - jq -e --arg label "$label" 'any(.labels[].name; . == $label)' >/dev/null - } - - for LABEL in "${STATUS_LABELS[@]}"; do - gh issue edit "$ISSUE_NUMBER" --repo "$REPOSITORY" --remove-label "$LABEL" || true - done - - gh issue edit "$ISSUE_NUMBER" --repo "$REPOSITORY" --add-label "status:merged" - - ISSUE_BODY="$(gh issue view "$ISSUE_NUMBER" --repo "$REPOSITORY" --json body --jq '.body // ""')" - UNBLOCKS_RAW="$(extract_field "Unblocks" "$ISSUE_BODY")" - mapfile -t UNBLOCKED_TASK_IDS < <(parse_task_list "$UNBLOCKS_RAW") - - UPDATED_DEPENDENTS=() - BLOCKED_DEPENDENTS=() - SKIPPED_DEPENDENTS=() - - for DEPENDENT_TASK_ID in "${UNBLOCKED_TASK_IDS[@]}"; do - DEPENDENT_ISSUE_NUMBER="$(find_issue_for_task "$DEPENDENT_TASK_ID")" - - if [ -z "$DEPENDENT_ISSUE_NUMBER" ]; then - SKIPPED_DEPENDENTS+=("- \`$DEPENDENT_TASK_ID\`: no unique issue title beginning with \`[$DEPENDENT_TASK_ID]\` was found.") - continue - fi - - DEPENDENT_BODY="$(gh issue view "$DEPENDENT_ISSUE_NUMBER" --repo "$REPOSITORY" --json body --jq '.body // ""')" - - if ! has_field "Depends on" "$DEPENDENT_BODY"; then - BLOCKED_DEPENDENTS+=("- \`$DEPENDENT_TASK_ID\` (#$DEPENDENT_ISSUE_NUMBER) remains blocked; missing \`Depends on:\` metadata.") - continue - fi - - DEPENDS_ON_RAW="$(extract_field "Depends on" "$DEPENDENT_BODY")" - mapfile -t DEPENDENCY_TASK_IDS < <(parse_task_list "$DEPENDS_ON_RAW") - MISSING_DEPENDENCIES=() - - for DEPENDENCY_TASK_ID in "${DEPENDENCY_TASK_IDS[@]}"; do - DEPENDENCY_ISSUE_NUMBER="$(find_issue_for_task "$DEPENDENCY_TASK_ID")" - - if [ -z "$DEPENDENCY_ISSUE_NUMBER" ]; then - MISSING_DEPENDENCIES+=("$DEPENDENCY_TASK_ID") - continue - fi - - if ! issue_has_label "$DEPENDENCY_ISSUE_NUMBER" "status:merged"; then - MISSING_DEPENDENCIES+=("$DEPENDENCY_TASK_ID") - fi - done - - if [ "${#MISSING_DEPENDENCIES[@]}" -gt 0 ]; then - MISSING_LIST="$(printf '`%s`, ' "${MISSING_DEPENDENCIES[@]}")" - MISSING_LIST="${MISSING_LIST%, }" - BLOCKED_DEPENDENTS+=("- \`$DEPENDENT_TASK_ID\` (#$DEPENDENT_ISSUE_NUMBER) remains blocked; missing merged dependencies: $MISSING_LIST.") - continue - fi - - if issue_has_label "$DEPENDENT_ISSUE_NUMBER" "status:merged"; then - SKIPPED_DEPENDENTS+=("- \`$DEPENDENT_TASK_ID\` (#$DEPENDENT_ISSUE_NUMBER) is already \`status:merged\`; labels were left unchanged.") - continue - fi - - if issue_has_label "$DEPENDENT_ISSUE_NUMBER" "status:ready"; then - SKIPPED_DEPENDENTS+=("- \`$DEPENDENT_TASK_ID\` (#$DEPENDENT_ISSUE_NUMBER) is already \`status:ready\`; labels were left unchanged.") - continue - fi - - if issue_has_label "$DEPENDENT_ISSUE_NUMBER" "status:in-progress"; then - SKIPPED_DEPENDENTS+=("- \`$DEPENDENT_TASK_ID\` (#$DEPENDENT_ISSUE_NUMBER) is already \`status:in-progress\`; labels were left unchanged.") - continue - fi - - if issue_has_label "$DEPENDENT_ISSUE_NUMBER" "status:in-review"; then - SKIPPED_DEPENDENTS+=("- \`$DEPENDENT_TASK_ID\` (#$DEPENDENT_ISSUE_NUMBER) is already \`status:in-review\`; labels were left unchanged.") - continue - fi - - if issue_has_label "$DEPENDENT_ISSUE_NUMBER" "status:needs-fix"; then - SKIPPED_DEPENDENTS+=("- \`$DEPENDENT_TASK_ID\` (#$DEPENDENT_ISSUE_NUMBER) is already \`status:needs-fix\`; labels were left unchanged.") - continue - fi - - for LABEL in "${ACTIVE_STATUS_LABELS[@]}"; do - gh issue edit "$DEPENDENT_ISSUE_NUMBER" --repo "$REPOSITORY" --remove-label "$LABEL" || true - done - - gh issue edit "$DEPENDENT_ISSUE_NUMBER" --repo "$REPOSITORY" --add-label "status:ready" - - DEPENDENT_COMMENT_BODY="$(printf 'Agent Task Watcher marked this issue as `status:ready` because `%s` merged in PR #%s and all declared dependencies are `status:merged`.' "$TASK_ID" "$PR_NUMBER")" - gh issue comment "$DEPENDENT_ISSUE_NUMBER" --repo "$REPOSITORY" --body "$DEPENDENT_COMMENT_BODY" - - UPDATED_DEPENDENTS+=("- \`$DEPENDENT_TASK_ID\` (#$DEPENDENT_ISSUE_NUMBER) marked \`status:ready\`.") - done - - COMMENT_BODY="$(printf 'Agent Task Watcher marked this issue as `status:merged`.\n\nDetected task: `%s`\nMerged PR: #%s' "$TASK_ID" "$PR_NUMBER")" - - if [ "${#UNBLOCKED_TASK_IDS[@]}" -eq 0 ]; then - COMMENT_BODY+=$'\n\nNo dependent tasks were declared in `Unblocks:`.' - else - COMMENT_BODY+=$'\n\nDependent task update summary:' - - if [ "${#UPDATED_DEPENDENTS[@]}" -gt 0 ]; then - COMMENT_BODY+=$'\n\nMarked ready:\n' - for LINE in "${UPDATED_DEPENDENTS[@]}"; do - COMMENT_BODY+="$LINE"$'\n' - done - fi - - if [ "${#BLOCKED_DEPENDENTS[@]}" -gt 0 ]; then - COMMENT_BODY+=$'\n\nStill blocked:\n' - for LINE in "${BLOCKED_DEPENDENTS[@]}"; do - COMMENT_BODY+="$LINE"$'\n' - done - fi - - if [ "${#SKIPPED_DEPENDENTS[@]}" -gt 0 ]; then - COMMENT_BODY+=$'\n\nSkipped:\n' - for LINE in "${SKIPPED_DEPENDENTS[@]}"; do - COMMENT_BODY+="$LINE"$'\n' - done - fi - fi - - gh issue comment "$ISSUE_NUMBER" --repo "$REPOSITORY" --body "$COMMENT_BODY" - - echo "ready_count=${#UPDATED_DEPENDENTS[@]}" >> "$GITHUB_OUTPUT" + OUTPUT="$(python scripts/agent_task_watcher.py --repo "$REPOSITORY" --pr-number "$PR_NUMBER")" + printf '%s\n' "$OUTPUT" + READY_COUNT="$(printf '%s\n' "$OUTPUT" | sed -nE 's/^.*; ([0-9]+) ready downstream update\(s\)\.$/\1/p' | tail -n 1)" + echo "ready_count=${READY_COUNT:-0}" >> "$GITHUB_OUTPUT" - name: "Trigger ready task dispatch claim" - if: steps.pr.outputs.should_process == 'true' && steps.mark_issue.outputs.ready_count != '0' + if: steps.mark_issue.outputs.ready_count != '0' env: GH_TOKEN: ${{ github.token }} REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/release-validation.yml b/.github/workflows/release-validation.yml new file mode 100644 index 00000000..e431edb5 --- /dev/null +++ b/.github/workflows/release-validation.yml @@ -0,0 +1,55 @@ +name: Release validation + +on: + pull_request: + branches: + - develop + - main + workflow_dispatch: + inputs: + include_integration: + description: "Run opt-in integration validation" + required: true + default: "false" + type: choice + options: + - "false" + - "true" + +permissions: + contents: read + +jobs: + phase-1-release-gate: + name: Phase 1 release gate + runs-on: ubuntu-latest + env: + CARBONOPS_RELEASE_GATE_RUN_INTEGRATION: ${{ github.event_name == 'workflow_dispatch' && inputs.include_integration == 'true' && '1' || '0' }} + CARBONOPS_RUN_POSTGRESQL_INTEGRATION: ${{ github.event_name == 'workflow_dispatch' && inputs.include_integration == 'true' && '1' || '0' }} + CARBONOPS_POSTGRESQL_TEST_DSN: ${{ secrets.CARBONOPS_POSTGRESQL_TEST_DSN }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + + - name: Install Python package + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[postgresql]" + python -m pip install pytest + + - name: Restore .NET solution + run: dotnet restore src/dotnet/CarbonOps.Parser.sln + + - name: Run Phase 1 release validation gate + # Gate includes: dotnet test tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj --configuration Release --no-restore --filter + run: python scripts/release_validation_gate.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d24ffb9e..1f0bf627 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,58 @@ Contributions are welcome for documentation, examples, parser mappings, source discovery notes, database schema design, and implementation improvements. -CarbonOps-Parser is currently in early Phase 1. Please keep changes small, reviewable, and aligned with the documented scope. +CarbonOps-Parser is project-level production-ready only in the narrow supported scope documented in the production-ready verdict. Keep changes small, reviewable, testable, and aligned with that scope. Do not broaden production, compliance, legal, source-owner, or carbon-accounting correctness claims. + +## Open Issues + +Use GitHub issues for reproducible bugs, scoped feature proposals, +documentation improvements, source-family questions, and production-readiness +scope questions. Choose the closest issue template and include: + +- Implementation target: Python, .NET, Docs, Database, or Shared. +- Source family when relevant: GHG Protocol, DEFRA/DESNZ, or IPCC EFDB. +- Source version, file name, or fixture name when relevant. +- Concise reproduction steps for bugs. +- Expected result and actual result for bugs. +- Clear proposed scope and out-of-scope notes for features. + +Do not include confidential company data, credentials, private source files, +private connection strings, or production infrastructure details in public +issues. + +## Propose Features + +Feature requests should describe the user need, the affected runtime or docs +area, and the smallest useful increment. Source-specific ingestion, parser, +database, scheduler, downloader, package publication, or production promotion +work must be explicitly requested and reviewed before implementation. + +## Report Bugs + +Bug reports should include commands, inputs, versions, source family, and +reproduction steps when available. For PostgreSQL reports, redact connection +details and share only non-sensitive configuration shape or error summaries. + +## Improve Documentation + +Documentation pull requests are welcome when they clarify usage, scope, +runbooks, examples, troubleshooting, source discovery notes, or contribution +workflow. Documentation changes must preserve the narrow production-ready scope +and must not add package publishing claims or broader correctness claims. + +## Branches And Forks + +External contributors should fork the repository and open pull requests into +`develop`. Maintainers may also create focused branches in the main repository. +Use short branch names with one of these prefixes: + +- `fix/...` +- `feature/...` +- `docs/...` +- `chore/...` + +One task should map to one branch, one commit, and one pull request whenever +practical. ## Good Contribution Areas @@ -13,13 +64,39 @@ CarbonOps-Parser is currently in early Phase 1. Please keep changes small, revie - Adding implementation work when a task explicitly calls for it. - Improving issue and pull request quality. -## Before Opening A Pull Request +## Pull Request Process + +Open pull requests into `develop`. Use the pull request template and include +summary, scope, runtime impact, validation performed, PostgreSQL impact, docs +impact, secrets/artifacts checklist, production-ready claim checklist, and +maintainer-only merge acknowledgement. + +Before opening a pull request: - Keep the change focused. - Avoid adding dependencies unless the task explicitly requires them. - Avoid committing local-only files such as `codex/`. -- Do not include confidential data, credentials, or private source files. +- Do not include confidential data, credentials, private source files, or private connection strings. +- Do not commit generated artifacts, build outputs, caches, database dumps, downloaded source files, or local test output. - Update documentation when behavior or scope changes. +- Respect the production-ready scope documented in the final verdict. +- Run `git diff --check`. +- Run the relevant tests for the affected area. + +For Python package behavior or examples, run: + +```bash +python -m pytest +``` + +Documentation-only changes should run any targeted documentation tests named by +the task. If a requested validation command cannot be run, state why in the PR. + +## Merge Policy + +Only maintainers merge pull requests. Contributors should not imply that they +can self-merge, close issues on behalf of maintainers, or approve production +scope changes. CODEOWNERS identifies the maintainer review boundary. ## Implementation Targets diff --git a/README.md b/README.md index 81d8c3bb..39a0358d 100644 --- a/README.md +++ b/README.md @@ -2,41 +2,73 @@ ![CarbonOps-Parser banner](docs/assets/carbonops-parser-banner.svg) -Scheduled carbon factor ingestion and parsing reference project with Python and .NET implementation options. +Auditable public carbon emission factor ingestion and validation for climate-tech data infrastructure. -![Status](https://img.shields.io/badge/status-documentation%20baseline-2f6f88) +![Status](https://img.shields.io/badge/status-narrow%20production--ready-2f6f88) ![Phase](https://img.shields.io/badge/phase-Phase%201%20ingestion-4f7cac) -![Python](https://img.shields.io/badge/Python-planned-3776ab) -![.NET](https://img.shields.io/badge/.NET-planned-512bd4) -![PostgreSQL](https://img.shields.io/badge/PostgreSQL-Phase%201-336791) -![Docs](https://img.shields.io/badge/docs-in%20progress-5c7cfa) +![Python](https://img.shields.io/badge/Python-operator%20path-3776ab) +![.NET](https://img.shields.io/badge/.NET-parity%20evidence-512bd4) +![PostgreSQL](https://img.shields.io/badge/PostgreSQL-source--specific%20persistence-336791) ![Release](https://img.shields.io/badge/release-not%20published%20yet-lightgrey) -![CI](https://img.shields.io/badge/CI-not%20configured%20yet-lightgrey) +[![Release validation](https://github.com/ktalpay/CarbonOps-Parser/actions/workflows/release-validation.yml/badge.svg)](https://github.com/ktalpay/CarbonOps-Parser/actions/workflows/release-validation.yml) ![Package](https://img.shields.io/badge/package-not%20published%20yet-lightgrey) ![License](https://img.shields.io/badge/license-Apache--2.0-green) -CarbonOps-Parser is a standalone public technical project for scheduled carbon factor source ingestion and parsing. It checks selected public emission factor sources, detects source version or hash changes, archives raw source files, parses source-specific structures, validates parsed records, and stores ingestion metadata and source-specific records in PostgreSQL. +CarbonOps-Parser is a public, reviewable climate-tech data ingestion project for carbon accounting source data. It focuses on auditable ingestion, parsing, validation, diagnostics, and PostgreSQL operation for public emission factors from GHG Protocol, DEFRA/DESNZ, and IPCC EFDB, with parallel Python and .NET runtime evidence. The Python package includes a configured PostgreSQL ingestion runtime for operator-managed deployments. CarbonOps-Parser is project-level production-ready in the narrow supported scope documented in the final verdict: operator-run/scheduled Python ingestion, PostgreSQL-backed source-specific persistence, and .NET parity evidence through service entrypoint, config/redaction, schema/year-state, source-cycle orchestration, persistence, Docker PostgreSQL E2E, and persisted parity validation. The repository is intentionally conservative: default examples are deterministic and local-only, production runs require explicit configuration and credentials, and the project does not claim production carbon-accounting, legal, compliance, source-owner, or factor correctness. The project is independent from `carbonops-assistant`. It is not a continuation, module, plugin, or dependency of that project. +## Start Here + +- [Production operator runbook](docs/production-packaging-operator-runbook.md) - supported Python operator path, PostgreSQL readiness, cron scheduling, validation, and troubleshooting. +- [Final project production-ready verdict](docs/final-project-production-ready-verdict.md) - narrow production-ready scope and explicit non-claims. +- [Production parity contract](docs/production-parity-contract.md) - Python production path and .NET parity evidence. +- [Python runtime docs](docs/python-ingestion-local-runbook.md) - local Docker PostgreSQL ingestion runbook for the packaged Python path. +- [.NET runtime docs](src/dotnet/README.md) - .NET Worker Service path and parity-oriented runtime notes. +- [Database model](docs/database-model.md) and [PostgreSQL startup](docs/database-startup.md) - shared metadata plus source-specific table groups. +- [Documentation index](docs/index.md) - curated documentation map for operators, contributors, and reviewers. +- [Maintainer release/sync checklist](docs/maintainer-release-checklist.md) - develop-to-main, stale PR/issue cleanup, and first alpha/review readiness. +- [Contribution guide](CONTRIBUTING.md) - issues, features, forks, branches, pull requests, validation, secrets, artifacts, and maintainer-only merge policy. +- [Issue templates](.github/ISSUE_TEMPLATE) - bug reports, feature requests, documentation requests, and production-readiness questions. +- [Pull request guide](.github/pull_request_template.md) - PR checklist for scope, validation, runtime impact, PostgreSQL impact, docs, secrets, artifacts, and production-ready claims. + +## Problem Statement + +Public carbon emissions workflows often depend on emission factor spreadsheets, databases, and reference documents that change over time and vary by source family. CarbonOps-Parser exists to make carbon factor ingestion reviewable: source identity, version or checksum evidence, parser output, validation issues, persistence readiness, and diagnostics should be visible before any operational use. The project is infrastructure for data ingestion and validation, not an emissions calculator or compliance decision engine. + ## Current Status -CarbonOps-Parser is in early Phase 1. The repository currently emphasizes project documentation, architecture, schema contract notes, source support planning, and public contribution structure before parser implementation begins. +CarbonOps-Parser is in Phase 1 and has a narrow project-level production-ready status for the documented operator path. The repository contains an active Python ingestion runtime, .NET parity evidence, PostgreSQL schema/runtime boundaries, deterministic examples, local dry-run validation, and production operator documentation. The production-ready claim applies only to the scope documented in [Final Project Production-Ready Verdict](docs/final-project-production-ready-verdict.md) and [Production Parity Contract](docs/production-parity-contract.md). It is not a published package release. -Implementation work is planned for two independent paths: +### Explicit Non-Claims -- Python in `src/python` -- .NET in `src/dotnet` +CarbonOps-Parser does not claim to be: + +- A production carbon-accounting calculator or emissions reporting engine. +- Legal, compliance, audit, or regulatory advice. +- A source-owner correctness guarantee for GHG Protocol, DEFRA/DESNZ, IPCC EFDB, or any source document. +- A universal carbon factor model across all source families. +- A published package, unless release/package files and repository releases prove otherwise. + +| Area | Phase 1 completed capabilities | Phase 2 roadmap | +| --- | --- | --- | +| Source families | Local fixture and contract coverage for GHG Protocol, DEFRA/DESNZ, and IPCC EFDB boundaries. | Broader source onboarding rules, fixture policy, and source-family hardening slices. | +| Python | Source acquisition contracts, parser contracts, DEFRA/DESNZ fixture parser path, normalization handoff, persistence previews, diagnostics, and local dry-run CLI. | Runtime hardening, richer validation, controlled source expansion, and opt-in execution boundaries. | +| .NET | Service entrypoint, config/redaction, PostgreSQL schema/year-state, source-cycle orchestration, source-specific persistence, Docker PostgreSQL E2E, and persisted parity validation baselines. | Runtime parity review where shared behavior changes; package/service promotion remains separately scoped. | +| PostgreSQL | Schema descriptors, DDL preview, additive runtime bootstrap, configured Python source-family writes, idempotent duplicate skipping, and opt-in integration boundaries. | Broader migration, rollback, recovery, and operational hardening slices. | +| Safety posture | Local-only examples, non-destructive dry runs, preview-only SQL, no default network calls, and no production credentials. | Release-gate expansion and production-readiness reviews before live source or write-path promotion. | -Users who clone or fork the repository should be able to choose either implementation path. +Users who clone or fork the repository should be able to inspect either implementation path without relying on production infrastructure. ## Phase 1 Scope Phase 1 focuses on scheduled ingestion and parsing for: -- GHG Protocol -- DEFRA/DESNZ -- IPCC EFDB +| Source family | Public discovery value | Phase 1 posture | +| --- | --- | --- | +| GHG Protocol | Greenhouse Gas Protocol tools and factor workbooks used in carbon accounting workflows. | Source discovery/download contracts, parser contracts, normalized content parser boundaries, and parity tests. | +| DEFRA/DESNZ | UK government conversion factors used for carbon emissions and greenhouse gas reporting workflows. | Deterministic local fixture parser and normalization path plus source discovery/download contracts. | +| IPCC EFDB | IPCC Emission Factor Database source family with heterogeneous emission factor records. | Source discovery/download contracts, parser contracts, normalized content parser boundaries, and parity tests. | The intended Phase 1 workflow is: @@ -69,19 +101,24 @@ source schedule Phase 1 uses shared ingestion metadata tables plus source-specific master/detail tables. It does not force GHG Protocol, DEFRA/DESNZ, and IPCC EFDB into one canonical factor table. A normalized or search-oriented projection may be considered in a later phase. +The Python path under `src/carbonfactor_parser` holds the current implementation boundaries for source acquisition, parser execution, normalization, PostgreSQL persistence previews, configured PostgreSQL ingestion, local dry-run composition, and diagnostics. The .NET path under `src/dotnet` holds shared contract records and parity tests for the same public concepts. PostgreSQL support includes schema descriptors, bootstrap/readiness checks, DDL previews, opt-in integration boundaries, and the Python configured cycle runner. Parity, validation, diagnostics, and non-destructive dry-run behavior are part of the public architecture so reviewers can inspect the handoff from source artifact to parser output to persistence input without connecting to a database or making network calls. + ## Implementation Options ### Python -The Python implementation is planned first because it is practical for source discovery, spreadsheet inspection, parser mapping, validation, and data engineering workflows. - -The initial Python source adapter contracts and in-memory registry live under `src/carbonfactor_parser/source_adapters`. +The Python implementation is the active Phase 1 path for source discovery contracts, parser mapping, validation, normalization handoff, persistence previews, and data engineering workflows. -See [src/python/README.md](src/python/README.md). +The active Python runtime path lives under `src/carbonfactor_parser` and exposes the local dry-run CLI plus the configured `carbonops-parser run-ingestion` operator command for PostgreSQL-backed source-family ingestion. The initial Python source adapter contracts and in-memory registry live under `src/carbonfactor_parser/source_adapters`. ### .NET -The .NET implementation is planned as an independent Worker Service path that follows the same conceptual workflow with .NET-oriented application structure. +The .NET implementation is an independent Worker Service path that follows the +same conceptual workflow with .NET-oriented application structure. The reviewed +production scope treats .NET as parity-validated through its service +entrypoint, configuration/redaction, PostgreSQL schema/year-state, +source-cycle orchestration, source-specific persistence, Docker PostgreSQL E2E, +and persisted parity baselines. See [src/dotnet/README.md](src/dotnet/README.md). @@ -230,8 +267,41 @@ required. This quickstart is local dry-run only. It does not connect to PostgreSQL, write records, execute SQL, run migrations, perform network calls, trigger source acquisition, load config files, or require credentials. It does not make production DEFRA/DESNZ correctness claims. +## Production Operator Command + +The supported Python production entrypoint is: + +```bash +carbonops-parser run-ingestion \ + --config /etc/carbonops-parser/ingestion.production.json \ + --cycles 1 +``` + +Before running it, operators must provide explicit `CARBONOPS_POSTGRESQL_*` +environment values, including the password through an external secret boundary, +and validate the config: + +```bash +carbonops-parser validate-ingestion-config \ + --config /etc/carbonops-parser/ingestion.production.json \ + --cycles 1 +``` + +See [Production Packaging And Operator Runbook](docs/production-packaging-operator-runbook.md) +for install, configuration, PostgreSQL readiness, cron scheduling, +verification SQL, rerun/idempotency checks, and troubleshooting. + +This is the supported Python runtime production operator path. Project-level +production-ready is limited to the scope in +[Final Project Production-Ready Verdict](docs/final-project-production-ready-verdict.md) +and [Production Parity Contract](docs/production-parity-contract.md). + For boundary details, see [Local Dry-Run CLI Boundary](docs/local-dry-run-cli-boundary.md), [Local File Normalized Persistence Dry-Run Boundary](docs/local-file-normalized-persistence-dry-run-boundary.md), [PostgreSQL Persistence Preview Boundary](docs/postgresql-persistence-preview-boundary.md), and [Local Dry-Run Troubleshooting](docs/local-dry-run-troubleshooting.md). +To run the packaged Python ingestion cycle against local Docker PostgreSQL with +the three checked-in source fixture families, see +[Python Ingestion Local Runbook](docs/python-ingestion-local-runbook.md). + ## Developer Tests Run the lightweight Python test suite from the repository root: @@ -351,6 +421,10 @@ See [examples/example_acquisition_artifact_parser_input_mapping.py](examples/exa The parser package exposes `ParserInputContract`, `create_parser_input_contract()`, `validate_parser_input_contract()`, `ParserFileContentInput`, local parser file content loading helpers, parser file content validation helpers, `parse_defra_desnz_file_content()`, raw parsed record payload contracts, the `ParserAdapter` protocol, `NoopParserAdapter`, `ArtificialParserAdapter`, `DefraDesnzParserAdapter`, parser adapter registry helpers, parser execution planning and runner helpers, and parser execution result contracts for future parser adapter input handoff. The normalization package exposes parser execution handoff helpers, normalization input helpers for successful parser results with raw payloads, and a minimal DEFRA/DESNZ fixture normalization mapper. The persistence package exposes normalized result persistence input contracts, a logical PostgreSQL schema descriptor, a review-only DDL preview helper, a deterministic insert SQL builder, PostgreSQL persistence preview helpers, repository protocol/result contracts, an explicit caller-provided PostgreSQL options contract, a default-disabled PostgreSQL integration test boundary, and a PostgreSQL repository skeleton that returns unsupported results without database runtime behavior. The pipeline package exposes a local DEFRA/DESNZ fixture dry-run helper that composes those boundaries to produce `PersistenceInput` plus DDL preview metadata without DB or network behavior. These contracts keep acquisition metadata, already-loaded content, raw parser output, parser output metadata, normalization input, normalization handoff metadata, persistence input metadata, schema metadata, repository options metadata, integration test metadata, preview metadata, and repository result metadata separate; they do not include database connection behavior or full source-specific correctness claims. +## Examples And Fixtures + +The examples entry point is [examples/README.md](examples/README.md). It identifies deterministic local examples, including the checked-in DEFRA/DESNZ fixture used by the local dry-run quickstart, and separates real examples from future placeholders. + ## Source Support Each Phase 1 source family will have its own schedule, source version/hash check, parser, validation rules, archive layout, and source-specific tables. @@ -358,7 +432,7 @@ Each Phase 1 source family will have its own schedule, source version/hash check | Source family | Phase 1 role | Table group | | --- | --- | --- | | GHG Protocol | Source-specific parser and workbook/tool mapping | `ghg_*` | -| DEFRA/DESNZ | First planned ingestion slice after discovery | `defra_*` | +| DEFRA/DESNZ | Active checked-in fixture and source-specific ingestion slice | `defra_*` | | IPCC EFDB | Heterogeneous source discovery and parser mapping | `ipcc_*` | See [docs/source-support.md](docs/source-support.md) and [docs/source-discovery.md](docs/source-discovery.md). @@ -389,6 +463,11 @@ PostgreSQL is the Phase 1 persistence target. The model includes: See [docs/database-model.md](docs/database-model.md), [docs/database-startup.md](docs/database-startup.md), and [database/postgres/README.md](database/postgres/README.md). +PostgreSQL persistence uses shared ingestion metadata plus source-specific +master/detail table groups for GHG Protocol, DEFRA/DESNZ, and IPCC EFDB. That +layout preserves source-family structure for reviewable carbon emission factor +ingestion instead of claiming one universal carbon accounting factor model. + ## Documentation Map - [Architecture](docs/architecture.md) @@ -400,7 +479,11 @@ See [docs/database-model.md](docs/database-model.md), [docs/database-startup.md] - [Ingestion Metadata Model](docs/ingestion-metadata-model.md) - [Codex-Assisted Runs](docs/codex-runs/README.md) - [Engineering Standards](docs/engineering-standards.md) -- [Linux Service Setup](docs/linux-service-setup.md) +- [Production Packaging And Operator Runbook](docs/production-packaging-operator-runbook.md) +- [Maintainer Release/Sync Checklist](docs/maintainer-release-checklist.md) +- [Production Parity Contract](docs/production-parity-contract.md) +- [Final Project Production-Ready Verdict](docs/final-project-production-ready-verdict.md) +- [Legacy Linux Service Planning - not supported production scheduling](docs/linux-service-setup.md) - [Source Support](docs/source-support.md) - [Source Discovery](docs/source-discovery.md) - [Source Ingestion Boundaries](docs/source-ingestion-boundaries.md) @@ -484,6 +567,7 @@ See [docs/database-model.md](docs/database-model.md), [docs/database-startup.md] - [PostgreSQL Repository Disabled Execution Preview Boundary](docs/postgresql-repository-disabled-execution-preview-boundary.md) - [PostgreSQL Runtime Execution Gate Boundary](docs/postgresql-runtime-execution-gate-boundary.md) - [PostgreSQL Runtime Readiness Checklist](docs/postgresql-runtime-readiness-checklist.md) +- [Real-Source Smoke Mode](docs/real-source-smoke-mode.md) - [Parser To Normalization Handoff Boundary](docs/parser-to-normalization-handoff-boundary.md) - [Parser To Normalization Integration Recap](docs/parser-to-normalization-integration-recap.md) - [Source To Normalization Pipeline Recap](docs/source-to-normalization-pipeline-recap.md) @@ -512,7 +596,9 @@ See [docs/database-model.md](docs/database-model.md), [docs/database-startup.md] ## Roadmap Summary -Near-term work moves from documentation polish to schema scripts, Python source discovery, PostgreSQL startup checks, raw archive handling, and the first DEFRA/DESNZ ingestion slice. The .NET Worker Service path follows as an independent implementation option. +Near-term work keeps the narrow production-ready scope conservative while +separating package publication, infrastructure ownership, live-source expansion, +and future runtime promotion into separately reviewed tasks. See [docs/roadmap.md](docs/roadmap.md) and [docs/task-breakdown.md](docs/task-breakdown.md). diff --git a/SECURITY.md b/SECURITY.md index 7eb851f8..c9e0c44a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,10 +1,10 @@ # Security -CarbonOps-Parser is a public reference project in early Phase 1. Please avoid posting credentials, connection strings, private source files, or confidential data in public issues, pull requests, or discussions. +CarbonOps-Parser is a public reference project. Do not post credentials, connection strings, private source files, confidential data, or vulnerability details in public issues, pull requests, or discussions. ## Reporting A Concern -If you believe a change exposes sensitive information or creates an unsafe default, open a concise issue with non-sensitive reproduction details. If the repository owner provides a private contact path, use that path for sensitive reports. +If you believe a change exposes sensitive information, creates an unsafe default, or identifies a vulnerability, report it privately to the maintainer when a private contact path is available. If no private path is available, open only a non-sensitive public issue that asks for a private contact path. Do not include exploit details, secrets, private URLs, production credentials, database dumps, or private source data in the public report. ## Scope @@ -16,4 +16,6 @@ Useful reports include: - Dependency concerns once dependencies are introduced. - Documentation that could lead users to mishandle sensitive data. +No production credentials should be shared with the project. Operators own their infrastructure, credential storage, database access, network policy, monitoring, backup, and incident response. + This project does not provide operational security support. Response timing depends on maintainer availability. diff --git a/SUPPORT.md b/SUPPORT.md index d19c6ee5..53de9157 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -1,6 +1,6 @@ # Support -CarbonOps-Parser is a public reference project. Community support happens through repository issues and discussions when they are available. +CarbonOps-Parser is a public reference project. Supported questions can be opened through GitHub issues and discussions when they are available. Good support topics include: @@ -14,3 +14,5 @@ Good support topics include: Please do not share confidential data, credentials, private source files, or private connection strings. For issues, include the relevant source family, source version or file name when available, implementation target, and concise reproduction steps when relevant. + +Maintainer support is best effort and has no service-level agreement. Operators own their infrastructure, PostgreSQL configuration, credentials, scheduling, monitoring, backups, and production operations. diff --git a/config/carbonops.config.example.yaml b/config/carbonops.config.example.yaml index c1addcf6..4d4560b3 100644 --- a/config/carbonops.config.example.yaml +++ b/config/carbonops.config.example.yaml @@ -10,7 +10,14 @@ database: # Phase 1 implements PostgreSQL only. # mysql and mssql are reserved for future configuration compatibility. provider: postgres - connectionString: "Host=localhost;Port=5432;Database=carbonops_parser;Username=carbonops;Password=change-me" + host: "${CARBONOPS_POSTGRESQL_HOST}" + port: 5432 + database: "${CARBONOPS_POSTGRESQL_DATABASE}" + username: "${CARBONOPS_POSTGRESQL_USERNAME}" + # Secret value must be supplied by the environment or secret manager, not this file. + passwordEnvVar: CARBONOPS_POSTGRESQL_PASSWORD + applicationName: "${CARBONOPS_POSTGRESQL_APPLICATION_NAME}" + initialYear: "${CARBONOPS_POSTGRESQL_INITIAL_YEAR}" schema: carbonops storage: diff --git a/config/carbonops.ingestion.example.json b/config/carbonops.ingestion.example.json new file mode 100644 index 00000000..f26e5a30 --- /dev/null +++ b/config/carbonops.ingestion.example.json @@ -0,0 +1,96 @@ +{ + "archive_root": "./data/raw", + "enabled_source_families": [ + "ghg_protocol", + "defra_desnz", + "ipcc_efdb" + ], + "initial_year": 2024, + "cycle": { + "interval_seconds": 0, + "max_cycles": 4 + }, + "real_source_smoke": { + "allow_live_source_access": false + }, + "source_years": { + "ghg_protocol": { + "2024": { + "artifact_url": "examples/fixtures/ingestion/ghg_protocol_2024.csv", + "publication_url": "examples/fixtures/ingestion/ghg_protocol_2024.csv", + "title": "Local GHG Protocol fixture 2024", + "version_label": "local-ghg-2024", + "content_type": "text/csv", + "format_hint": "csv" + }, + "2025": { + "artifact_url": "examples/fixtures/ingestion/ghg_protocol_2025.csv", + "publication_url": "examples/fixtures/ingestion/ghg_protocol_2025.csv", + "title": "Local GHG Protocol fixture 2025", + "version_label": "local-ghg-2025", + "content_type": "text/csv", + "format_hint": "csv" + }, + "2026": { + "artifact_url": "examples/fixtures/ingestion/ghg_protocol_2026.csv", + "publication_url": "examples/fixtures/ingestion/ghg_protocol_2026.csv", + "title": "Local GHG Protocol fixture 2026", + "version_label": "local-ghg-2026", + "content_type": "text/csv", + "format_hint": "csv" + } + }, + "defra_desnz": { + "2024": { + "artifact_url": "examples/fixtures/ingestion/defra_desnz_2024.csv", + "publication_url": "examples/fixtures/ingestion/defra_desnz_2024.csv", + "title": "Local DEFRA/DESNZ fixture 2024", + "version_label": "local-defra-2024", + "content_type": "text/csv", + "format_hint": "csv" + }, + "2025": { + "artifact_url": "examples/fixtures/ingestion/defra_desnz_2025.csv", + "publication_url": "examples/fixtures/ingestion/defra_desnz_2025.csv", + "title": "Local DEFRA/DESNZ fixture 2025", + "version_label": "local-defra-2025", + "content_type": "text/csv", + "format_hint": "csv" + }, + "2026": { + "artifact_url": "examples/fixtures/ingestion/defra_desnz_2026.csv", + "publication_url": "examples/fixtures/ingestion/defra_desnz_2026.csv", + "title": "Local DEFRA/DESNZ fixture 2026", + "version_label": "local-defra-2026", + "content_type": "text/csv", + "format_hint": "csv" + } + }, + "ipcc_efdb": { + "2024": { + "artifact_url": "examples/fixtures/ingestion/ipcc_efdb_2024.csv", + "publication_url": "examples/fixtures/ingestion/ipcc_efdb_2024.csv", + "title": "Local IPCC EFDB fixture 2024", + "version_label": "local-ipcc-2024", + "content_type": "text/csv", + "format_hint": "csv" + }, + "2025": { + "artifact_url": "examples/fixtures/ingestion/ipcc_efdb_2025.csv", + "publication_url": "examples/fixtures/ingestion/ipcc_efdb_2025.csv", + "title": "Local IPCC EFDB fixture 2025", + "version_label": "local-ipcc-2025", + "content_type": "text/csv", + "format_hint": "csv" + }, + "2026": { + "artifact_url": "examples/fixtures/ingestion/ipcc_efdb_2026.csv", + "publication_url": "examples/fixtures/ingestion/ipcc_efdb_2026.csv", + "title": "Local IPCC EFDB fixture 2026", + "version_label": "local-ipcc-2026", + "content_type": "text/csv", + "format_hint": "csv" + } + } + } +} diff --git a/config/carbonops.ingestion.production.example.json b/config/carbonops.ingestion.production.example.json new file mode 100644 index 00000000..573d60fa --- /dev/null +++ b/config/carbonops.ingestion.production.example.json @@ -0,0 +1,48 @@ +{ + "archive_root": "/var/lib/carbonops-parser/raw-archive", + "enabled_source_families": [ + "ghg_protocol", + "defra_desnz", + "ipcc_efdb" + ], + "initial_year": 2024, + "cycle": { + "interval_seconds": 0, + "max_cycles": 1 + }, + "real_source_smoke": { + "allow_live_source_access": false + }, + "source_years": { + "ghg_protocol": { + "2024": { + "artifact_url": "/var/lib/carbonops-parser/sources/ghg_protocol_2024.csv", + "publication_url": "https://", + "title": "GHG Protocol reviewed artifact 2024", + "version_label": "", + "content_type": "text/csv", + "format_hint": "csv" + } + }, + "defra_desnz": { + "2024": { + "artifact_url": "/var/lib/carbonops-parser/sources/defra_desnz_2024.csv", + "publication_url": "https://", + "title": "DEFRA/DESNZ reviewed artifact 2024", + "version_label": "", + "content_type": "text/csv", + "format_hint": "csv" + } + }, + "ipcc_efdb": { + "2024": { + "artifact_url": "/var/lib/carbonops-parser/sources/ipcc_efdb_2024.csv", + "publication_url": "https://", + "title": "IPCC EFDB reviewed artifact 2024", + "version_label": "", + "content_type": "text/csv", + "format_hint": "csv" + } + } + } +} diff --git a/database/postgres/001_initial_schema.sql b/database/postgres/001_initial_schema.sql index c259eeae..2f8fff31 100644 --- a/database/postgres/001_initial_schema.sql +++ b/database/postgres/001_initial_schema.sql @@ -90,6 +90,18 @@ CREATE TABLE IF NOT EXISTS carbonops.carbon_job_locks ( CONSTRAINT uq_carbon_job_locks_source_lock UNIQUE (source_code, lock_key) ); +CREATE TABLE IF NOT EXISTS carbonops.source_family_year_states ( + source_family_year_state_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_family TEXT NOT NULL, + ingested_year INTEGER NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uq_source_family_year_states_family_year UNIQUE (source_family, ingested_year) +); + +CREATE INDEX IF NOT EXISTS idx_source_family_year_states_family_year + ON carbonops.source_family_year_states (source_family, ingested_year); + CREATE TABLE IF NOT EXISTS carbonops.defra_categories ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), source_version_id UUID NOT NULL REFERENCES carbonops.carbon_source_versions(id), diff --git a/database/postgres/README.md b/database/postgres/README.md index 2fe510e2..9e1b7a5d 100644 --- a/database/postgres/README.md +++ b/database/postgres/README.md @@ -19,6 +19,7 @@ The script creates the `carbonops` schema, enables `pgcrypto` for UUID generatio The schema baseline should include: - Shared ingestion metadata tables. +- Source-family ingestion year-state table for production E2E targeting. - DEFRA/DESNZ source-specific master/detail tables. - GHG Protocol source-specific master/detail tables. - IPCC EFDB source-specific master/detail tables. diff --git a/docs/agent/local-agent-launchd.md b/docs/agent/local-agent-launchd.md new file mode 100644 index 00000000..34c53d44 --- /dev/null +++ b/docs/agent/local-agent-launchd.md @@ -0,0 +1,111 @@ +# Local Agent launchd Job + +OPS-027 adds user-level macOS `launchd` installation for the local agent +supervisor. The job runs `scripts/local_agent_supervisor.py --config +--once` on a fixed interval and does not merge PRs, approve PRs, close issues, +delete branches, or delete worktrees. + +## Install + +Prepare a local config first, usually from `.agent/local-agent.example.json`. +Keep credentials in your normal local tools (`gh`, Git, Codex), not in this +file. + +```bash +cp .agent/local-agent.example.json .agent/local-agent.json +$EDITOR .agent/local-agent.json +``` + +Preview the generated plist and load command: + +```bash +scripts/install_local_agent_launchd.sh \ + --config .agent/local-agent.json \ + --repo-root "$PWD" \ + --python-bin "$(command -v python3)" \ + --dry-run +``` + +Install and load the user-level job: + +```bash +scripts/install_local_agent_launchd.sh \ + --config .agent/local-agent.json \ + --repo-root "$PWD" \ + --python-bin "$(command -v python3)" +``` + +The default label is `local.carbonops.agent.supervisor`. The default interval is +600 seconds. Override them with `--label` and `--interval-seconds`. + +The plist is written to: + +```text +~/Library/LaunchAgents/local.carbonops.agent.supervisor.plist +``` + +The installer does not require `sudo`. + +## Verify + +Check whether launchd knows about the job: + +```bash +launchctl print "gui/$(id -u)/local.carbonops.agent.supervisor" +``` + +Inspect the plist: + +```bash +plutil -p ~/Library/LaunchAgents/local.carbonops.agent.supervisor.plist +``` + +The job uses `RunAtLoad` and `StartInterval`, so it runs once after bootstrap and +then periodically. + +## Logs + +Supervisor runner logs are controlled by the supervisor config. launchd stdout +and stderr are written under the config `log_directory` when present, otherwise +under `/.logs` when `agents_root` is present. If neither can be +read, the installer uses this safe default: + +```text +~/FutureOps/Agents/CarbonOps-Parser/.logs +``` + +Expected launchd log files: + +```text +local-agent-supervisor.launchd.out.log +local-agent-supervisor.launchd.err.log +``` + +## Unload + +Preview uninstall actions: + +```bash +scripts/uninstall_local_agent_launchd.sh --dry-run +``` + +Unload the job and remove only its plist: + +```bash +scripts/uninstall_local_agent_launchd.sh +``` + +The uninstall script does not remove logs, config files, repository files, +branches, or worktrees. + +## Common Failures + +- `Bootstrap failed`: the job may already be loaded. Run + `scripts/uninstall_local_agent_launchd.sh`, then install again. +- `python executable not found or not executable`: pass an absolute Python path + with `--python-bin "$(command -v python3)"`. +- `Config file not found`: pass the intended local config with `--config`. +- `launchd job starts but exits`: read the stderr log first, then the supervisor + runner log directory from the local config. +- `gh` or Codex auth failures: fix local CLI authentication outside the plist; + do not place production credentials in the config or plist. diff --git a/docs/architecture.md b/docs/architecture.md index d76602fd..b20e5496 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,6 +2,8 @@ CarbonOps-Parser is organized around scheduled carbon factor source ingestion. The project checks configured public source families, detects source changes, downloads changed documents, archives raw files, parses source-specific structures, validates parsed records, and persists both shared ingestion metadata and source-specific records. +The public architecture is intentionally non-destructive by default. Local examples and dry-run paths can inspect carbon accounting emission factor fixtures, parser handoffs, validation issues, PostgreSQL schema metadata, and SQL previews without making network calls, connecting to PostgreSQL, executing SQL, loading production credentials, or claiming production carbon-accounting correctness. + ## Ingestion Workflow The conceptual workflow is: @@ -22,8 +24,8 @@ The service must not download, parse, or import source documents before the requ The repository contains two independent implementation paths: -- `src/python` -- `src/dotnet` +- Python package code in `src/carbonfactor_parser` +- .NET contracts and future Worker Service path in `src/dotnet` Both implementations should follow the same conceptual workflow, but they may use language-appropriate structure and tooling. @@ -33,6 +35,20 @@ The .NET implementation is an independent Worker Service path for users who pref The Python and .NET implementations should not depend on each other. +Shared contract names, statuses, issue semantics, diagnostics, and serialized field expectations are kept reviewable through parity tests and parity review documents. A runtime slice can be Python-first only when the task explicitly allows it and records the parity impact. + +## Runtime Readiness And Diagnostics + +Phase 1 separates readiness checks from execution: + +- Source acquisition can validate descriptors and plan local dry-run targets without acquiring content by default. +- Parser execution can consume already-loaded local content and report structured parser validation issues. +- Normalization and persistence handoffs keep raw parser output, normalized fields, source provenance, and persistence input metadata separate. +- PostgreSQL bootstrap and readiness boundaries expose schema descriptors, DDL previews, runtime config gates, disabled execution adapters, and opt-in integration checks before any database write path is promoted. +- Operational diagnostics expose run identity, status, issue counts, and failure context while avoiding production credentials and private source data. + +This posture lets first-time reviewers inspect data ingestion, validation, and PostgreSQL readiness behavior without source-owner credentials or production infrastructure. + ## Data Architecture Phase 1 uses shared ingestion metadata tables for operational tracking: diff --git a/docs/configuration-model.md b/docs/configuration-model.md index 6a46d71a..23730aed 100644 --- a/docs/configuration-model.md +++ b/docs/configuration-model.md @@ -18,7 +18,7 @@ Both implementations should follow the same conceptual model so users can choose The shared example contains: - `app`: application identity, environment label, and log level. -- `database`: provider, connection string, and PostgreSQL schema name. +- `database`: provider, split PostgreSQL connection fields, secret reference, and schema name. - `storage`: raw source file archive path. - `execution`: retry and single-instance lock settings. - `sources`: source-specific check, download, schedule, and import settings. @@ -63,9 +63,34 @@ Provider validation must happen before source checks, downloads, parsing, or imp | Field | Required | Purpose | Phase 1 guidance | | --- | --- | --- | --- | | `provider` | Yes | Selects the database provider. | Use `postgres`. | -| `connectionString` | Yes | Provides the PostgreSQL connection details. | Keep secrets out of committed real configs. | +| `host` | Yes | PostgreSQL host name. | Use an environment-specific value outside committed production config. | +| `port` | Yes | PostgreSQL port. | Must be an integer from 1 to 65535. | +| `database` | Yes | PostgreSQL database name. | Use an environment-specific value outside committed production config. | +| `username` | Yes | PostgreSQL user name. | Use an environment-specific value outside committed production config. | +| `passwordEnvVar` | Yes | Names the environment variable or secret binding that supplies the PostgreSQL password. | Use `CARBONOPS_PARSER_POSTGRES_PASSWORD`; do not store the password here. | | `schema` | Yes | Names the PostgreSQL schema for CarbonOps-Parser tables. | Use `carbonops`. | +Raw PostgreSQL connection strings are not accepted for production configuration because they commonly combine host, username, and password into one value that is easy to leak in diagnostics. Python and .NET both expect split non-secret fields plus `CARBONOPS_PARSER_POSTGRES_PASSWORD` as the secret boundary. + +## Production Environment Boundary + +Production startup validation is fail-closed. The runtime should validate these required keys before source checks, downloads, parsing, imports, or database execution: + +- `CARBONOPS_PARSER_ENV` +- `CARBONOPS_PARSER_DATABASE_PROVIDER` +- `CARBONOPS_PARSER_POSTGRES_HOST` +- `CARBONOPS_PARSER_POSTGRES_PORT` +- `CARBONOPS_PARSER_POSTGRES_DATABASE` +- `CARBONOPS_PARSER_POSTGRES_USERNAME` +- `CARBONOPS_PARSER_POSTGRES_PASSWORD` +- `CARBONOPS_PARSER_POSTGRES_SCHEMA` +- `CARBONOPS_PARSER_RAW_ARCHIVE_PATH` +- `CARBONOPS_PARSER_LOG_LEVEL` + +`CARBONOPS_PARSER_POSTGRES_PASSWORD` is the only required secret in this Phase 1 boundary. Validators may confirm that it is present, but validation results, logs, and diagnostics must not echo its value. Missing or invalid configuration messages should name the key and the expected shape only. + +The Python and .NET contracts are intentionally aligned: both validate caller-provided mappings, both require PostgreSQL provider `postgres`, both reject raw connection-string config, and neither reads environment variables, config files, credentials, opens a PostgreSQL connection, or runs SQL during validation. + ## Storage Section The `storage` section defines where raw source files are archived: @@ -171,7 +196,11 @@ app: database: provider: postgres - connectionString: "Host=localhost;Port=5432;Database=carbonops_parser;Username=carbonops;Password=change-me" + host: "${CARBONOPS_PARSER_POSTGRES_HOST}" + port: 5432 + database: "${CARBONOPS_PARSER_POSTGRES_DATABASE}" + username: "${CARBONOPS_PARSER_POSTGRES_USERNAME}" + passwordEnvVar: CARBONOPS_PARSER_POSTGRES_PASSWORD schema: carbonops storage: diff --git a/docs/database-model.md b/docs/database-model.md index ebce8056..590f97d3 100644 --- a/docs/database-model.md +++ b/docs/database-model.md @@ -2,6 +2,11 @@ CarbonOps-Parser uses PostgreSQL for Phase 1. The schema contract separates shared ingestion metadata from source-specific master/detail records. +PH-018 is the authoritative production-ready table contract for source-specific +master/detail persistence: +[PH-018 Real Production-Ready Ingestion Contract](ph-018-real-production-ready-ingestion-contract.md). +The model below remains conceptual background where it differs from PH-018. + The contract in this document is intentionally conceptual. The first SQL script should implement these table responsibilities without adding parser logic or runtime behavior to the documentation task that defines the contract. ## Design Rules diff --git a/docs/defra-desnz-production-e2e-ingestion.md b/docs/defra-desnz-production-e2e-ingestion.md new file mode 100644 index 00000000..469f6ccd --- /dev/null +++ b/docs/defra-desnz-production-e2e-ingestion.md @@ -0,0 +1,39 @@ +# DEFRA/DESNZ Production E2E Ingestion + +PH-014 adds the DEFRA/DESNZ year-based production E2E path behind explicit +runtime dependencies. + +## Runtime Behavior + +- PostgreSQL year state selects the target year: no DEFRA state targets 2024, + otherwise the latest ingested DEFRA year plus one. +- The DEFRA/DESNZ source adapter discovers the GOV.UK flat-file publication for + the selected target year. +- If the selected year is not configured or a flat-file link cannot be found, + the orchestrator returns `no_available_source_year` and performs no parse, + insert, or year-state update. +- Downloaded artifacts are archived under the configured target root with a + sidecar metadata JSON file containing source metadata, local path, size, and + SHA-256 checksum. +- The parser reads CSV or XLSX flat-file artifacts into normalized parser output + rows without making DEFRA/DESNZ factor correctness claims. +- Phase 2 data-quality validation runs before PostgreSQL insertion. +- Validated rows are inserted through the existing PostgreSQL normalized factor + runtime repository, which uses idempotent conflict handling. +- The DEFRA year-state row is recorded only after a successful insert result. + +## Docker PostgreSQL Integration Test + +The PH-014 integration test is opt-in and uses the existing repository +environment contract: + +```bash +CARBONOPS_RUN_POSTGRESQL_INTEGRATION=1 \ +CARBONOPS_POSTGRESQL_TEST_DSN=postgresql://user:password@localhost:5432/carbonops \ +python -m pytest tests/test_defra_desnz_production_e2e.py -m postgresql_integration +``` + +The integration test creates an isolated schema, bootstraps the Phase 1 runtime +tables, runs the DEFRA/DESNZ 2024 path against Docker PostgreSQL, reruns the +same source with reset year state, and asserts that normalized factor insertion +is idempotent. diff --git a/docs/engineering-standards.md b/docs/engineering-standards.md index 7c431d34..429efa75 100644 --- a/docs/engineering-standards.md +++ b/docs/engineering-standards.md @@ -32,7 +32,7 @@ Phase 1 should remain focused on scheduled source ingestion, raw file archiving, Changes should be small enough to review in one pass. -Each change should have a clear purpose, such as documentation baseline work, schema documentation, source discovery, parser mapping, validation behavior, or implementation-specific service wiring. +Each change should have a clear purpose, such as repository documentation, schema documentation, source discovery, parser mapping, validation behavior, or implementation-specific service wiring. Contributors should avoid combining unrelated work in one change. Documentation updates, schema changes, parser behavior, and service runtime changes should be split when practical. diff --git a/docs/final-phase1-production-readiness-review.md b/docs/final-phase1-production-readiness-review.md new file mode 100644 index 00000000..4411b54b --- /dev/null +++ b/docs/final-phase1-production-readiness-review.md @@ -0,0 +1,165 @@ +# Final Phase 1 Production Readiness Review + +## Executive Summary + +This review is a historical Phase 1 production readiness checkpoint for +CarbonOps-Parser. It consolidates the prior Phase 1 readiness reviews, the +production packaging/operator runbook, the release validation gate, and the +production release-candidate dry-run verification path. + +PROD-002 narrowed this historical verdict to the Python Phase 1 contract and +operator boundary. PROD-011 now issues the separate final project-level +production-ready verdict for the narrow supported scope in +[Final Project Production-Ready Verdict](final-project-production-ready-verdict.md). + +This historical verdict does not claim full production source correctness, +carbon-accounting correctness, live-source availability, complete parser +coverage for arbitrary upstream formats, managed production infrastructure, or +package registry publication. Those remain explicit accepted risks, operator +responsibilities, or backlog items. + +## Scope Reviewed + +- Python package/runtime boundaries for source acquisition, parser execution, + normalization, persistence handoff, service-host contracts, diagnostics, and + local validation commands. +- .NET contract surface for shared Phase 1 records, parser/source acquisition + contracts, PostgreSQL runtime configuration gates, service-host contracts, + and diagnostics parity. +- PostgreSQL Phase 1 schema, configuration, runtime execution gate, disabled + adapter, schema-bootstrap readiness, and preview-only persistence boundaries. +- Phase 1 source acquisition, source-download, parser, normalization, parsed + factor persistence writer, orchestration, service-host, and observability + readiness reviews. +- Production packaging/operator runbook, including install, configuration, + validation, run, stop, diagnosis, recovery, and PR-footer expectations. +- OPS-032 release validation gate and OPS-033 production RC dry-run + verification path. + +Review exclusions: + +- No product/runtime source code was modified for this task. +- No credentials, raw connection strings, live source endpoints, destructive + database operations, branch/worktree deletion, PR merge, or issue closure + were used by this review. +- Full Python suite, full .NET suite, and full repository public-safety scan + are not yet default release gate checks. + +## Assessment Matrix + +| Area | Assessment | Evidence | Remaining risk classification | +| --- | --- | --- | --- | +| Python runtime | Ready for Phase 1 contract/operator release. Local package commands, dry-run boundaries, orchestration contracts, service-host contracts, redaction helpers, and focused release checks are present. | `scripts/release_validation_gate.py`, `scripts/production_rc_verification.py`, local fixture/dry-run docs, RV-050 through RV-054 reviews. | Acceptable risk: full Python suite is outside the default gate until cleanup makes it a stable release check. | +| .NET contracts | Ready for Phase 1 contract parity release. Focused stable production-safety tests cover production config, diagnostics, and PostgreSQL runtime config gates. | .NET contracts project, parity fixtures, release gate focused `.NET` filter, RV-052 through RV-054 reviews. | Acceptable risk: full .NET contract suite is outside the default gate while known deterministic parser assertion failures remain. | +| PostgreSQL runtime/config/schema boundary | Ready as a fail-closed, reviewable boundary. Runtime config uses split fields and secret presence signaling; schema readiness is passive/check-only by default; runtime execution remains gated. | PostgreSQL runtime/config/schema docs, `config/carbonops.config.example.yaml`, runtime gate tests, production RC schema-bootstrap check. | Phase 2 backlog: complete runtime execution hardening, migration/rollback operations, transaction retry, conflict handling, and isolated integration promotion. | +| Source acquisition | Ready for Phase 1 local and dry-run operation. Source family selection, discovery/download boundaries, artifact metadata, and parser handoff are documented and tested without default live calls. | Source acquisition boundary docs, RV-050, release gate source acquisition validation and dry-run commands. | Acceptable risk: live-source availability, retry/rate-limit policy, authentication, and downloader scheduling are not proven by the default gate. | +| Parsing/normalization/persistence | Ready for Phase 1 deterministic local fixture and contract handoff. Parser outputs carry source identity/provenance; DEFRA/DESNZ local normalization and preview persistence paths exist; parsed factor persistence writer validates and deduplicates before repository handoff. | RV-050, RV-051, local parser dry-run with PostgreSQL preview, parity fixtures. | Phase 2 backlog: broaden source-specific normalization coverage and production parser fidelity; no carbon-accounting correctness claim. | +| Orchestration/service execution | Ready for Phase 1 sequential service-host contract release. Orchestration is explicit by source family, sequential, injected, status-rich, and gated by PostgreSQL readiness. Service host validates startup, prevents local overlap, and supports graceful stop semantics. | RV-052, RV-053, production RC service entrypoint and orchestrator dry-run checks. | Acceptable risk: no distributed lock, deployed worker binary, queue/cron runtime, cancellation propagation, retry/backoff, or replay policy yet. | +| Observability/diagnostics/redaction | Ready for contract-level operational diagnostics. Python and .NET diagnostic helpers redact PostgreSQL and credential-shaped values, include run/correlation identity, and expose structured run/failure context. | RV-054, parity fixture, production RC diagnostics redaction check. | Phase 2 backlog: metrics, traces, alerting, dashboards, retention policy, centralized log ingestion, and deployment log pipeline wiring. | +| Production config/secret boundary | Ready for documented operator use. Production configuration uses split non-secret fields, `CARBONOPS_PARSER_POSTGRES_PASSWORD` as the secret boundary, rejects raw connection strings, and documents credential-handling rules. | Production packaging/operator runbook, config example, production config tests, release gate sample config safety checks. | Acceptable risk: actual secret store integration and deployment-specific policy are outside repository scope. | +| Packaging/operator runbook | Ready as operator documentation and packaging guidance. Install, configure, validate, run, stop, diagnose, failure recovery, safe modes, and PR footer expectations are documented. | `docs/production-packaging-operator-runbook.md`, release gate runbook marker checks. | Acceptable risk: no packaged daemon command or .NET Worker Service executable is published yet. | +| CI release validation gate | Ready and passed for its default local-only safety scope. OPS-032 added a focused gate with static safety checks, focused Python tests, local dry-runs, focused stable .NET checks, optional integration opt-in, and whitespace validation. | OPS-032 release validation gate exists and passed in PR #550; `scripts/release_validation_gate.py --check-only` validates the current static gate path. | Acceptable risk: full Python suite, full .NET suite, and full public-safety scan are known non-default checks. | +| Production RC dry-run verification | Ready and passed for default local-only RC verification. OPS-033 added a dry-run verifier for production-like config validation, schema readiness, service-host entrypoint, orchestrator dry-run behavior, diagnostics redaction, and CI gate status. | OPS-033 production RC dry-run verification exists and passed in PR #551; `scripts/production_rc_verification.py --output-format json` validates the current RC path. | Acceptable risk: integration/live modes remain explicit opt-in and were not used for this review. | + +## Validation Evidence + +OPS-032 release validation gate exists and passed in PR #550. The gate is +implemented in `scripts/release_validation_gate.py` and is referenced by the +production packaging/operator runbook. Its default design is local-only and +non-destructive, with integration checks skipped unless explicitly opted in. + +OPS-033 production RC dry-run verification exists and passed in PR #551. The +verifier is implemented in `scripts/production_rc_verification.py` and is +referenced by the production packaging/operator runbook. Its default mode is +`dry-run`, with destructive operations, live source calls, and database +connections disabled. + +Known default-gate exclusions: + +- Full Python suite is not a default gate check yet. +- Full .NET suite is not a default gate check yet. +- Full repository public-safety scan is not a default gate check yet. +- PostgreSQL integration validation remains opt-in only with explicit + integration environment variables and an externally supplied test DSN. + +Validation required for this review: + +```bash +python scripts/release_validation_gate.py --check-only +python scripts/production_rc_verification.py --output-format json +git diff --check +``` + +## Remaining Risks + +### Blocker + +None identified in the repository documentation or validation path for the +Phase 1 production readiness review scope. + +### Acceptable Risk + +- The default release gate is focused and local-only; it does not run the full + Python suite. +- The default release gate runs focused stable .NET production-safety checks, + not the full .NET contract suite. +- The default release gate does not run the full repository public-safety scan. +- Live source availability, upstream document variability, downloader retry and + rate-limit behavior, source authentication, and live network reliability are + not proven by default validation. +- No production daemon command, .NET Worker Service executable, distributed + scheduler, distributed lock, retry/backoff policy, dead-letter handling, or + replay policy is released as part of Phase 1. +- Secret-store integration and deployment-specific credential policy remain + outside repository scope. +- Production database writes remain behind explicit runtime/gate boundaries and + were not executed during this review. + +### Phase 2 Backlog + +- Promote broader Python and .NET test coverage into the release gate after + known baseline/noise and deterministic parser assertion cleanup. +- Add allowlist-backed full public-safety validation to the default gate when + ready. +- Harden PostgreSQL runtime execution, transaction policy, migration/rollback + runbooks, integration promotion, and recovery behavior. +- Add production service packaging, deployed worker entrypoints, process + supervision, health probes, scheduler identity, distributed lock/lease + behavior, cancellation propagation, retry/backoff, and replay/dead-letter + semantics. +- Expand source-specific normalization and parser fidelity beyond current + deterministic local/fixture contracts. +- Add production observability beyond contract diagnostics: metrics, traces, + alerts, dashboards, retention policy, and centralized log ingestion. + +## Final Production Readiness Verdict + +Python runtime production path: yes, with accepted risks under the documented +operator boundary. + +.NET runtime production path: no. + +Project-level production-ready: yes, as of PROD-011 and only in the narrow +scope documented by the final project verdict. + +The Phase 1 Python contract, operator, validation, and dry-run verification path +is ready for release with the accepted risks above. Later PROD-003 through +PROD-010 work completed the .NET parity evidence required for the final +project-level verdict. + +## Release Recommendation + +Proceed with the Phase 1 release candidate under the current gate boundaries: + +- Treat `python scripts/release_validation_gate.py --check-only`, + `python scripts/production_rc_verification.py --output-format json`, and + `git diff --check` as required evidence for this review-only task. +- Keep full Python suite, full .NET suite, full public-safety scan, opt-in + PostgreSQL integration, and live source validation as separately tracked + hardening items until they are explicitly promoted. +- Do not enable production database writes, live source execution, new + scheduler/runtime packaging, or destructive recovery operations without a + separately scoped task and operator approval. + +Task-ID: RV-055 +Task-Issue: #501 diff --git a/docs/final-project-production-ready-verdict.md b/docs/final-project-production-ready-verdict.md new file mode 100644 index 00000000..71e4ce05 --- /dev/null +++ b/docs/final-project-production-ready-verdict.md @@ -0,0 +1,123 @@ +# Final Project Production-Ready Verdict + +## Final Verdict + +CarbonOps-Parser is project-level production-ready in the narrow supported +scope defined here. + +This verdict is yes for the reviewed Phase 1 production boundary: an +operator-run or scheduled Python runtime, plus .NET runtime parity evidence +through service entrypoint, configuration/redaction, PostgreSQL schema and +year-state, source-cycle orchestration, source-specific persistence, Docker +PostgreSQL E2E validation, and Python/.NET persisted parity validation. + +No final production-readiness blockers remain for this supported scope. + +## Supported Scope + +- Operator-run or scheduled Python ingestion through + `carbonops-parser run-ingestion --config --cycles 1`. +- Python PostgreSQL schema bootstrap, configured source-cycle runner, + idempotent rerun behavior, and source-specific master/detail persistence. +- Public carbon factor ingestion for GHG Protocol, DEFRA/DESNZ, and IPCC EFDB + through reviewed configured artifacts. +- PostgreSQL-backed source-specific master/detail persistence using the + `ghg_*`, `defra_*`, and `ipcc_*` table families. +- Explicit Python config/secret boundary through externally supplied + `CARBONOPS_POSTGRESQL_*` settings. +- .NET parity validated through the scheduled-worker service entrypoint, + production config loader and redaction boundary, PostgreSQL schema/year-state + primitives, source-cycle orchestration, source-specific master/detail insert, + three-source Docker PostgreSQL E2E baseline, and Python/.NET persisted parity + baseline. + +## Validation Evidence + +- Python production operator command exists: + `carbonops-parser run-ingestion`. +- Python schema bootstrap exists: + `bootstrap_postgresql_phase1_schema`. +- Python source-specific master/detail insert exists: + `PostgreSQLSourceFamilyRuntimeRepository`. +- Python configured cycle runner exists: + `run_configured_cycle_runner`. +- Python idempotency/rerun behavior is covered by configured cycle runner, + source-family repository, and persisted parity validation tests. +- GHG Protocol, DEFRA/DESNZ, and IPCC EFDB are covered by checked-in fixture + paths, source-family repositories, .NET Docker PostgreSQL E2E validation, and + Python/.NET persisted parity validation. +- .NET service entrypoint exists: + `src/dotnet/CarbonOps.Parser.Service`. +- .NET config/redaction boundary exists: + `validate-config`. +- .NET PostgreSQL schema/year-state boundary exists: + `validate-postgresql-runtime`. +- .NET source-cycle orchestration exists: + `preview-source-cycle` and `validate-source-cycle`. +- .NET source-specific master/detail insert and three-source Docker + PostgreSQL E2E evidence are covered by + `DotNetPostgreSQLIntegrationE2ETests`. +- Python/.NET persisted PostgreSQL parity evidence is covered by + `tests/test_postgresql_persisted_parity_validation.py`. + +## Remaining Non-Blocker Responsibilities + +- Operators own production infrastructure, process supervision, scheduler + locking, network access policy, backup/restore, monitoring, alerting, log + retention, credential rotation, and incident response. +- Operators must stage or approve source artifacts and live source access + before production runs. +- Operators must provide PostgreSQL credentials externally and must not commit + passwords, DSNs, tokens, or private URLs with credentials. +- Operators must run deployment-specific smoke checks against their approved + database/schema before first production use. +- Package registry publishing remains a separate release responsibility. + +## Explicit Non-Claims + +This verdict does not claim: + +- Legal or compliance correctness. +- Source-owner endorsement. +- Certified emission factor correctness. +- Carbon-accounting correctness. +- Managed backup, restore, monitoring, alerting, or production infrastructure + ownership by this repository. +- Package registry publication. +- Uncontrolled live source behavior. +- Production readiness outside GHG Protocol, DEFRA/DESNZ, IPCC EFDB, and the + PostgreSQL-backed source-specific Phase 1 persistence surface. + +## Production Operation Assumptions + +- Production runs use explicit reviewed configuration and an operator-owned + PostgreSQL database/schema. +- Production scheduling uses cron, a scheduler, or manual single-cycle + execution with external overlap prevention when needed. +- Runtime secrets are supplied through the deployment environment or secret + manager and are not written to repository files. +- Source artifacts are reviewed local paths, `file:`/`local:` URIs, or + explicitly approved HTTPS artifacts with live access opt-in. +- PostgreSQL bootstrap remains additive and idempotent; destructive migration + or cleanup is outside this verdict. + +## Task History Summary + +- PROD-001 completed the Python operator-run production baseline. +- PROD-002 corrected the project-level production-ready definition to require + Python and .NET parity evidence. +- PROD-003 added the .NET service/scheduled-worker entrypoint baseline. +- PROD-004 added the .NET production config loader and redaction boundary. +- PROD-005 added the .NET PostgreSQL schema bootstrap and year-state baseline. +- PROD-006 added .NET source-cycle orchestration for configured local + artifacts and parser handoff. +- PROD-007 added .NET source-specific master/detail insert behavior. +- PROD-008 added .NET idempotency and rerun E2E validation for the initial + Docker PostgreSQL path. +- PROD-009 extended the .NET Docker PostgreSQL E2E baseline across GHG + Protocol, DEFRA/DESNZ, and IPCC EFDB. +- PROD-010 added Python/.NET persisted PostgreSQL parity validation. +- PROD-011 issues this final project-level production-ready verdict. + +Task-ID: PROD-011 +Task-Issue: #634 diff --git a/docs/index.md b/docs/index.md index ca30e7d1..a04bae18 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,57 @@ # Documentation Index +CarbonOps-Parser is a public climate-tech data infrastructure project for auditable carbon accounting source ingestion and validation. It documents Python, .NET, and PostgreSQL boundaries for public emission factors from GHG Protocol, DEFRA/DESNZ, and IPCC EFDB without claiming production carbon-accounting correctness. + +## Start Here + +- [README](../README.md) - project positioning, safe quickstart, supported source families, status, and roadmap summary. +- [Final Project Production-Ready Verdict](final-project-production-ready-verdict.md) - narrow project-level production-ready verdict and explicit non-claims. +- [Production Parity Contract](production-parity-contract.md) - Python production path and .NET parity evidence. +- [Production Packaging And Operator Runbook](production-packaging-operator-runbook.md) - supported Python operator path, PostgreSQL readiness, cron scheduling, verification, and troubleshooting. + +## Runtime Runbooks + +- [Python Ingestion Local Runbook](python-ingestion-local-runbook.md) - local Docker PostgreSQL ingestion runbook for the packaged Python path. +- [.NET Runtime Docs](../src/dotnet/README.md) - .NET Worker Service path and parity-oriented runtime notes. +- [Architecture](architecture.md) - Python, .NET, PostgreSQL, validation, diagnostics, and dry-run boundaries. +- [Examples](../examples/README.md) - deterministic local examples and fixture entry points. + +## Database Model And PostgreSQL Startup + +- [Database Model](database-model.md) - shared ingestion metadata and source-specific table groups for GHG Protocol, DEFRA/DESNZ, and IPCC EFDB. +- [Database Startup](database-startup.md) - PostgreSQL startup expectations and local database notes. +- [PostgreSQL Runtime Readiness Checklist](postgresql-runtime-readiness-checklist.md) - database readiness checks for the supported PostgreSQL boundary. +- [PostgreSQL Phase 1 Schema Contract](postgresql-phase1-schema-contract.md) - Phase 1 schema contract details. + +## Phase 2 Roadmap And Review Gate + +- [Phase 2 Roadmap And Execution Boundary](phase2-roadmap.md) - next-stage roadmap and safety gates. +- [Phase 2 Runtime And Source Expansion Review Gate](phase2-review-gate.md) - review gate before runtime or source-family expansion. +- [Release Notes Draft](release-notes-draft.md) - draft notes for a first public alpha/review release. +- [Maintainer Release And Sync Checklist](maintainer-release-checklist.md) - develop-to-main alignment, stale PR/issue cleanup, and alpha/review tag readiness. + +## Contribution, Support, And Security + +- [Contribution Guide](../CONTRIBUTING.md) - issues, features, forks, branches, pull requests, validation, secrets, artifacts, and maintainer-only merge policy. +- [Support Policy](../SUPPORT.md) - support boundaries and safe information-sharing guidance. +- [Security Policy](../SECURITY.md) - responsible disclosure and no-secret expectations. +- [Issue Templates](../.github/ISSUE_TEMPLATE) - bug reports, feature requests, documentation requests, and production-readiness questions. +- [Pull Request Template](../.github/pull_request_template.md) - PR checklist for scope, validation, runtime impact, PostgreSQL impact, docs, secrets, artifacts, and production-ready claims. + +## Public Discoverability + +Repository topics cannot be configured through repository files, but maintainers can use these suggested GitHub topics when publishing or reviewing the public repository: + +`carbon-accounting`, `emission-factors`, `carbon-emissions`, `ghg-protocol`, `defra-desnz`, `ipcc-efdb`, `climate-tech`, `data-ingestion`, `python`, `dotnet`, `postgresql`, `postgres`, `etl`, `data-validation`, `open-source` + +Search-friendly project description: + +> Auditable public carbon emission factor ingestion and validation for climate-tech data infrastructure, with Python and .NET contracts, PostgreSQL readiness, non-destructive dry runs, and documented support boundaries for GHG Protocol, DEFRA/DESNZ, and IPCC EFDB. + +Discovery wording should stay conservative. The repository may describe source ingestion, parsing, validation, diagnostics, dry-run previews, and reviewable PostgreSQL readiness. It should not claim production carbon-accounting correctness, compliance correctness, legal correctness, source-owner correctness, or factor correctness. + +## Full Map + - [Architecture](architecture.md) - [Configuration Model](configuration-model.md) - [Background Job Model](background-job-model.md) @@ -8,6 +60,8 @@ - [Ingestion Metadata Model](ingestion-metadata-model.md) - [Ingestion Contracts](ingestion-contracts.md) - [Engineering Standards](engineering-standards.md) +- [Production Packaging And Operator Runbook](production-packaging-operator-runbook.md) +- [Python Ingestion Local Runbook](python-ingestion-local-runbook.md) - [Source Support](source-support.md) - [Source Discovery](source-discovery.md) - [Source Ingestion Boundaries](source-ingestion-boundaries.md) @@ -118,6 +172,15 @@ - [Stabilization Checkpoint](stabilization-checkpoint.md) - [Production Readiness Gap Analysis](production-readiness-gap-analysis.md) - [Production Readiness Sequencing Roadmap](production-readiness-sequencing-roadmap.md) +- [Production Parity Contract](production-parity-contract.md) +- [Final Project Production-Ready Verdict](final-project-production-ready-verdict.md) +- [Production E2E Ingestion Readiness Contract](production-e2e-ingestion-readiness-contract.md) +- [PH-017 Production E2E Docker PostgreSQL Release Validation](ph-017-production-e2e-docker-postgresql-release-validation.md) +- [PH-018 Real Production-Ready Ingestion Contract](ph-018-real-production-ready-ingestion-contract.md) +- [Final Phase 1 Production Readiness Review](final-phase1-production-readiness-review.md) +- [Phase 2 Roadmap And Execution Boundary](phase2-roadmap.md) +- [Phase 2 Runtime And Source Expansion Review Gate](phase2-review-gate.md) +- [Release Notes Draft](release-notes-draft.md) - [Repository Navigation Guide](repository-navigation-guide.md) - [Review Readiness Checklist](review-readiness-checklist.md) - [Documentation Map Consistency Checklist](documentation-map-consistency-checklist.md) diff --git a/docs/ingestion-metadata-model.md b/docs/ingestion-metadata-model.md index 2ff3834c..fc12ba15 100644 --- a/docs/ingestion-metadata-model.md +++ b/docs/ingestion-metadata-model.md @@ -138,9 +138,9 @@ The intended architecture is shared ingestion metadata plus source-specific stor ## Python And .NET Implementation Expectations -Python is expected to implement the first ingestion metadata behavior because it is planned first for source discovery, file handling, and parser experimentation. +Python implements the first ingestion metadata behavior because it is the active runtime path for source discovery, file handling, parser experimentation, and the current operator-run ingestion workflow. -The .NET implementation should aim for conceptual parity later. It should use language-appropriate structure while preserving the same metadata concepts and source traceability expectations. +The .NET implementation should preserve conceptual parity within the reviewed scope and track future parity changes through explicit review tasks. It should use language-appropriate structure while preserving the same metadata concepts and source traceability expectations. The two implementation paths should remain independent and should not share runtime code. diff --git a/docs/limitations.md b/docs/limitations.md index 31954f3f..40f18e2b 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -19,10 +19,10 @@ CarbonOps-Parser does not: Phase 1 is limited to: - PostgreSQL as the implemented database provider. -- GHG Protocol, DEFRA/DESNZ, and IPCC EFDB as the planned source families. +- GHG Protocol, DEFRA/DESNZ, and IPCC EFDB as the supported Phase 1 source families. - Shared ingestion metadata tables. - Source-specific master/detail tables. -- Documentation, schema, discovery, and early ingestion slices. +- Documentation, schema, discovery, operator-run ingestion, and bounded source-family ingestion slices. The conceptual configuration model includes `mysql` and `mssql`, but those providers are not implemented in Phase 1. diff --git a/docs/linux-service-setup.md b/docs/linux-service-setup.md index a3000c41..57fcd391 100644 --- a/docs/linux-service-setup.md +++ b/docs/linux-service-setup.md @@ -1,12 +1,23 @@ # Linux Service Setup -CarbonOps-Parser is intended to run as a Linux background service in either the Python or .NET implementation path. +This document is legacy planning material for a possible future Linux service. +It is not the supported production scheduling mode for PROD-001. Supported +production scheduling is cron or manual one-cycle execution of the packaged +Python command documented in the production operator runbook. -This document is conceptual for the documentation baseline. Implementation-specific service files should be added after the runtime entry points exist. +This document is a non-installing template for operator planning. It does not +install a service, enable a service, start a service, create a user, read +configuration, load credentials, connect to PostgreSQL, run SQL, or download +sources. It must not be used to imply daemon, system service, or installer +support for production operation. The .NET runtime has a directly runnable +scheduled-worker entrypoint baseline, but its run command remains a +not-yet-implemented placeholder and does not make .NET production-ready. +Implementation-specific service files should be added only after a future task +explicitly scopes and reviews service support for the selected implementation. ## Service Responsibilities -A Linux service setup should define: +A future Linux service setup would need to define: - Working directory - Environment variables @@ -19,7 +30,12 @@ A Linux service setup should define: ## Conceptual systemd Unit -The exact command depends on the selected implementation. A future Python service might run a Python module, while a future .NET service might run a Worker Service binary. +The example below is conceptual and not a supported production unit. The exact +command would depend on a future reviewed service implementation. Until that +implementation exists, keep `ExecStart` as an operator-owned placeholder and use +cron or manual one-cycle execution for supported production scheduling. Do not +point production systemd units at the PROD-003 .NET `run-once` placeholder for +real ingestion. ```ini [Unit] @@ -28,28 +44,44 @@ After=network.target [Service] WorkingDirectory=/opt/carbonops-parser -Environment=CARBONOPS_PARSER_ENV=default -ExecStart=/opt/carbonops-parser/run-service +EnvironmentFile=/etc/carbonops-parser/runtime.env +ExecStart=/opt/carbonops-parser/ Restart=on-failure User=carbonops Group=carbonops +KillSignal=SIGTERM +TimeoutStopSec=300 [Install] WantedBy=multi-user.target ``` +The environment file path above is illustrative. It must be created and managed +outside the repository and must not be committed with environment-specific +values. + ## Management Commands -Typical service management commands: +Typical service management commands after a future reviewed unit is installed: ```bash sudo systemctl daemon-reload -sudo systemctl enable carbonops-parser sudo systemctl start carbonops-parser sudo systemctl status carbonops-parser sudo journalctl -u carbonops-parser -f +sudo systemctl stop carbonops-parser ``` ## Notes -Linux service documentation should explain how configuration is provided, where raw files are archived, and how logs are reviewed. Deployment hardening is outside the Phase 1 documentation baseline. +Linux service documentation must explain how configuration is provided, where +raw files are archived, how logs are reviewed, and how graceful stop is handled. +Do not add automatic enablement, destructive cleanup, branch or worktree +cleanup, schema deletion, or ad hoc database mutation to service management +steps. + +For the supported production install, configure, validate, run, stop, diagnose, +and rollback flow, see +[Production Packaging And Operator Runbook](production-packaging-operator-runbook.md). +For project-level Python/.NET parity requirements, see +[Production Parity Contract](production-parity-contract.md). diff --git a/docs/maintainer-release-checklist.md b/docs/maintainer-release-checklist.md new file mode 100644 index 00000000..bd0a41e0 --- /dev/null +++ b/docs/maintainer-release-checklist.md @@ -0,0 +1,35 @@ +# Maintainer Release And Sync Checklist + +This checklist is for maintainers preparing CarbonOps-Parser branch synchronization, public repository cleanup, or a first alpha/review tag. It does not authorize automatic release creation, issue closure, or branch merges. + +## Branch State + +- Verify `develop` contains the current accepted project state for the Python operator path, PostgreSQL-backed source-specific persistence, supported Phase 1 source families, and .NET parity evidence. +- Confirm the working tree is clean before any branch sync, release tag, or public handoff. +- Merge or sync `develop` to `main` only after review approval and repository state confirmation. +- Confirm the README on the default branch no longer contains stale baseline-status wording that contradicts the accepted project state. + +## Pull Requests And Issues + +- Review open pull requests for stale duplicates, especially duplicate documentation polish or baseline-cleanup PRs that are superseded by the accepted branch state. +- Close or supersede stale duplicate PRs only after confirming their useful changes are already represented or intentionally rejected. +- Review open issues whose GitHub state and labels disagree, such as issues that remain open while carrying a `status:merged` label. +- Update issue labels or close issues only through the normal maintainer workflow; do not use this checklist as automatic closure authority. + +## Release Readiness + +- Create a first alpha/review tag only after branch state, duplicate PRs, issue labels, README status, and validation evidence are clean. +- Confirm release notes describe the narrow project-level production-ready scope without expanding into carbon-accounting, legal, compliance, source-owner, or package-publication claims. +- Confirm package publication status is accurate before mentioning any install channel beyond repository-local or editable installs. +- Confirm release validation references exact commands and does not rely on private infrastructure. + +## Repository Hygiene + +- Confirm no secrets, DSNs, API keys, local database credentials, private source files, or environment-specific values are committed. +- Confirm no generated artifacts, caches, `.venv`, `__pycache__`, `.pytest_cache`, coverage outputs, screenshots, binaries, database dumps, or downloaded source files are committed. +- Confirm docs and templates still require scoped PRs, validation evidence, no production credentials, no generated artifacts, and no production-ready claim expansion without an explicit review task. +- Confirm public examples remain deterministic and local-only unless a reviewed task explicitly changes that boundary. + +## Handoff Notes + +Record the final branch, commit, validation commands, skipped checks with reasons, open maintainer follow-ups, and tag decision in the release or sync handoff. diff --git a/docs/ph-017-production-e2e-docker-postgresql-release-validation.md b/docs/ph-017-production-e2e-docker-postgresql-release-validation.md new file mode 100644 index 00000000..483fa032 --- /dev/null +++ b/docs/ph-017-production-e2e-docker-postgresql-release-validation.md @@ -0,0 +1,216 @@ +# PH-017 Production E2E Docker PostgreSQL Release Validation + +PH-018 supersedes the production-ready interpretation in this PH-017 review. +The PH-017 evidence remains useful Docker PostgreSQL test-harness validation, +but it is not sufficient for the real production-ready definition documented in +[PH-018 Real Production-Ready Ingestion Contract](ph-018-real-production-ready-ingestion-contract.md). +PROD-002 further narrows the historical verdict below: it is not a current +project-level production-ready claim. The Python runtime has a production +operator path, while the .NET runtime is not production-ready yet. Project-level +production-ready is blocked until both runtimes satisfy +[Production Parity Contract](production-parity-contract.md). + +## Verdict + +Historical PH-017 verdict: production-ready with accepted risks for the Docker +PostgreSQL validation boundary. Current project-level production-ready: no. + +PH-017 requires final production E2E validation against Docker PostgreSQL on the +user's Apple M3 machine. M3 Docker PostgreSQL validation is now complete. + +This verdict is limited to the PH-017 release-validation decision. It does not +claim source-owner correctness, factor correctness, legal correctness, or +compliance correctness. + +## Scope Reviewed + +This review assessed the production E2E path for: + +- GHG Protocol: `ghg_protocol`. +- DEFRA/DESNZ: `defra_desnz`. +- IPCC EFDB: `ipcc_efdb`. + +The review covered the year-based orchestrator, source-family production E2E +adapters, parser boundaries, validation boundaries, PostgreSQL schema bootstrap, +PostgreSQL year-state storage, normalized factor insert behavior, release gate, +production RC verifier, and Docker PostgreSQL runbook. + +No production credentials, production DSNs, live customer data, destructive +database operations, PR merge, PR approval, issue closure, branch deletion, or +worktree deletion were used. + +## Docker PostgreSQL Evidence + +Required PH-017 Docker PostgreSQL validation status: passed. + +```bash +CARBONOPS_RUN_POSTGRESQL_INTEGRATION=1 \ +CARBONOPS_POSTGRESQL_TEST_DSN='' \ +python -m pytest -m postgresql_integration \ + tests/test_ghg_protocol_production_e2e.py \ + tests/test_defra_desnz_production_e2e.py \ + tests/test_ipcc_efdb_production_e2e.py \ + tests/test_postgresql_runtime_year_state.py +# 4 passed, 22 deselected +``` + +M3 validation evidence: + +- Docker PostgreSQL E2E integration passed on the user's Apple M3 machine. +- The focused opt-in PostgreSQL integration run reported `4 passed, 22 + deselected`. +- The run used the canonical external controls and did not record a DSN, + password, credential, token, or secret value. +- Docker PostgreSQL evidence covers all three PH-017 production E2E source + families and the runtime year-state path. + +## Repository Evidence Found + +Existing focused tests and runtime boundaries support the intended production +E2E behavior: + +- `tests/test_production_e2e_year_orchestrator.py` covers all three canonical + source families, default `2024` selection when no year exists, `2024 -> 2025`, + `2025 -> 2026`, `2026 -> 2027`, and `no_available_source_year` safe no-op + behavior. +- `tests/test_ghg_protocol_production_e2e.py` covers GHG Protocol local + download/archive metadata, parse, validation, insert handoff, next-year + targeting, future-year no-op behavior, idempotent duplicate replay, and an + opt-in Docker PostgreSQL integration test. +- `tests/test_defra_desnz_production_e2e.py` covers DEFRA/DESNZ local + download/archive metadata, parse, validation, insert handoff, next-year + targeting, future-year no-op behavior, idempotent duplicate replay, XLSX flat + file parsing, and an opt-in Docker PostgreSQL integration test. +- `tests/test_ipcc_efdb_production_e2e.py` covers IPCC EFDB local + download/archive metadata, parse, validation, insert handoff, next-year + targeting, future-year no-op behavior, idempotent duplicate replay, and an + opt-in Docker PostgreSQL integration test. +- `src/carbonfactor_parser/persistence/postgresql_runtime_schema_bootstrap.py` + uses additive `CREATE TABLE IF NOT EXISTS` and `CREATE INDEX IF NOT EXISTS` + statements for required Phase 1 tables. +- `src/carbonfactor_parser/persistence/postgresql_year_state_repository.py` + returns `2024` for no existing year by default and otherwise computes + `latest_year + 1`. +- `src/carbonfactor_parser/persistence/postgresql_normalized_factor_repository.py` + inserts normalized factor records with idempotent conflict handling and + returns safe structured database/config errors with redacted messages. +- `scripts/release_validation_gate.py` redacts password, token, secret, and + PostgreSQL DSN-shaped output. +- `scripts/production_rc_verification.py` validates production-like config, + passive schema readiness, service entrypoint wiring, dry-run orchestration, + diagnostics redaction, and release-gate status without default database + connections or live source calls. + +## Local Validation Run + +Commands run in this session: + +```bash +dotnet restore +# completed + +python scripts/release_validation_gate.py +# passed + +focused .NET production-safety contract tests +# 17 passed + +python scripts/production_rc_verification.py +# Passed true + +python -m pytest +# 2062 passed + +git diff --check +# passed +``` + +The executable release gate, production RC verifier, default Python test suite, +focused .NET production-safety contract tests, and whitespace check passed. + +## Required Behavior Assessment + +| Required PH-017 behavior | Assessment | +| --- | --- | +| First run checks/creates database schema safely | Implemented by additive runtime schema bootstrap and verified by focused Docker PostgreSQL integration evidence. | +| No data targets 2024 per source family | Covered by local orchestrator and per-family E2E tests, with Docker PostgreSQL evidence for the opt-in production E2E path. | +| Existing 2024 targets 2025 | Covered by local orchestrator and per-family E2E tests, with Docker PostgreSQL evidence for the opt-in production E2E path. | +| Existing 2025 targets 2026 | Covered by local orchestrator tests and release validation evidence. | +| Existing 2026 targets 2027 | Covered by local orchestrator tests and release validation evidence. | +| 2027 unavailable no-ops with `no_available_source_year` | Covered by local orchestrator behavior and release validation evidence. | +| Available target year downloads, archives metadata, parses, validates, inserts, then updates latest year only after successful insert | Covered by local per-family E2E tests, orchestrator ordering, and Docker PostgreSQL opt-in evidence. | +| Repeated runs are idempotent and do not duplicate records | Covered by local per-family duplicate replay tests and Docker PostgreSQL opt-in evidence. | +| DB errors and config errors are safe/redacted | Covered by release gate, RC verifier, and repository error-shaping tests. | + +## Source Family Assessment + +### GHG Protocol + +Local evidence exists for `2024` first-run ingestion, `2025` next-year +selection, future-year unavailable no-op handling, archive metadata creation, +parse/validate/insert handoff, and duplicate replay idempotency. + +PH-017 Docker PostgreSQL evidence now includes the focused GHG Protocol +production E2E integration path on the required M3 Docker PostgreSQL +environment. + +### DEFRA/DESNZ + +Local evidence exists for `2024` first-run ingestion, `2025` next-year +selection, `2026` and `2027` unavailable no-op handling, archive metadata +creation, CSV/XLSX parse support, validation/insert handoff, and duplicate +replay idempotency. + +PH-017 Docker PostgreSQL evidence now includes the focused DEFRA/DESNZ +production E2E integration path on the required M3 Docker PostgreSQL +environment. + +### IPCC EFDB + +Local evidence exists for `2024` first-run ingestion, `2025` next-year +selection, future-year unavailable no-op handling, archive metadata creation, +parse/validate/insert handoff, and duplicate replay idempotency. + +PH-017 Docker PostgreSQL evidence now includes the focused IPCC EFDB production +E2E integration path on the required M3 Docker PostgreSQL environment. + +## Accepted Risks + +The PH-017 release verdict accepts these explicit risks: + +- Live source URL/default discovery remains a release risk. +- No source-owner correctness claim is made. +- No factor correctness claim is made. +- No legal correctness claim is made. +- No compliance correctness claim is made. + +## Release Decision + +The repository has local, opt-in Docker PostgreSQL, release-gate, production RC, +focused .NET production-safety contract, and default Python test evidence for +the PH-017 behavior. The release decision for PH-017 is therefore: + +Historical PH-017 verdict: production-ready with accepted risks for the Docker +PostgreSQL validation boundary. Current project-level production-ready: no. + +Merge readiness evidence: + +1. Docker PostgreSQL E2E integration: `4 passed, 22 deselected`. +2. `dotnet restore`: completed. +3. `python scripts/release_validation_gate.py`: passed. +4. Focused .NET production-safety contract tests: `17 passed`. +5. `python scripts/production_rc_verification.py`: `Passed true`. +6. `python -m pytest`: `2062 passed`. +7. `git diff --check`: passed. + +## PR Body Footer + +The pull request body must end with: + +```text +Task-ID: PH-017 +Task-Issue: #584 +``` + +Task-ID: PH-017 +Task-Issue: #584 diff --git a/docs/ph-018-real-production-ready-ingestion-contract.md b/docs/ph-018-real-production-ready-ingestion-contract.md new file mode 100644 index 00000000..23adb9a3 --- /dev/null +++ b/docs/ph-018-real-production-ready-ingestion-contract.md @@ -0,0 +1,529 @@ +# PH-018 Real Production-Ready Ingestion Contract + +This document replaced the previous test-harness-oriented production-ready +interpretation with the project owner's runtime ingestion-loop definition. +PROD-002 clarifies that project-level production-ready also requires Python and +.NET runtime parity. See +[Production Parity Contract](production-parity-contract.md). + +It is documentation only. It does not implement runtime code, add credentials, +connect to PostgreSQL, download source files, parse live data, write records, +start a scheduler, merge pull requests, approve pull requests, close issues, +delete branches, delete worktrees, or claim unrelated product tasks. + +## Runtime Production-Ready Definition + +For a selected runtime, the production ingestion loop is ready only when that +runtime can perform this complete operational loop: + +1. Start the application. +2. Load approved runtime configuration. +3. Check PostgreSQL connectivity and schema state. +4. Create missing required source-specific master/detail tables safely. +5. For GHG Protocol, DEFRA/DESNZ, and IPCC EFDB, read the latest successfully + ingested year from PostgreSQL. +6. If no year has been ingested for a source family, target the configured + initial year. The default initial year is `2024`. +7. If a year has been ingested, target `latest_year + 1` for that source family. +8. Download real target-year source data when the source family has published + that year. +9. Archive the raw source artifact and record archive metadata. +10. Parse the archived artifact. +11. Validate parsed records. +12. Insert accepted records into PostgreSQL source-specific master/detail tables. +13. Update the source family's latest-ingested year only after successful insert. +14. Repeat on the configured schedule. + +The expected near-term cycle is: + +- First successful run ingests `2024`. +- Next successful run ingests `2025`. +- Next successful run ingests `2026`. +- A later run attempts `2027` and returns `no_available_source_year` when the + source has not published that year. + +`no_available_source_year` is a safe no-op. It must not insert source records, +must not update latest-year state, and must be visible in run output. + +## Current Status + +The repository must not be described as project-level production-ready under +this definition until both the Python and .NET runtimes pass the production +parity contract and final validation is complete. + +PROD-001 completed the Python operator-run production baseline only. The Python +runtime has a production operator path. The .NET runtime is not production-ready +yet and remains a contract/parity path without an equivalent production service +or scheduled-worker entrypoint. + +PH-017 Docker PostgreSQL validation evidence is useful test-harness evidence, +but it is not sufficient for this production-ready definition because this +definition requires real source downloads, source-specific master/detail +persistence, startup schema creation, latest-year-driven scheduling, and +idempotent repeated runtime cycles. + +## Required Runtime Configuration + +Production runtime configuration must define: + +- PostgreSQL connection ownership and non-secret connection fields. +- PostgreSQL secret boundary for the password or approved credential mechanism. +- PostgreSQL schema name. +- Archive root for raw downloaded source artifacts. +- Enabled source families from `ghg_protocol`, `defra_desnz`, and `ipcc_efdb`. +- Initial year, defaulting to `2024`. +- Cycle interval or schedule. +- Max target year behavior. +- Schema bootstrap mode for additive creation of missing required objects. +- Runtime no-op behavior when a target year is unavailable. +- Logging level and diagnostics redaction policy. + +Raw DSNs, passwords, tokens, and source credentials must not be committed. + +## Shared Tables + +Shared tables exist only where they support source-specific ingestion. They do +not replace source-specific master/detail persistence. + +### `ingestion_runs` + +Purpose: records one application cycle or source-family attempt. + +Required columns: + +- `ingestion_run_id uuid primary key` +- `cycle_id text not null` +- `source_family text not null` +- `target_year integer not null` +- `run_status text not null` +- `started_at timestamp with time zone not null` +- `completed_at timestamp with time zone null` +- `error_code text null` +- `error_message text null` +- `metadata jsonb not null` + +Required constraints and indexes: + +- Index on `(source_family, target_year, started_at)`. +- `run_status` must use explicit values such as `started`, `completed`, + `completed_noop`, `failed_validation`, and `failed_runtime`. + +### `source_artifacts` + +Purpose: records downloaded raw source artifacts stored under the configured +archive root. + +Required columns: + +- `source_artifact_id uuid primary key` +- `ingestion_run_id uuid not null references ingestion_runs` +- `source_family text not null` +- `source_year integer not null` +- `source_version text not null` +- `source_archive_version text null` +- `source_url text not null` +- `archive_path text not null` +- `content_type text null` +- `byte_size bigint null` +- `sha256 text not null` +- `downloaded_at timestamp with time zone not null` +- `metadata jsonb not null` + +Required constraints and indexes: + +- Unique `(source_family, source_year, source_version, sha256)`. +- Index on `(source_family, source_year)`. + +### `source_family_year_states` + +Purpose: records successfully completed year ingestion by source family. + +Required columns: + +- `source_family text not null` +- `ingested_year integer not null` +- `source_version text not null` +- `source_artifact_id uuid not null references source_artifacts` +- `completed_ingestion_run_id uuid not null references ingestion_runs` +- `completed_at timestamp with time zone not null` +- `metadata jsonb not null` + +Required constraints and indexes: + +- Primary key or unique constraint on `(source_family, ingested_year)`. +- Index on `(source_family, ingested_year desc)`. + +Year state is updated only after source-specific master/detail insert commits. + +### `ingestion_validation_issues` + +Purpose: records structured parse, validation, and persistence issues. + +Required columns: + +- `validation_issue_id uuid primary key` +- `ingestion_run_id uuid not null references ingestion_runs` +- `source_family text not null` +- `source_year integer not null` +- `table_name text null` +- `record_external_key text null` +- `source_row_reference text null` +- `severity text not null` +- `code text not null` +- `message text not null` +- `field_name text null` +- `raw_value text null` +- `created_at timestamp with time zone not null` + +## Source-Specific Persistence Rule + +Production persistence is source-specific. The source-specific master/detail +tables below are the system of record for Phase 1 production ingestion. + +The existing normalized-only persistence path is not sufficient for production +readiness under PH-018. A normalized projection may be added later for search or +cross-source lookup, but it must be derived from source-specific master/detail +records or explicitly linked to them. It must not be the only production +persistence model. + +## GHG Protocol Tables + +### `ghg_protocol_masters` + +Purpose: one row per logical GHG Protocol source-year record group, workbook +tool, sheet section, or other parser-owned master grouping. + +Required columns: + +- `ghg_protocol_master_id uuid primary key` +- `source_artifact_id uuid not null references source_artifacts` +- `ingestion_run_id uuid not null references ingestion_runs` +- `source_family text not null default 'ghg_protocol'` +- `source_year integer not null` +- `source_version text not null` +- `source_archive_version text null` +- `master_external_key text not null` +- `tool_name text null` +- `worksheet_name text null` +- `category_name text null` +- `subcategory_name text null` +- `region text null` +- `lifecycle_status text not null` +- `record_checksum_sha256 text not null` +- `metadata jsonb not null` +- `created_at timestamp with time zone not null` +- `updated_at timestamp with time zone not null` + +Required constraints and indexes: + +- Unique `(source_family, source_year, source_version, master_external_key)`. +- Index on `(source_family, source_year)`. +- Index on `source_artifact_id`. + +### `ghg_protocol_details` + +Purpose: one row per parsed GHG Protocol factor value or factor component. + +Required columns: + +- `ghg_protocol_detail_id uuid primary key` +- `ghg_protocol_master_id uuid not null references ghg_protocol_masters` +- `source_artifact_id uuid not null references source_artifacts` +- `ingestion_run_id uuid not null references ingestion_runs` +- `source_family text not null default 'ghg_protocol'` +- `source_year integer not null` +- `source_version text not null` +- `detail_external_key text not null` +- `source_row_reference text null` +- `activity_name text null` +- `activity_unit text null` +- `fuel_or_material text null` +- `gas text null` +- `factor_value numeric not null` +- `factor_unit text not null` +- `co2e_factor_value numeric null` +- `quality_flag text null` +- `notes text null` +- `record_checksum_sha256 text not null` +- `metadata jsonb not null` +- `created_at timestamp with time zone not null` +- `updated_at timestamp with time zone not null` + +Required constraints and indexes: + +- Unique `(ghg_protocol_master_id, detail_external_key)`. +- Unique `(source_family, source_year, source_version, detail_external_key)`. +- Index on `(source_family, source_year)`. + +## DEFRA/DESNZ Tables + +### `defra_desnz_masters` + +Purpose: one row per logical DEFRA/DESNZ source-year factor set, category, +sheet, tab, or parser-owned grouping. + +Required columns: + +- `defra_desnz_master_id uuid primary key` +- `source_artifact_id uuid not null references source_artifacts` +- `ingestion_run_id uuid not null references ingestion_runs` +- `source_family text not null default 'defra_desnz'` +- `source_year integer not null` +- `source_version text not null` +- `source_archive_version text null` +- `master_external_key text not null` +- `dataset_name text null` +- `worksheet_name text null` +- `category_name text null` +- `subcategory_name text null` +- `scope_hint text null` +- `region text null` +- `lifecycle_status text not null` +- `record_checksum_sha256 text not null` +- `metadata jsonb not null` +- `created_at timestamp with time zone not null` +- `updated_at timestamp with time zone not null` + +Required constraints and indexes: + +- Unique `(source_family, source_year, source_version, master_external_key)`. +- Index on `(source_family, source_year)`. +- Index on `source_artifact_id`. + +### `defra_desnz_details` + +Purpose: one row per parsed DEFRA/DESNZ factor value or factor component. + +Required columns: + +- `defra_desnz_detail_id uuid primary key` +- `defra_desnz_master_id uuid not null references defra_desnz_masters` +- `source_artifact_id uuid not null references source_artifacts` +- `ingestion_run_id uuid not null references ingestion_runs` +- `source_family text not null default 'defra_desnz'` +- `source_year integer not null` +- `source_version text not null` +- `detail_external_key text not null` +- `source_row_reference text null` +- `activity_name text null` +- `activity_unit text null` +- `gas text null` +- `factor_value numeric not null` +- `factor_unit text not null` +- `conversion_factor_type text null` +- `quality_flag text null` +- `notes text null` +- `record_checksum_sha256 text not null` +- `metadata jsonb not null` +- `created_at timestamp with time zone not null` +- `updated_at timestamp with time zone not null` + +Required constraints and indexes: + +- Unique `(defra_desnz_master_id, detail_external_key)`. +- Unique `(source_family, source_year, source_version, detail_external_key)`. +- Index on `(source_family, source_year)`. + +## IPCC EFDB Tables + +### `ipcc_efdb_masters` + +Purpose: one row per logical IPCC EFDB source-year sector, category, reference, +or parser-owned factor record grouping. + +Required columns: + +- `ipcc_efdb_master_id uuid primary key` +- `source_artifact_id uuid not null references source_artifacts` +- `ingestion_run_id uuid not null references ingestion_runs` +- `source_family text not null default 'ipcc_efdb'` +- `source_year integer not null` +- `source_version text not null` +- `source_archive_version text null` +- `master_external_key text not null` +- `sector_code text null` +- `sector_name text null` +- `category_code text null` +- `category_name text null` +- `reference_title text null` +- `reference_year integer null` +- `region text null` +- `country text null` +- `lifecycle_status text not null` +- `record_checksum_sha256 text not null` +- `metadata jsonb not null` +- `created_at timestamp with time zone not null` +- `updated_at timestamp with time zone not null` + +Required constraints and indexes: + +- Unique `(source_family, source_year, source_version, master_external_key)`. +- Index on `(source_family, source_year)`. +- Index on `source_artifact_id`. + +### `ipcc_efdb_details` + +Purpose: one row per parsed IPCC EFDB factor value, default/min/max value, or +measurement component. + +Required columns: + +- `ipcc_efdb_detail_id uuid primary key` +- `ipcc_efdb_master_id uuid not null references ipcc_efdb_masters` +- `source_artifact_id uuid not null references source_artifacts` +- `ingestion_run_id uuid not null references ingestion_runs` +- `source_family text not null default 'ipcc_efdb'` +- `source_year integer not null` +- `source_version text not null` +- `detail_external_key text not null` +- `source_row_reference text null` +- `parameter_name text null` +- `activity_name text null` +- `technology_or_practice text null` +- `gas text null` +- `factor_value numeric null` +- `default_value numeric null` +- `min_value numeric null` +- `max_value numeric null` +- `factor_unit text not null` +- `uncertainty text null` +- `data_quality text null` +- `notes text null` +- `record_checksum_sha256 text not null` +- `metadata jsonb not null` +- `created_at timestamp with time zone not null` +- `updated_at timestamp with time zone not null` + +Required constraints and indexes: + +- Unique `(ipcc_efdb_master_id, detail_external_key)`. +- Unique `(source_family, source_year, source_version, detail_external_key)`. +- Index on `(source_family, source_year)`. + +## Idempotency Keys + +Every source-specific master row must have a deterministic +`master_external_key` built from: + +- `source_family` +- `source_year` +- `source_version` +- source artifact identity or source archive version +- source-specific logical grouping identity + +Every source-specific detail row must have a deterministic `detail_external_key` +built from: + +- `source_family` +- `source_year` +- `source_version` +- parent `master_external_key` +- source row reference or parser-stable row identity +- factor field identity when one source row contains multiple factor values + +`record_checksum_sha256` must hash the canonical persisted record payload used +for conflict detection. A repeated run with the same idempotency key and same +checksum is a duplicate no-op. A repeated run with the same idempotency key and +a different checksum is a conflict and must fail or require a future reviewed +replacement policy. + +## Runtime Cycle Behavior + +At application startup: + +1. Load configuration. +2. Validate PostgreSQL provider and required configuration. +3. Open a PostgreSQL connection using the approved credential boundary. +4. Check the required schema and tables. +5. Create missing tables and indexes using additive, idempotent DDL when schema + bootstrap is enabled. +6. Stop startup if PostgreSQL is unavailable, schema bootstrap is disabled with + missing objects, or incompatible existing objects are detected. +7. Start the configured ingestion scheduler only after schema readiness passes. + +On each scheduled cycle, each enabled source family runs independently: + +1. Read `max(ingested_year)` from `source_family_year_states`. +2. Select `target_year = initial_year` when no year exists. +3. Select `target_year = latest_year + 1` when a year exists. +4. Apply configured max target year behavior. +5. Discover whether real source data exists for `target_year`. +6. If unavailable, record `no_available_source_year` in run output only. +7. If available, download and archive the source artifact. +8. Parse the archived artifact. +9. Validate parsed records. +10. Insert source-specific master/detail rows in one reviewed transaction + boundary. +11. Commit source-specific rows and then update `source_family_year_states`. +12. Report inserted, duplicate, validation-failed, and no-op counts. + +The scheduler must prevent overlapping cycles for the same source family. +Repeated cycles must be idempotent. + +## Follow-Up Implementation Tasks + +PH-018 is complete only when this contract is documented. PROD-001 completed the +Python operator-run baseline. Project-level production-ready remains future +work until the .NET runtime reaches parity and cross-runtime validation passes. + +Required follow-up tasks for project-level production readiness: + +1. Replace the current normalized-only PostgreSQL runtime schema with the + source-specific master/detail schema in this contract. +2. Implement additive startup schema bootstrap for the shared and + source-specific tables. +3. Implement runtime configuration for PostgreSQL, archive root, enabled source + families, initial year, cycle schedule, and max target year behavior. +4. Implement latest-successful-year lookup and year-state update semantics. +5. Implement GHG Protocol real source discovery and target-year download from + 2024 onward. +6. Implement DEFRA/DESNZ real source discovery and target-year download from + 2024 onward. +7. Implement IPCC EFDB real source discovery and target-year download from 2024 + onward. +8. Implement raw artifact archive layout and metadata persistence. +9. Implement source-specific parsers that write to the source-specific + master/detail model. +10. Implement idempotent insert and conflict handling for source-specific + tables. +11. Implement scheduled runtime cycles with overlap prevention and structured + no-op reporting. +12. Add Docker PostgreSQL integration tests for startup bootstrap, 2024 initial + ingestion, 2025/2026 next-year progression, 2027 unavailable no-op, + idempotent replay, and conflict handling. +13. Implement the .NET service/scheduled-worker entrypoint. +14. Implement the .NET production config loader and redaction behavior. +15. Implement .NET PostgreSQL schema bootstrap and year-state behavior. +16. Implement .NET source discovery/download/parsing orchestration. +17. Implement .NET source-specific master/detail inserts. +18. Implement .NET idempotency and rerun behavior. +19. Add .NET Docker PostgreSQL E2E tests. +20. Add Python/.NET parity validation. +21. Add final validation before any project-level production-ready claim is + restored. + +## Non-Goals + +This task does not: + +- Implement runtime code. +- Implement live source downloads. +- Implement parser changes. +- Implement PostgreSQL DDL changes. +- Add credentials. +- Claim production readiness. +- Claim source-owner, factor, unit-conversion, legal, compliance, or + carbon-accounting correctness. +- Merge, approve, or close any pull request or issue. + +## PR Body Footer + +The pull request body for PH-018 must end with: + +```text +Task-ID: PH-018 +Task-Issue: #594 +``` + +Task-ID: PH-018 +Task-Issue: #594 diff --git a/docs/phase2-review-gate.md b/docs/phase2-review-gate.md new file mode 100644 index 00000000..311b273e --- /dev/null +++ b/docs/phase2-review-gate.md @@ -0,0 +1,343 @@ +# Phase 2 Runtime And Source Expansion Review Gate + +Task-ID: PH-004 +Task-Issue: #561 + +## Purpose + +This document defines the Phase 2 review gate that must pass before any newly +onboarded source family is considered production-ready. It turns the Phase 2 +roadmap into concrete acceptance gates for source onboarding, data quality, +Python/.NET parity, runtime safety, persistence and idempotency, observability, +and release readiness. + +This is a review-only gate. It does not implement runtime code, add parsers, +call live endpoints, execute database operations, add credentials, or certify +source, legal, compliance, carbon-accounting, unit-conversion, or factor +correctness. + +## When To Use This Gate + +Use this gate for any Phase 2 source-family readiness review that proposes one +of these outcomes: + +- Marking a new source family production-ready. +- Marking a source family production-ready with accepted risks. +- Blocking a source family from production readiness until follow-up work is + complete. +- Promoting source onboarding, parser/runtime hardening, persistence, + observability, or release checks from local review evidence to production + readiness evidence. + +The review must name the exact source family, implementation slice, branch or +PR, fixture set, contracts, tests, and docs under review. Readiness applies only +to that named slice. + +## Required Review Inputs + +A Phase 2 readiness review must include: + +- Source-family identity, source keys, and supported document or artifact + types. +- Scope statement that separates source acquisition, parsing, normalization, + persistence, scheduler/service behavior, and release packaging. +- Links to source onboarding, parser, normalization, persistence, runtime, + diagnostic, and release documents changed by the slice. +- Python and .NET contract/test evidence, or a documented reason why the slice + has `no-parity-impact`. +- Deterministic local fixtures and expected outputs used for review. +- Validation command results, including CI checks that are required for the + slice. +- Known limitations, accepted risks, and deferred work. +- Explicit recommendation: production-ready, production-ready with accepted + risks, or blocked. + +## Source Onboarding Review Checklist + +Before a source family can be considered production-ready, reviewers must +confirm: + +- Source-family identity is stable and uses the same canonical key across + acquisition, parser input, parser output, normalization, persistence + readiness, diagnostics, docs, and tests. +- Supported source documents, versions, formats, artifact types, and update + cadence are documented. +- Public availability and licensing assumptions are documented without adding + confidential, private, or copied source data. +- Local deterministic fixtures exist for the supported artifacts under review. +- Fixture provenance, checksum or hash expectations, source document version, + reporting year or effective period, and row/document identity are captured + where the contracts require them. +- Unsupported formats, missing files, malformed inputs, empty inputs, duplicate + identities, and warning-only cases have deterministic review evidence. +- Live endpoint use is either out of scope or explicitly reviewed as opt-in + behavior with timeout, retry, rate-limit, authentication, redaction, and + operator controls. +- Source discovery/download remains separate from parser execution, + normalization, persistence, scheduling, and database execution unless the + reviewed task explicitly scopes the coupling. +- No production source coverage, source-owner correctness, legal correctness, + compliance correctness, carbon-accounting correctness, unit-conversion + correctness, or factor correctness claim is made. + +## Data Quality Validation Checklist + +Reviewers must confirm that validation evidence is structured, deterministic, +and reviewable: + +- Required provenance fields are present for records, documents, artifacts, and + run summaries. +- Source identity, document identity, parser identity, version/checksum + metadata, row identity, and traceability fields are preserved from parser + output through normalization and persistence readiness where applicable. +- Structural validation, parser-readiness validation, normalization validation, + persistence-readiness validation, and production-readiness validation are + distinguishable. +- Validation issues carry stable severity, code, message, field/path, source + document, row/document identity, stage, and run correlation fields where the + relevant contract supports them. +- Summary outputs expose accepted, rejected, warning, skipped, persisted, and + failed counts without hiding row-level or document-level issues needed for + review. +- Fixture expectations cover accepted records, rejected records, partial + success, warning-only records, duplicate identities, missing required fields, + unsupported values, and empty inputs. +- Warning-to-blocking promotion is explicitly documented and tested; it is not + inferred from wording alone. +- Data quality checks do not require network access, production credentials, + database writes, scheduler behavior, or live source availability by default. + +## Python/.NET Parity Checklist + +For every shared source-family readiness slice, reviewers must assign and +verify a parity mode: + +- `no-parity-impact`: no shared contract, serialized payload, status, fixture, + diagnostic, or public behavior changed. +- `parity-planning`: the task defines future parity expectations but changes no + runtime behavior. +- `python-first-with-parity-follow-up`: a Python change is accepted only with a + named follow-up parity review for affected shared behavior. +- `lockstep-parity`: Python and .NET contracts, fixtures, tests, and docs move + together in the same slice. +- `parity-review`: the task compares existing Python and .NET behavior without + adding runtime behavior. + +When parity applies, reviewers must confirm: + +- Source-family keys, parser keys, status values, enum/wire names, issue codes, + severities, stage names, and diagnostic event names are aligned or explicitly + documented as accepted drift. +- Required and optional serialized fields match the shared contract + expectations. +- Null, empty, default, duplicate, and unsupported cases have equivalent + observable behavior. +- Python and .NET tests use equivalent fixture expectations or a shared parity + fixture where available. +- Contract docs identify which behavior is public, internal, deferred, or + implementation-specific. +- Coarser behavior in one language is accepted only when structured details + still preserve reviewable failure stage, code, and source-family context. +- No source family is marked production-ready when shared public behavior has an + unexplained parity mismatch. + +## Runtime Safety Checklist + +Reviewers must confirm that Phase 2 runtime behavior remains bounded and +operator-safe: + +- Default execution is local, deterministic, non-destructive, and fail-closed. +- Live network calls are disabled by default or require explicit opt-in, + configured timeout, retry/backoff policy, rate-limit handling, authentication + boundary, redaction boundary, and operator-visible failure reporting. +- Database execution is disabled by default unless a separate reviewed task + explicitly promotes opt-in runtime writes. +- Runtime configuration does not load production credentials, raw connection + strings, secret files, or environment-derived secrets unless the reviewed task + explicitly scopes and tests that behavior. +- Parser, downloader, scheduler, database, and service-host coupling is limited + to the reviewed slice. +- Unsupported, not-ready, skipped, warning, failed, and completed states are + deterministic and structured. +- Cancellation, retry, replay, and dead-letter behavior are either out of scope + or explicitly reviewed with bounded semantics and idempotency evidence. +- Failures do not expose credential-shaped values, private paths, raw source + payloads, or confidential material in logs, exceptions, diagnostics, fixtures, + or docs. +- Runtime changes preserve existing public APIs unless the task explicitly + scopes an API change and includes migration notes. + +## Persistence And Idempotency Checklist + +Before a source family can be production-ready with persistence enabled, +reviewers must confirm: + +- Persistence execution mode is explicit: disabled, preview-only, opt-in test, + or production-enabled for the reviewed slice. +- Source document identity, source family, source key, artifact checksum/hash, + parser version or mapping identity, row identity, and normalized record + identity are stable enough for replay review. +- Duplicate input records, duplicate source documents, repeated runs, partial + failures, and retry attempts have deterministic expected outcomes. +- Conflict handling is documented for insert, update, skip, reject, and + warning-only paths. +- Transaction boundaries, rollback behavior, commit timing, and partial + persistence summaries are reviewed for the exact execution mode. +- Persistence summaries distinguish attempted, accepted, rejected, skipped, + persisted, duplicate, and failed counts. +- Idempotency expectations are tested against local deterministic fixtures or + explicitly documented as a blocker. +- No destructive DDL, destructive DML, migration, rollback cleanup, table drop, + data deletion, or production write behavior is introduced without a separate + production-readiness review. + +## Observability And Redaction Checklist + +Safe diagnostics are required for production readiness. Reviewers must confirm: + +- Diagnostic payloads include run identity, correlation identity, source family, + source key, document or artifact identity, stage, status, issue code, severity, + and counts needed for triage. +- Logs and diagnostics expose enough context to locate a failed source-family + slice without exposing raw credentials, tokens, connection strings, private + paths, confidential material, copied source payloads, or arbitrary raw + exception payloads. +- Credential-shaped fields are redacted in Python and .NET diagnostic helpers + where shared diagnostics apply. +- Safe checksums, document identifiers, artifact references, and parser/source + metadata are used instead of raw payload dumps. +- Error messages distinguish operator action, unsupported source input, + validation failure, parser failure, persistence failure, runtime block, and + infrastructure failure where the reviewed contracts support those stages. +- Metrics, traces, alerts, dashboards, centralized log ingestion, retention, + and SLOs are either reviewed with evidence or listed as limitations. +- Redaction tests or review evidence cover common secret-bearing fields such as + password, token, secret, credential, DSN, connection string, URI, database + URL, username, host, and application name where those values can appear in + diagnostics. + +## CI And Release Gate Expectations + +A source-family readiness review must state which checks are required for the +slice and whether each check passed, failed, or was intentionally skipped. + +Minimum local validation for documentation-only gate changes: + +```bash +git diff --check +``` + +Minimum validation when Python package behavior, examples, fixtures, or tests +are affected: + +```bash +python -m pytest +git diff --check +``` + +Release readiness reviews should also consider, when applicable: + +- Focused Python tests for the changed source-family, parser, validation, + normalization, persistence, or diagnostic behavior. +- Focused .NET tests for changed shared contracts or diagnostic behavior. +- Parity fixtures or parity-review evidence for shared wire names, statuses, + issues, and summaries. +- Public-safety wording checks. +- Documentation index/map checks when docs are added or renamed. +- Opt-in integration tests only when the reviewed task explicitly scopes + database or live endpoint behavior. +- CI stability evidence for checks proposed as release gates. + +Generated artifacts must not be tracked unless the task explicitly requests and +reviews them. + +## Decision Criteria + +### Production-Ready + +Use this decision only when all required checklists pass for the reviewed +source-family slice: + +- Source onboarding evidence is complete for the supported formats and versions. +- Data quality validation is deterministic and covers accepted, rejected, + partial, duplicate, empty, malformed, and warning-only cases. +- Python/.NET parity is satisfied or marked `no-parity-impact` with evidence. +- Runtime behavior is bounded, fail-closed by default, and operator-safe. +- Persistence and idempotency expectations are proven for the promoted + execution mode. +- Diagnostics include safe correlation and redacted failure context. +- Required CI/release gates pass. +- Known limitations do not undermine the production claim for the exact slice. + +### Production-Ready With Accepted Risks + +Use this decision when the slice is safe to promote but has documented residual +risks that owners explicitly accept: + +- Each accepted risk has an owner, impact, mitigation, review date or follow-up + task, and reason it does not block the exact production slice. +- The risk does not involve unredacted secrets, confidential/private data, + destructive database behavior, unexplained Python/.NET contract drift, + missing idempotency for enabled writes, or unsupported production correctness + claims. +- CI/release gates required for the slice still pass. +- Operators can detect and safely respond to the accepted risk through + diagnostics, run summaries, rollback procedure, or documented disablement. + +### Blocked + +Use this decision when any blocking condition is present: + +- Source-family identity, supported formats, fixtures, or provenance evidence + are incomplete for the claimed production slice. +- Data quality validation is not deterministic or misses material failure cases. +- Python/.NET shared behavior has unexplained parity drift. +- Safe diagnostics are missing, ambiguous, or leak credential-shaped values, + private data, copied source payloads, or raw sensitive exception context. +- Runtime behavior requires live endpoints, credentials, scheduler coupling, or + database writes without explicit opt-in controls and review evidence. +- Persistence/idempotency evidence is missing for enabled write paths. +- Required tests or CI/release gates fail or are skipped without accepted risk + approval. +- The review relies on production, legal, compliance, source-owner, + carbon-accounting, unit-conversion, or factor correctness claims that are not + explicitly scoped and proven. + +## Review Output Template + +Each Phase 2 readiness review should record: + +- Reviewed source family and production slice. +- Reviewed files, fixtures, contracts, tests, docs, and CI runs. +- Checklist results by section. +- Python/.NET parity mode and findings. +- Diagnostics and redaction findings. +- Persistence/idempotency findings. +- Known limitations. +- Accepted risks, if any. +- Final decision: production-ready, production-ready with accepted risks, or + blocked. +- Required follow-up tasks. + +## Non-Goals + +This gate does not add, implement, prove, or claim: + +- Runtime code. +- Source parsers. +- Live endpoint access. +- Database operations. +- Production credentials or secret handling. +- Scheduler, queue, distributed lock, replay, or dead-letter behavior. +- Production source coverage. +- Legal, compliance, source-owner, carbon-accounting, unit-conversion, or factor + correctness. +- Release approval beyond the exact reviewed slice. + +## Related Documents + +- [Phase 2 Roadmap And Execution Boundary](phase2-roadmap.md) +- [Review Readiness Checklist](review-readiness-checklist.md) +- [Source Acquisition Review Gate Boundary](source-acquisition-review-gate-boundary.md) +- [PostgreSQL Runtime Readiness Checklist](postgresql-runtime-readiness-checklist.md) +- [Final Phase 1 Production Readiness Review](final-phase1-production-readiness-review.md) diff --git a/docs/phase2-roadmap.md b/docs/phase2-roadmap.md new file mode 100644 index 00000000..bd9409bb --- /dev/null +++ b/docs/phase2-roadmap.md @@ -0,0 +1,266 @@ +# Phase 2 Roadmap And Execution Boundary + +## Executive Summary + +Phase 2 starts from the accepted Phase 1 production readiness checkpoint and +turns the repository from a local-only, fail-closed, reviewable production +candidate into a broader implementation program. The priority is controlled +expansion: source onboarding, parser fidelity, data quality validation, +PostgreSQL runtime hardening, operational packaging, and release-gate coverage +must advance through small tasks with explicit planning, implementation, +Python/.NET parity review, and production-readiness review stages. + +Phase 2 does not relax the Phase 1 safety posture. Runtime writes, live source +execution, scheduler behavior, credentials, destructive database operations, +and production correctness claims remain disabled or out of scope until a +separately reviewed task explicitly enables them. + +## Confirmed Phase 1 Baseline And Assumptions + +Phase 2 assumes the Phase 1 baseline recorded by the final production +readiness review: + +- Python source acquisition, parser execution, normalization, persistence + handoff, orchestration, service-host, diagnostics, and local validation + boundaries are ready for the Phase 1 contract and operator release. +- .NET contract records and focused production-safety tests are ready for + Phase 1 parity release. +- PostgreSQL behavior is fail-closed by default. Schema readiness and preview + paths exist, but production database writes remain explicitly gated. +- Source acquisition supports local and dry-run operation. Live source + availability, upstream variability, authentication, retry/rate-limit + behavior, and scheduling are not proven by the default gate. +- Parser, normalization, and persistence paths are validated against + deterministic local fixtures and contract handoffs. They do not claim + complete source correctness or carbon-accounting correctness. +- Operational diagnostics redact credential-shaped values and expose run + identity and failure context, but metrics, traces, alerts, dashboards, and + centralized log ingestion remain future work. +- Default validation remains local-only and non-destructive. Full Python tests, + full .NET tests, full public-safety scan, opt-in PostgreSQL integration, and + live source checks are not default release requirements until separately + promoted. + +Any task that depends on a stronger assumption must first add a planning or +review task that proves and documents the new baseline. + +## Phase 2 Workstreams And Sequencing + +Phase 2 work should proceed through the following workstreams. Workstreams may +overlap only when their tasks do not change the same public contracts or +runtime behavior. + +| Sequence | Workstream | Primary outcome | Required gate before next stage | +| --- | --- | --- | --- | +| 1 | Planning and scope confirmation | Convert roadmap items into small task tickets with explicit non-goals and validation. | Review confirms no runtime behavior change in planning-only tasks. | +| 2 | Source onboarding readiness | Define source selection criteria, fixture capture rules, source metadata expectations, and local-only onboarding checks. | Source onboarding review confirms no live calls or parser claims unless explicitly scoped. | +| 3 | Data quality and validation | Extend deterministic validation rules, issue taxonomy, fixture expectations, and reviewable quality summaries. | Python and .NET contracts agree on status, issue, and summary semantics where shared. | +| 4 | Parser/runtime hardening | Improve parser fidelity, acquisition resilience, runtime handoffs, and failure reporting in narrow source-specific slices. | Focused tests and parity review pass for each changed contract or shared expectation. | +| 5 | PostgreSQL runtime hardening | Promote disabled/preview persistence toward opt-in execution with transaction, conflict, migration, rollback, and recovery boundaries. | Integration remains opt-in until production-readiness review promotes it. | +| 6 | Operational packaging and scheduling | Define service packaging, process supervision, health probes, scheduler identity, leases, cancellation, retry/backoff, replay, and dead-letter behavior. | Review confirms non-destructive defaults and no production credential coupling. | +| 7 | Release-gate expansion | Promote broader Python, .NET, public-safety, integration, and production RC checks into stable default gates. | Gate promotion review proves checks are deterministic and local-safe by default. | +| 8 | Production-readiness review | Consolidate evidence, accepted risks, operator rules, and release recommendation for Phase 2 slices. | Release candidate remains blocked until review signs off on the exact slice. | + +The first practical implementation sequence is: + +1. Create planning tickets for source onboarding, data quality, parser + hardening, PostgreSQL runtime execution, operations packaging, and release + gate promotion. +2. Add a source onboarding readiness document and fixture policy before any new + source parser or live source behavior. +3. Add deterministic validation/quality contracts and parity expectations for + shared status and issue semantics. +4. Implement one narrow parser/source hardening slice using local fixtures only. +5. Run a parity review for the changed Python and .NET contracts. +6. Run a production-readiness review before enabling any live execution, + scheduler behavior, or database write path. + +## Source Onboarding Expansion Strategy + +Source onboarding must be evidence-led and fixture-first: + +- Start with source selection criteria: source family, document format, + licensing/public availability, update cadence, expected stability, + provenance fields, and known variability. +- Add local deterministic fixtures before adding runtime source code. +- Document source identity, source document versioning, checksum expectations, + parser input mapping, and normalization handoff fields. +- Keep source discovery/download execution separate from parser execution and + persistence. +- Add one source or one source-family slice per task. Do not bundle unrelated + source families in the same implementation task. +- Treat live endpoints as opt-in validation only after a planning task defines + network, retry, timeout, rate-limit, authentication, redaction, and operator + controls. + +Source onboarding tasks must avoid production claims. A source can be marked +implemented for a fixture/local path without claiming live-source correctness, +complete upstream coverage, or carbon-accounting correctness. + +## Data Quality And Validation Strategy + +Phase 2 data quality work should make validation outcomes reviewable before +making them operationally decisive: + +- Define required provenance, identity, version, checksum, timestamp, parser, + and traceability fields for every source-family slice. +- Separate structural validation, parser-readiness validation, normalization + validation, persistence-readiness validation, and production-readiness + validation. +- Use deterministic fixtures for accepted rows, malformed rows, empty inputs, + unsupported formats, duplicate identities, missing metadata, and warning-only + cases. +- Preserve structured issue severity, issue code, field/path, source document, + and run correlation fields. +- Add summary outputs that operators and reviewers can compare without reading + raw records. +- Promote validation from warning to blocking only through a task that updates + tests, docs, and parity expectations. + +Validation must not introduce network calls, database writes, environment +credential loading, or scheduler behavior unless those behaviors are the +explicit subject of the task. + +## Parser And Runtime Hardening Strategy + +Parser and runtime hardening should advance through narrow, reversible slices: + +- Harden one source family, adapter, parser, or runtime boundary at a time. +- Prefer already-loaded content and fixture-driven tests before downloader or + scheduler integration. +- Keep acquisition, parsing, normalization, persistence, and orchestration + contracts independently reviewable. +- Preserve explicit unsupported and not-ready statuses instead of raising + ambiguous runtime failures. +- Add retry, timeout, cancellation, replay, and dead-letter semantics only + behind documented operational boundaries. +- Preserve fail-closed defaults for database execution and production runtime + modes. + +Any task that changes runtime behavior must include focused tests and a +documentation update describing the before/after boundary. + +## Python/.NET Parity Expectations + +Parity is required for shared contracts, public statuses, serialized field +names, issue semantics, run identity, source-family identity, and operational +diagnostics. Phase 2 tasks must identify one of these parity modes: + +- `no-parity-impact`: Documentation-only or Python-internal work with no + shared contract change. +- `parity-planning`: Planning or review task that defines a future shared + contract expectation. +- `python-first-with-parity-follow-up`: Narrow implementation allowed only when + the task creates or references a follow-up parity review. +- `lockstep-parity`: Python and .NET contract/test updates happen in the same + task because serialized behavior or public contract shape changes. +- `parity-review`: Review-only task that compares Python, .NET, docs, and + fixtures without adding runtime behavior. + +Parity reviews must check wire names, enum/status values, required and optional +fields, validation issue semantics, fixture expectations, and documentation +links. They must not approve runtime enablement unless the task explicitly +includes production-readiness review. + +## Operational Safety And Non-Destructive Execution Rules + +Phase 2 tasks must preserve these rules unless a separately approved task +changes them: + +- No PR merges, PR approvals, issue closures, branch deletion, or worktree + deletion from implementation tasks. +- No production credentials, raw connection strings, secrets, or private source + data in repository files, examples, logs, docs, fixtures, or tests. +- No live source endpoint calls by default. +- No database operations by default. Integration checks must remain opt-in and + use externally supplied test configuration. +- No destructive database commands, migrations, rollback operations, table + drops, data deletion, or production writes without explicit production + readiness approval. +- No scheduler, downloader, parser, normalizer, persistence, or service-host + coupling unless the task explicitly requests it. +- No generated artifacts unless the task explicitly requests and reviews them. +- No production, compliance, legal, or carbon-accounting correctness claims. +- Keep examples deterministic, local-only, and reviewable unless the task + explicitly scopes an opt-in integration path. + +## Explicit Out Of Scope + +The following are outside PH-001 and outside default Phase 2 execution unless a +future task explicitly scopes them: + +- Implementing Phase 2 runtime code in this roadmap task. +- Adding new source parsers or source-specific ingestion in this roadmap task. +- Adding production credentials, secret-store integration, or raw connection + string handling. +- Calling live source endpoints or proving live source availability. +- Executing database operations, migrations, DDL, DML, rollback, or cleanup. +- Enabling production database writes. +- Adding scheduler, distributed lock, queue, cron, replay, or dead-letter + runtime behavior. +- Publishing production daemon or worker-service packaging. +- Claiming complete source coverage, carbon-accounting correctness, compliance + readiness, legal correctness, or production data quality correctness. +- Changing Phase 1 production behavior, release candidate assumptions, or + operator safety rules. + +## Initial Phase 2 Backlog Proposal + +The first backlog should separate planning, implementation, parity review, and +production-readiness review: + +| Proposed task | Type | Purpose | Dependencies | +| --- | --- | --- | --- | +| PH-002 Source onboarding readiness plan | Planning | Define source selection, fixture capture, metadata, live-call controls, and review gates. | PH-001 | +| PH-003 Data quality validation plan | Planning | Define validation layers, issue taxonomy expansion, fixture matrix, and summary expectations. | PH-001 | +| PH-004 Parser/runtime hardening plan | Planning | Define first parser/source hardening slice and runtime failure semantics. | PH-001 | +| PH-005 Python/.NET parity review protocol | Review | Define reusable parity checklist for Phase 2 shared contracts and fixtures. | PH-001 | +| PH-006 Source onboarding fixture policy | Implementation | Add docs/tests for deterministic fixture requirements without adding live ingestion. | PH-002, PH-005 | +| PH-007 First source-family hardening slice | Implementation | Improve one existing source-family parser or acquisition boundary using local fixtures. | PH-003, PH-004, PH-006 | +| PH-008 First Phase 2 parity review | Review | Compare changed Python/.NET contracts, docs, and fixtures after PH-007. | PH-005, PH-007 | +| PH-009 PostgreSQL runtime execution hardening plan | Planning | Define opt-in execution, transaction, conflict, migration, rollback, and recovery boundaries. | PH-001 | +| PH-010 Operational packaging and scheduling plan | Planning | Define worker packaging, health probes, scheduler identity, leases, cancellation, retry, and replay boundaries. | PH-001 | +| PH-011 Release-gate promotion plan | Planning | Define how full Python, .NET, public-safety, integration, and RC checks become stable default gates. | PH-001 | +| PH-012 Phase 2 production-readiness review | Review | Consolidate validation evidence, accepted risks, and release recommendation for the first Phase 2 slice. | PH-008, PH-009, PH-010, PH-011 as applicable | + +Task identifiers are proposals, not claims that the tasks already exist. + +## Risk Register + +| Risk | Impact | Mitigation | +| --- | --- | --- | +| Phase 2 implementation disturbs Phase 1 release assumptions. | Regression in accepted production candidate behavior. | Require small tasks, focused tests, docs updates, parity review, and production-readiness review for runtime changes. | +| Source onboarding expands faster than fixture and validation evidence. | Unsupported correctness claims or brittle parser behavior. | Require fixture-first onboarding, explicit source selection criteria, and local-only validation before live behavior. | +| Python and .NET contracts drift. | Broken consumers or inconsistent operational semantics. | Assign parity mode per task and run parity reviews for shared status, issue, and wire-name changes. | +| PostgreSQL execution is enabled before transaction/recovery rules are mature. | Data loss, duplicate persistence, or unsafe operational behavior. | Keep writes fail-closed until opt-in execution, transaction, conflict, migration, rollback, and recovery boundaries are reviewed. | +| Release gates become slow or flaky. | Teams bypass validation or lose confidence in release evidence. | Promote only deterministic checks into default gates; keep integration/live checks explicitly opt-in until stable. | +| Operational packaging couples scheduler, downloader, parser, and persistence too early. | Hard-to-review runtime failures and larger blast radius. | Define packaging and scheduling boundaries before implementation; keep coupling task-scoped. | +| Validation summaries hide row-level issues. | Reviewers miss parser or data quality regressions. | Preserve structured issue details and fixture-level assertions alongside summaries. | +| Documentation and implementation diverge. | Review decisions rely on stale boundaries. | Require docs/index updates and focused tests for changed public behavior. | + +## Dependency Map + +- PH-001 is the root Phase 2 roadmap and execution-boundary task. +- PH-002, PH-003, and PH-004 are unblocked by PH-001 and should remain + planning-first. +- Source onboarding implementation depends on source onboarding planning, + fixture policy, and parity protocol. +- Parser/runtime hardening depends on data quality expectations and source + onboarding readiness for the affected source family. +- PostgreSQL runtime execution depends on a separate hardening plan and must + remain independent of source onboarding until opt-in execution is reviewed. +- Operational packaging depends on service-host and orchestration boundaries, + but must not enable scheduler/runtime coupling without its own task. +- Release-gate promotion depends on deterministic checks and should not depend + on live endpoints or production databases by default. +- Production-readiness review depends on implementation evidence, parity review + results, validation evidence, and updated risk acceptance. + +## Review Boundary + +This document is a planning artifact. It does not implement Phase 2 runtime +code, add source parsers, change production configuration, enable live calls, +execute database operations, or change Phase 1 production behavior. + +Task-ID: PH-001 +Task-Issue: #558 diff --git a/docs/postgresql-opt-in-integration-runbook.md b/docs/postgresql-opt-in-integration-runbook.md index f434f624..67854f51 100644 --- a/docs/postgresql-opt-in-integration-runbook.md +++ b/docs/postgresql-opt-in-integration-runbook.md @@ -1,14 +1,14 @@ # PostgreSQL Opt-In Integration Runbook -This runbook defines how future PostgreSQL integration tests should be prepared -and run safely. +This runbook defines how PostgreSQL integration tests should be prepared and +run safely. The default test suite remains deterministic and DB-free; real +PostgreSQL checks require explicit opt-in controls and an externally supplied +test DSN. -It is documentation and test-harness guidance only. It does not create a -PostgreSQL connection, create a cursor, run SQL, write records, start a -transaction, finish a transaction, roll back a transaction, create tables, run -migrations, load environment variables in library code, load configuration -files in library code, load credentials, perform HTTP or network calls, schedule -work, or claim production persistence readiness. +The current Python production runtime is documented in +[Production Packaging And Operator Runbook](production-packaging-operator-runbook.md). +This integration runbook is test-harness guidance; it must not be used to store +production DSNs, passwords, tokens, or database dumps. ## Why Integration Tests Are Opt-In @@ -16,10 +16,10 @@ The default test suite must remain deterministic and local-only. Normal `python -m pytest` runs must not require PostgreSQL, credentials, network access, a local database, migrations, or table setup. -PostgreSQL integration tests are reserved for future runtime tasks that -explicitly add database behavior behind the runtime execution gate. Until then, -the repository remains unsupported/no-execution and integration test behavior is -represented only by metadata. +PostgreSQL integration tests are opt-in because they can open external +connections, create isolated schemas, bootstrap Phase 1 tables, and write test +rows. They must not run unless the operator provides the canonical controls +described below. ## Existing Boundary @@ -138,7 +138,8 @@ Allowed status values: - `not_run` - `passed` - `failed_sanitized` -- `blocked_environment` +- `blocked_environment` for historical environment-unavailable smoke attempts + only; completed release records should use `passed` or `failed_sanitized`. Current execution record: @@ -259,6 +260,148 @@ python -m pytest -m postgresql_integration tests/test_postgresql_connection_smok This command is a future/manual integration path. It is not part of the default test suite and does not exist as runtime persistence enablement. +## PH-011 Docker Runtime Schema And Year-State Integration + +PH-011 adds an opt-in integration test that proves runtime schema bootstrap and +source-family year-state behavior against Docker PostgreSQL. The default test +suite remains DB-free. Run this only against an isolated local test container on +the user's M3 test machine. + +Start PostgreSQL locally: + +```bash +docker run --rm --name carbonops-ph011-postgres \ + -e POSTGRES_PASSWORD=carbonops_local_test \ + -e POSTGRES_USER=carbonops \ + -e POSTGRES_DB=carbonops_parser_integration_test \ + -p 54329:5432 \ + postgres:16 +``` + +In a second shell, run the focused integration test with an externally supplied +local DSN: + +```bash +CARBONOPS_RUN_POSTGRESQL_INTEGRATION=1 \ +CARBONOPS_POSTGRESQL_TEST_DSN='' \ +python -m pytest -m postgresql_integration tests/test_postgresql_runtime_year_state.py +``` + +The test creates a unique `carbonops_ph011_` schema, creates missing Phase +1 tables with `CREATE TABLE IF NOT EXISTS` and `CREATE INDEX IF NOT EXISTS`, +records minimal GHG Protocol year-state rows, verifies latest-year and next-year +behavior, and verifies DEFRA/DESNZ no-data behavior returns the default initial +year `2024`. + +Do not use production, staging, shared development, customer, or confidential +databases. Do not commit DSNs, passwords, container logs, or machine-specific +paths. After the run, unset both integration controls: + +```bash +unset CARBONOPS_RUN_POSTGRESQL_INTEGRATION +unset CARBONOPS_POSTGRESQL_TEST_DSN +``` + +## PH-017 Production E2E Docker PostgreSQL Validation + +PH-017 is the final production E2E release-validation pass for the source +families `ghg_protocol`, `defra_desnz`, and `ipcc_efdb`. Run it only on the +user's isolated Apple M3 Docker PostgreSQL test machine with an externally +supplied local test DSN. + +Start PostgreSQL locally: + +```bash +docker run --rm --name carbonops-ph017-postgres \ + -e POSTGRES_PASSWORD=carbonops_local_test \ + -e POSTGRES_USER=carbonops \ + -e POSTGRES_DB=carbonops_parser_integration_test \ + -p 54329:5432 \ + postgres:16 +``` + +In a second shell, run the focused PH-017 integration tests: + +```bash +CARBONOPS_RUN_POSTGRESQL_INTEGRATION=1 \ +CARBONOPS_POSTGRESQL_TEST_DSN='' \ +python -m pytest -m postgresql_integration \ + tests/test_ghg_protocol_production_e2e.py \ + tests/test_defra_desnz_production_e2e.py \ + tests/test_ipcc_efdb_production_e2e.py \ + tests/test_postgresql_runtime_year_state.py +``` + +Then run the default release checks: + +```bash +python scripts/release_validation_gate.py +python scripts/production_rc_verification.py +git diff --check +``` + +PH-017 M3 execution record: + +- status: `passed` +- Docker PostgreSQL E2E integration: `4 passed, 22 deselected`. +- `dotnet restore`: completed. +- `python scripts/release_validation_gate.py`: passed. +- focused .NET production-safety contract tests: `17 passed`. +- `python scripts/production_rc_verification.py`: `Passed true`. +- `python -m pytest`: `2062 passed`. +- `git diff --check`: passed. +- result: PH-017 source-family Docker PostgreSQL E2E validation passed. +- secret handling: no DSN, password, credential, token, or secret value is + recorded in this runbook. + +Accepted risks remain explicit: + +- Live source URL/default discovery remains a release risk. +- No source-owner correctness claim is made. +- No factor correctness claim is made. +- No legal correctness claim is made. +- No compliance correctness claim is made. + +Expected PH-017 evidence shape: + +- GHG Protocol, DEFRA/DESNZ, and IPCC EFDB are all explicitly reported. +- Schema bootstrap creates or verifies required tables additively. +- No existing data selects target year `2024`. +- Existing `2024` selects `2025`. +- Existing `2025` selects `2026`. +- Existing `2026` selects `2027`. +- Unavailable target-year source data reports `no_available_source_year` + without inserts or year-state advancement. +- Available target-year runs download, archive metadata, parse, validate, + insert, and advance latest year only after successful insert. +- Repeated execution does not duplicate normalized factor records. +- DB/config failures are sanitized and do not expose DSNs, passwords, tokens, or + raw configured values. + +Current PH-017 execution record: + +- status: `passed` +- date: `2026-05-15` +- environment: user's Apple M3 Docker PostgreSQL machine. +- opt-in PostgreSQL E2E command: `4 passed, 22 deselected`. +- release validation gate: passed. +- production RC verification: `Passed true`. +- default Python test suite: `2062 passed`. +- focused .NET production-safety contract tests: `17 passed`. +- `dotnet restore`: completed. +- `git diff --check`: passed. +- validation record: + [PH-017 Production E2E Docker PostgreSQL Release Validation](ph-017-production-e2e-docker-postgresql-release-validation.md). +- boundaries: live source URL/default discovery remains operator-reviewed; no + source-owner, factor, legal, or compliance correctness claim is made. + +After the manual run, unset both integration controls: + +```bash +unset CARBONOPS_RUN_POSTGRESQL_INTEGRATION +unset CARBONOPS_POSTGRESQL_TEST_DSN +``` + ## Verifying Default Tests Remain DB-Free Before and after future integration-test work, reviewers should run: diff --git a/docs/postgresql-runtime-readiness-checklist.md b/docs/postgresql-runtime-readiness-checklist.md index 96907712..3ecd0d64 100644 --- a/docs/postgresql-runtime-readiness-checklist.md +++ b/docs/postgresql-runtime-readiness-checklist.md @@ -1,146 +1,348 @@ # PostgreSQL Runtime Readiness Checklist -This document defines the go/no-go checklist before any future task enables real -PostgreSQL runtime execution. - -It is checklist documentation only. It does not create a PostgreSQL connection, -create a cursor, run SQL, write records, start a transaction, finish a -transaction, roll back a transaction, create tables, run migrations, load -environment variables, load configuration files, load credentials, perform HTTP -or network calls, schedule work, or claim production persistence readiness. - -## Current Boundary - -Runtime PostgreSQL execution remains disabled. The current implementation -supports planning, preview, and diagnostic metadata only: - -- `PostgreSQLPersistenceRepository.persist()` remains unsupported/no-execution. -- The insert SQL builder produces deterministic parameterized metadata only. -- The persistence preview and local dry-run preview are deterministic and - no-execution. -- The disabled runtime execution adapter returns no-execution metadata. -- The repository disabled execution preview composes diagnostics only. -- The runtime execution gate is disabled by default and does not enable - persistence when requested. - -## Go/No-Go Criteria - -A future real runtime execution task is blocked until all of these are true: - -- Dependency boundary verified: `psycopg` is the approved PostgreSQL driver and - no competing driver or ORM is introduced for Phase 1. -- `psycopg` imports isolated: pure preview/domain modules, local dry-run code, - insert builder, schema descriptor, and repository skeleton remain driver-free. -- Caller-provided session contract ready: future execution consumes a - caller-provided session boundary and does not create implicit connections. -- psycopg session adapter skeleton isolated: the dedicated skeleton remains the - only psycopg-specific boundary until a scoped runtime adapter task. -- Transaction policy agreed: single-batch, caller-provided-session, - no-partial-success policy is explicitly accepted or replaced by a reviewed - policy. -- Idempotency/conflict strategy agreed: Phase 1 duplicate handling is explicit - and not ambiguous. -- Runtime execution gate default disabled: default gate decision remains - disabled/no-execution. -- Repository disabled preview available: repository-level diagnostics can be - reviewed without runtime persistence. -- Disabled runtime execution result available: future execution metadata can be - inspected without connecting or running SQL. -- Public safety checks pass without weakening rules. -- Integration test opt-in plan exists, the opt-in integration runbook is - reviewed, marker enforcement tests pass, the connection smoke skeleton remains - default-skipped, and normal test runs do not touch PostgreSQL. -- No credentials, config files, or environment variables are loaded by library - code. -- No secrets appear in docs, tests, logs, fixtures, examples, exceptions, or - result metadata. -- No production persistence readiness claim is made. -- Repository `persist()` remains unsupported until an explicit future runtime - task changes it with tests and safety review. - -## Must Not Proceed If - -Do not begin a real execution task if any of these are true: - -- DB credentials are not isolated. -- Schema, table creation, or migration lifecycle ownership is unclear. -- PostgreSQL integration tests are not opt-in and disabled by default. -- Duplicate or conflict policy is ambiguous. -- Transaction rollback policy is ambiguous. -- The public safety script would need weakening to pass. -- Repository `persist()` would imply success without integration and rollback - tests. -- Real source parser correctness is assumed but unverified. -- Local dry-run default behavior would change. -- Preview output would become nondeterministic. - -## Future Task Sequence - -Suggested follow-up sequence after this checklist: - -1. CO-103A: opt-in PostgreSQL integration test environment and runbook, with no - default execution. -2. CO-103B: PostgreSQL integration marker enforcement with no DB connection. -3. CO-103C: opt-in connection smoke skeleton, default-skipped and no SQL. -4. CO-103D: runtime session adapter execution smoke behind an explicit test - fixture, opt-in only. -5. CO-103E: repository execution adapter implementation behind the runtime - execution gate, opt-in only. -6. CO-103F: transaction rollback integration tests. -7. CO-103G: conflict and idempotency runtime behavior tests. -8. CO-103H: CLI/config ownership for local PostgreSQL validation if needed. - -Each task should remain separately scoped, reviewed, and validated. None should -silently convert default repository behavior into runtime persistence. - -## First Real Execution Task Acceptance Criteria - -The first task that adds real PostgreSQL execution must satisfy all of these: - -- It is opt-in. -- It uses a caller-provided session. -- It does not read environment variables or config files in library code. -- It does not run in the default test suite. -- It reports sanitized errors. -- It proves secrets do not appear in logs, docs, fixtures, exceptions, or result - metadata. -- It proves rollback behavior or explicitly defers rollback to a named - follow-up before any broad execution path exists. -- It shows the exact PostgreSQL table and schema expectation. -- It does not change local dry-run default behavior. -- It keeps preview behavior deterministic. -- It keeps `PostgreSQLPersistenceRepository.persist()` unsupported unless the - task explicitly changes repository runtime behavior with gate checks and - opt-in tests. - -## Risk Register - -- Credential leakage: require redacted metadata, no committed credentials, and - no secret values in logs or exceptions. -- Accidental default execution: keep the runtime execution gate disabled by - default and require explicit opt-in. -- Partial writes: start with one batch transaction policy and deterministic - rollback reporting. -- Schema drift: compare schema descriptor, DDL review text, and insert builder - columns before runtime execution. -- Duplicate inserts: require an approved conflict policy and explicit counts. -- Placeholder incompatibility: verify psycopg placeholder behavior against - insert-builder output before runtime execution. -- Test DB pollution: isolate opt-in test databases and document cleanup. -- Production misuse: avoid default DB targets and production-readiness claims. -- Dependency footprint: keep database imports inside runtime adapter boundaries. -- Unclear operational ownership: document who owns migrations, config, rollback, - logging, and audit metadata. +This checklist defines the operator checks that must pass before a +CarbonOps-Parser Python ingestion deployment is treated as ready for the +supported production operator path. +It reflects the current packaged runtime: `carbonops-parser run-ingestion` +opens PostgreSQL, runs additive schema bootstrap, ingests configured source +families, and writes source-family master/detail tables. + +This checklist supports the final project-level production-ready verdict only +inside the narrow scope documented in +[Final Project Production-Ready Verdict](final-project-production-ready-verdict.md). +PROD-009 adds an opt-in .NET Docker PostgreSQL E2E/idempotency validation +baseline for all three Phase 1 source families using checked-in local source +fixtures. PROD-010 adds an opt-in Python/.NET persisted PostgreSQL parity +baseline for the same fixture-backed source-specific output. These baselines are +final readiness evidence for the supported scope, but they do not make the .NET +service `run-once` command a production ingestion command. + +## Supported Runtime Boundary + +Python production path: + +- Entrypoint: `carbonops-parser run-ingestion --config --cycles 1`. +- Configuration validation: `carbonops-parser validate-ingestion-config --config --cycles 1`. +- Scheduling: cron or manual scheduled execution of a one-cycle command. +- Source families: `ghg_protocol`, `defra_desnz`, and `ipcc_efdb`. +- Source access: local paths, `file:` URIs, `local:` URIs, or reviewed HTTPS + artifacts when live access is explicitly enabled. +- PostgreSQL driver: psycopg through the `postgresql` Python extra. + +.NET parity baseline: + +- Entrypoint/status: `dotnet run --project src/dotnet/CarbonOps.Parser.Service -- validate-postgresql-runtime`. +- PostgreSQL driver boundary: Npgsql, opened only by explicit runtime methods. +- Schema baseline: additive `CREATE TABLE IF NOT EXISTS` and + `CREATE INDEX IF NOT EXISTS` statements for the Phase 1 runtime catalog. +- Year-state baseline: latest successful year lookup, default initial year + `2024`, next-year calculation, and idempotent successful-year recording. +- Source-cycle preview baseline: configured local CSV/text artifacts only; no + live network calls. +- Source-specific persistence baseline: additive source-family master/detail + inserts with duplicate skip counts and year-state update after successful + persistence. +- Opt-in E2E baseline: Docker PostgreSQL contract tests validate schema + bootstrap, GHG Protocol, DEFRA/DESNZ, and IPCC EFDB local fixture parsing, + first insert, duplicate rerun skips, year-state progression, + `no_available_source_year`, failure rollback, and redaction. +- Opt-in persisted parity baseline: Python and .NET test paths write the same + checked-in local fixtures into isolated PostgreSQL schemas and compare stable + source-specific master/detail output, year-state, idempotent rerun behavior, + source-family identifiers, core row counts, and deterministic external keys + without comparing volatile timestamps. +- Unsupported in .NET today: production `run-once` ingestion execution and + uncontrolled live source access. + +The older preview-only `PostgreSQLPersistenceRepository.persist()` boundary is +still unsupported. Production ingestion uses +`PostgreSQLSourceFamilyRuntimeRepository` through the configured cycle runner. + +## Required PostgreSQL Privileges + +The runtime role needs the minimum privileges below on the target database and +schema: + +- `CONNECT` on the configured database. +- `USAGE` on the target schema. +- `CREATE` on the target schema for additive bootstrap. +- `SELECT`, `INSERT`, and `UPDATE` on the Phase 1 runtime tables. +- Sequence privileges if the operator changes table definitions to use + sequences. + +Do not grant destructive privileges just to run the parser. Backup/restore, +monitoring, retention, audit export, and credential rotation are owned by the +operator's production platform unless this repository later implements a +specific integration. + +## Schema Bootstrap + +Schema bootstrap is additive and idempotent: + +- Tables use `CREATE TABLE IF NOT EXISTS`. +- Indexes use `CREATE INDEX IF NOT EXISTS`. +- Bootstrap commits after the DDL batch. +- Bootstrap reports required, present, created, and still-missing table names. +- Bootstrap does not drop, truncate, rename, or destructively migrate tables. + +Required Phase 1 tables: + +- `source_family_year_states` +- `ingestion_runs` +- `source_documents` +- `parser_runs` +- `schema_bootstrap_states` +- `normalized_factor_records` +- `ghg_emission_factor_masters` +- `ghg_emission_factor_details` +- `defra_emission_factor_masters` +- `defra_emission_factor_details` +- `ipcc_emission_factor_masters` +- `ipcc_emission_factor_details` + +## Before First Run + +Every item must be recorded as PASS or FAIL: + +- PASS/FAIL: Clean install completed with `python -m pip install -e ".[postgresql]"`. +- PASS/FAIL: `carbonops-parser --help` works in the deployment environment. +- PASS/FAIL: Production JSON is present outside the repository and contains + `archive_root`, `enabled_source_families`, `initial_year`, `cycle.max_cycles`, + `cycle.interval_seconds`, `real_source_smoke.allow_live_source_access`, and + explicit `source_years` entries for enabled families. +- PASS/FAIL: Required `CARBONOPS_POSTGRESQL_*` environment variables are + supplied externally, including `CARBONOPS_POSTGRESQL_PASSWORD`. +- PASS/FAIL: `carbonops-parser validate-ingestion-config --config --cycles 1` + reports `status=ready`. +- PASS/FAIL: Archive root exists or can be created by the runtime user. +- PASS/FAIL: Source artifact paths exist, or reviewed HTTPS live source access + is explicitly enabled. +- PASS/FAIL: The target database and schema are correct. +- PASS/FAIL: A backup/restore point exists according to operator policy. +- PASS/FAIL: Runtime role privileges match the minimum privilege list. +- PASS/FAIL: DB connectivity passed with an isolated/pre-production smoke. +- PASS/FAIL: Logs and validation output do not contain passwords, tokens, or + private DSNs. + +## First Run Command + +Use one cycle for production scheduling: + +```bash +export CARBONOPS_POSTGRESQL_HOST='' +export CARBONOPS_POSTGRESQL_PORT='5432' +export CARBONOPS_POSTGRESQL_DATABASE='' +export CARBONOPS_POSTGRESQL_USERNAME='' +export CARBONOPS_POSTGRESQL_PASSWORD='' +export CARBONOPS_POSTGRESQL_APPLICATION_NAME='carbonops-parser-prod' +export CARBONOPS_POSTGRESQL_SSL_MODE='require' +export CARBONOPS_POSTGRESQL_INITIAL_YEAR='2024' + +carbonops-parser run-ingestion \ + --config /etc/carbonops-parser/ingestion.production.json \ + --cycles 1 +``` + +## SQL Verification + +Required schema/table presence: + +```sql +SELECT table_name +FROM information_schema.tables +WHERE table_schema = current_schema() + AND table_name IN ( + 'source_family_year_states', + 'ghg_emission_factor_masters', + 'ghg_emission_factor_details', + 'defra_emission_factor_masters', + 'defra_emission_factor_details', + 'ipcc_emission_factor_masters', + 'ipcc_emission_factor_details' + ) +ORDER BY table_name; +``` + +GHG Protocol counts: + +```sql +SELECT 'ghg_emission_factor_masters' AS table_name, count(*) AS records +FROM ghg_emission_factor_masters +UNION ALL +SELECT 'ghg_emission_factor_details', count(*) +FROM ghg_emission_factor_details; +``` + +DEFRA/DESNZ counts: + +```sql +SELECT 'defra_emission_factor_masters' AS table_name, count(*) AS records +FROM defra_emission_factor_masters +UNION ALL +SELECT 'defra_emission_factor_details', count(*) +FROM defra_emission_factor_details; +``` + +IPCC EFDB counts: + +```sql +SELECT 'ipcc_emission_factor_masters' AS table_name, count(*) AS records +FROM ipcc_emission_factor_masters +UNION ALL +SELECT 'ipcc_emission_factor_details', count(*) +FROM ipcc_emission_factor_details; +``` + +Latest ingested year / year-state: + +```sql +SELECT source_family, max(ingested_year) AS latest_ingested_year +FROM source_family_year_states +GROUP BY source_family +ORDER BY source_family; +``` + +## Idempotent Rerun Check + +Run the same one-cycle command again against the same database/schema. PASS +requires: + +- The command exits successfully, or reports only expected + `no_available_source_year` outcomes. +- Duplicate source-family master/detail rows are reported as skipped, not + inserted again. +- Latest-year state does not advance when the source year is unavailable. +- Logs remain redacted. + +## .NET Docker PostgreSQL E2E Check + +The .NET Docker PostgreSQL E2E path is opt-in and test-only. The default test +suite must not require PostgreSQL or credentials. + +Enable it only against an operator-owned disposable PostgreSQL database or a +local Docker PostgreSQL instance: + +```bash +export CARBONOPS_RUN_DOTNET_POSTGRESQL_INTEGRATION=1 +export CARBONOPS_DOTNET_POSTGRESQL_TEST_DSN='' + +dotnet test tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj \ + --configuration Release \ + --no-restore \ + --filter "FullyQualifiedName~DotNetPostgreSQLIntegrationE2ETests" +``` + +Instead of a DSN, the tests may use externally supplied split +`CARBONOPS_PARSER_POSTGRES_*` settings accepted by the .NET production config +boundary. Do not put credentials in repository files. Do not paste raw DSNs into +logs or tickets. + +The test path creates a generated PostgreSQL-safe schema name when possible so +first-run and rerun counts are deterministic without destructive SQL. The +runtime role therefore needs permission to create an isolated test schema, or +the operator must provide a disposable pre-approved schema through +`CARBONOPS_DOTNET_POSTGRESQL_TEST_SCHEMA`. + +PASS requires: + +- PostgreSQL is not contacted unless + `CARBONOPS_RUN_DOTNET_POSTGRESQL_INTEGRATION=1` is set. +- Missing DSN/split config fails closed before a connection attempt. +- Schema bootstrap succeeds twice and the second run creates no tables. +- The checked-in GHG Protocol, DEFRA/DESNZ, and IPCC EFDB local CSV fixtures + parse and insert source-specific master/detail rows. +- Rerunning each same source-family/year/artifact reports `inserted=0` and + duplicate skipped counts greater than zero. +- `source_family_year_states` records one logical successful year for each + source family at `2024`, with next target year `2025`. +- Missing target-year artifacts report `no_available_source_year` and do not + advance year-state. +- A persistence failure rolls back and does not advance year-state. +- Diagnostics redact passwords, connection strings, and DSNs. + +This check is project-level readiness evidence for the .NET Docker PostgreSQL +E2E slice. It does not prove .NET service `run-once` readiness, live source +handling, or Python/.NET persisted parity by itself. + +## Python/.NET Persisted Parity Check + +The persisted parity check is opt-in and test-only. It must run only against an +operator-owned disposable PostgreSQL database or local Docker PostgreSQL +instance: + +```bash +export CARBONOPS_RUN_PERSISTED_PARITY_VALIDATION=1 +export CARBONOPS_RUN_POSTGRESQL_INTEGRATION=1 +export CARBONOPS_POSTGRESQL_TEST_DSN='' + +dotnet restore src/dotnet/CarbonOps.Parser.sln +python -m pytest -q tests/test_postgresql_persisted_parity_validation.py +``` + +The test creates isolated generated schemas for Python and .NET, invokes the +.NET fixture persistence baseline with an externally supplied schema, and +compares stable persisted output only. It does not print DSNs or credentials. +It does not drop, truncate, or destructively clean shared database objects. + +PASS requires: + +- PostgreSQL is not contacted unless + `CARBONOPS_RUN_PERSISTED_PARITY_VALIDATION=1` and + `CARBONOPS_RUN_POSTGRESQL_INTEGRATION=1` are set with an external test DSN. +- GHG Protocol, DEFRA/DESNZ, and IPCC EFDB fixtures all persist in both runtime + paths. +- Persisted source-family identifiers are `ghg_protocol`, `defra_desnz`, and + `ipcc_efdb`. +- Master/detail row counts, source years, source versions, external keys, + factor fields, statuses, latest successful year `2024`, and next target year + `2025` match. +- Same-fixture reruns skip duplicate master/detail records instead of inserting + duplicates. +- Volatile timestamps and generated UUIDs are not compared. + +This check is fixture-backed persisted parity evidence for the final verdict. +It does not make the `.NET run-once` command production-ready. + +## Failure Blocks + +Treat any item below as a production-readiness failure until resolved: + +- Missing DB config such as `POSTGRESQL_RUNTIME_CONFIG_MISSING_HOST`, + `POSTGRESQL_RUNTIME_CONFIG_MISSING_DATABASE`, + `POSTGRESQL_RUNTIME_CONFIG_MISSING_USERNAME`, or + `POSTGRESQL_RUNTIME_CONFIG_MISSING_PASSWORD`. +- Bad DB credentials, unreachable host, invalid port, wrong database, or + rejected SSL mode. +- Missing or unwritable `archive_root`. +- Unsupported source family outside `ghg_protocol`, `defra_desnz`, and + `ipcc_efdb`. +- Missing `source_years...artifact_url`. +- HTTPS artifact configured without explicit live source access. +- Unexpected `no_available_source_year` for a planned source family/year. +- Schema bootstrap reports missing required tables after execution. +- Any log, ticket, artifact, or test output exposes a password, token, private + DSN, or real secret value. + +## Production Checklist + +All items must be PASS: + +- PASS/FAIL: Clean install. +- PASS/FAIL: Config loaded and validated. +- PASS/FAIL: DB connectivity. +- PASS/FAIL: Schema bootstrap. +- PASS/FAIL: One source-family smoke or full three-source smoke. +- PASS/FAIL: Idempotent rerun. +- PASS/FAIL: Redaction check. +- PASS/FAIL: Full Python test baseline with `python -m pytest`. +- PASS/FAIL: `git diff --check`. +- PASS/FAIL: `git status --short` clean after validation. ## Related Documents -- [PostgreSQL Implementation Safety Gate](postgresql-implementation-safety-gate.md) -- [PostgreSQL Runtime Execution Gate Boundary](postgresql-runtime-execution-gate-boundary.md) -- [PostgreSQL Runtime Persistence Implementation Plan](postgresql-runtime-persistence-implementation-plan.md) -- [PostgreSQL Driver Dependency Decision](postgresql-driver-dependency-decision.md) -- [PostgreSQL Connection Session Contract Boundary](postgresql-connection-session-contract-boundary.md) -- [PostgreSQL psycopg Session Adapter Boundary](postgresql-psycopg-session-adapter-boundary.md) -- [PostgreSQL Disabled Runtime Execution Adapter Boundary](postgresql-disabled-runtime-execution-adapter-boundary.md) -- [PostgreSQL Repository Disabled Execution Preview Boundary](postgresql-repository-disabled-execution-preview-boundary.md) -- [PostgreSQL Integration Test Boundary](postgresql-integration-test-boundary.md) +- [Production Packaging And Operator Runbook](production-packaging-operator-runbook.md) +- [Production Parity Contract](production-parity-contract.md) +- [Python Ingestion Local Runbook](python-ingestion-local-runbook.md) +- [Real-Source Smoke Mode](real-source-smoke-mode.md) - [PostgreSQL Opt-In Integration Runbook](postgresql-opt-in-integration-runbook.md) +- [PostgreSQL Phase 1 Schema Contract](postgresql-phase1-schema-contract.md) diff --git a/docs/production-e2e-ingestion-readiness-contract.md b/docs/production-e2e-ingestion-readiness-contract.md new file mode 100644 index 00000000..352f9ce0 --- /dev/null +++ b/docs/production-e2e-ingestion-readiness-contract.md @@ -0,0 +1,282 @@ +# Production E2E Ingestion Readiness Contract + +This document defines the production end-to-end ingestion readiness contract for +CarbonOps-Parser. + +PH-018 superseded earlier test-harness-oriented production-ready +interpretations. PROD-002 further clarifies that project-level production-ready +requires Python and .NET runtime parity. The authoritative parity definition is +[Production Parity Contract](production-parity-contract.md), with +[PH-018 Real Production-Ready Ingestion Contract](ph-018-real-production-ready-ingestion-contract.md) +retained as the runtime ingestion loop contract. + +It is documentation only. It does not implement runtime code, call live +endpoints, execute database operations, create credentials, download source +files, parse real upstream documents, validate factor correctness, or claim +production carbon-accounting correctness. + +## Production Definition + +For either runtime, production E2E ingestion means one run performs this +operational sequence: + +1. Check PostgreSQL connectivity, schema state, and required tables. +2. Create missing required PostgreSQL tables safely if the configured runtime + mode allows schema bootstrap. +3. For each selected source family, inspect PostgreSQL for the latest ingested + source year. +4. If no data exists for that source family, select the configured initial year. +5. If data exists, calculate `next_year = latest_year + 1`. +6. Attempt source-specific discovery and download for `next_year` only. +7. If source data exists for `next_year`, download it, archive it, parse it, + validate it, and insert accepted records into PostgreSQL. +8. If source data does not exist or is unavailable for `next_year`, do not fail + the whole run; report `no_available_source_year`. +9. Return a run summary with per-family year state, action taken, inserted + counts, skipped/no-op counts, validation failures, and failure details. + +The default configured initial year is `2024`. A deployment may explicitly +configure a different initial year. The effective initial year must be visible in +run configuration and run output. + +## Source Families + +The contract applies to these Phase 1 source families: + +- `ghg_protocol` for GHG Protocol. +- `defra_desnz` for DEFRA/DESNZ. +- `ipcc_efdb` for IPCC EFDB. + +Each source family owns its own discovery, download, archive, parser, +validation, and PostgreSQL table mapping. A production E2E run must not infer +availability for one source family from another source family. + +## PostgreSQL Readiness + +On first run, the service must inspect PostgreSQL before source work begins. + +The readiness check must verify: + +- PostgreSQL can be reached with configured non-secret connection metadata. +- The expected schema exists or can be created safely. +- Required source-family tables exist or can be created safely. +- Required indexes and uniqueness constraints for idempotent insertion exist or + can be created safely. +- Schema bootstrap is explicit in configuration and observable in run output. +- Schema bootstrap failures stop source execution and report a structured + PostgreSQL readiness failure. + +Table creation must be additive and safe. It must not drop tables, truncate +tables, delete records, rewrite existing source data, weaken constraints, or +silently migrate incompatible existing schemas. + +## Year-State Contract + +For each selected source family, the run must determine exactly one target year. + +If PostgreSQL has no ingested data for the source family: + +- `latest_year` is absent. +- `target_year` is the configured initial year. +- The default `target_year` is `2024` unless explicitly configured otherwise. +- The run status for year selection is `initial_year_selected`. + +If PostgreSQL has ingested data for the source family: + +- `latest_year` is the greatest year already committed for that source family. +- `target_year` is `latest_year + 1`. +- The run status for year selection is `next_year_selected`. + +The run must not scan, download, backfill, or skip ahead to other years unless a +future task explicitly defines that behavior. This contract selects only the +initial year or the single next year. + +## Discovery And Download Contract + +Discovery and download are source-specific and target-year scoped. + +For each selected source family: + +- Discovery must be requested for `target_year` only. +- Download must be attempted only when discovery confirms an available source + document for `target_year`. +- Downloaded source files must be archived with enough metadata to support + replay and audit of the run. +- Discovery and download must produce structured status metadata without + exposing credentials or secrets. + +If a source owner has not published data for `target_year`, or the source is +temporarily unavailable in a way the source-specific adapter classifies as no +available year, the source-family result must be `no_available_source_year`. + +`no_available_source_year` is a successful no-op for that source family. It must +not be treated as a failed production run unless every selected family is blocked +by an unrelated hard failure. + +## Parse, Validate, And Insert Contract + +When `target_year` source data exists: + +- The archived source document is parsed by the source-family parser. +- Parsed records are validated before insertion. +- Validation failures are reported with structured counts and reasons. +- Accepted records are inserted into PostgreSQL in source-family tables. +- Insert results are reported with attempted, inserted, skipped, failed, and + validation-failed counts. + +Validation proves only that records satisfy the repository's explicit structural +and contract checks. It does not prove source-owner correctness, emission factor +correctness, unit conversion correctness, legal correctness, compliance +correctness, or carbon-accounting correctness. + +## Idempotency Contract + +Production inserts must be idempotent and safe on repeated execution. + +For the same source family, source year, source document identity, and parsed +record identity, repeated execution must not create duplicate logical records. +The implementation must use reviewed PostgreSQL uniqueness constraints and a +documented conflict policy. + +Repeated execution of a completed year must report deterministic duplicate or +already-ingested outcomes. It must not silently change existing records unless a +future reviewed task defines an explicit upsert or replacement policy. + +If a prior run partially failed before commit, retry behavior must be governed by +the PostgreSQL transaction policy. A failed transaction must not leave ambiguous +partial ingestion state. + +## No-Op Semantics + +A source family no-ops when: + +- PostgreSQL readiness succeeds. +- A target year is selected. +- Source-specific discovery determines that target-year data is not available. +- No download, parse, validation, or insert work is performed for that family. +- The family result is `no_available_source_year`. + +A no-op must include: + +- source family, +- latest ingested year, when present, +- target year, +- initial year configuration, when used, +- source discovery status, +- no-op reason, +- timestamp or run identifier, and +- zero attempted/inserted record counts. + +The whole run may complete with a no-op result when all selected source families +report `no_available_source_year`. + +## Docker PostgreSQL Integration Expectations + +Integration tests for future implementation work must run against Docker +PostgreSQL on the user's Apple M3 development machine. + +The expected integration test boundary is: + +- Docker starts a PostgreSQL container for tests. +- Tests use an isolated database or schema per run. +- Tests create or verify required tables through the same bootstrap path used by + the runtime. +- Tests cover first-run schema bootstrap. +- Tests cover no-existing-data initial year selection with default `2024`. +- Tests cover configured initial year override. +- Tests cover latest-year lookup and `next_year = latest_year + 1`. +- Tests cover `no_available_source_year` without failing the whole run. +- Tests cover idempotent repeated execution for already-ingested records. +- Tests cover transaction rollback or another reviewed no-partial-write policy. +- Tests must not call live endpoints. +- Tests must not require production credentials. + +Default lightweight test runs may remain local-only, but implementation tasks +that claim production E2E ingestion behavior must include opt-in Docker +PostgreSQL integration validation. + +## Required Observable Statuses + +Future runtime results should expose stable statuses at the run and per-family +level. + +Required per-family statuses: + +- `postgresql_not_ready` +- `initial_year_selected` +- `next_year_selected` +- `source_year_available` +- `no_available_source_year` +- `downloaded` +- `archived` +- `parsed` +- `validated` +- `inserted` +- `completed` +- `completed_with_validation_failures` +- `failed` + +The exact enum names may be refined by a future implementation task, but the +observable meanings must remain explicit. + +## Non-Goals + +This contract does not add or certify: + +- Runtime source ingestion. +- Live endpoint calls. +- Database execution. +- Scheduler behavior. +- Parser correctness. +- Factor correctness. +- Source-owner correctness. +- Unit conversion correctness. +- Carbon-accounting correctness. +- Compliance or legal correctness. +- Production credentials. +- Backfill across multiple years. +- Multi-source-year catch-up in one family run. +- Destructive database migrations. + +## Follow-Up Implementation Tasks + +Future work should be split into focused tasks: + +1. Define runtime configuration for selected source families, initial years, + schema bootstrap mode, and PostgreSQL connection ownership. +2. Implement PostgreSQL schema bootstrap with additive table creation and + integration tests. +3. Implement latest-ingested-year queries per source family. +4. Implement target-year planning using configured initial year or + `latest_year + 1`. +5. Implement target-year-only discovery adapters for GHG Protocol, DEFRA/DESNZ, + and IPCC EFDB without broad scans. +6. Implement target-year download and archive metadata. +7. Implement source-family parser and validation execution against archived + documents. +8. Implement PostgreSQL idempotent insert behavior with reviewed uniqueness and + conflict policy. +9. Implement structured run summaries and `no_available_source_year` no-op + reporting. +10. Add Docker PostgreSQL integration tests for first run, next-year selection, + no-op availability, idempotency, and rollback behavior on Apple M3. +11. Add operator runbook updates after implementation behavior exists. + +## Runtime Parity + +The Python runtime currently has a documented production operator path. The .NET +runtime is not production-ready yet. Project-level production-ready is blocked +until both runtimes implement this ingestion behavior against the same +PostgreSQL schema with equivalent idempotency, redaction, no-op, and operator +behavior. + +See [Production Parity Contract](production-parity-contract.md). + +## PR Footer Requirement + +The pull request body for this task must end with: + +```text +Task-ID: PH-010 +Task-Issue: #576 +``` diff --git a/docs/production-e2e-year-orchestrator-boundary.md b/docs/production-e2e-year-orchestrator-boundary.md new file mode 100644 index 00000000..03d54834 --- /dev/null +++ b/docs/production-e2e-year-orchestrator-boundary.md @@ -0,0 +1,54 @@ +# Production E2E Year Orchestrator Boundary + +PH-013 adds a Python runtime boundary for one production E2E source-year step. +The boundary is implemented in +`carbonfactor_parser.pipeline.production_e2e_year_orchestrator`. + +The orchestrator is dependency-injected. It coordinates: + +- PostgreSQL source-family year state. +- Canonical source-family selection for `ghg_protocol`, `defra_desnz`, and + `ipcc_efdb`. +- Initial-year and next-year target calculation. +- Source-family target-year discovery and download interfaces. +- Parser execution through an injected parser boundary. +- Validation through an injected validation boundary. +- PostgreSQL normalized-factor insert through an injected insert repository. + +The implementation does not add live source adapters, source-specific parser +details, network calls, credentials, scheduling, or carbon-accounting +correctness claims. + +## Year Selection + +For each enabled source family: + +- If PostgreSQL has no latest ingested year, the target year is the configured + initial year. The default is `2024`. +- If PostgreSQL returns a latest ingested year, the target year is + `latest_year + 1`. + +The run selects exactly one target year per source family. It does not backfill, +skip ahead, or scan multiple years. + +## No Available Source Year + +If the source-family adapter reports `no_available_source_year`, the family is a +safe no-op: + +- no download is required, +- parser execution is skipped, +- validation is skipped, +- PostgreSQL insert is skipped, +- year state is not advanced, and +- the whole run may still complete successfully. + +## Adapter Scope + +The source-family adapter contract intentionally has only two runtime methods: + +- `discover_target_year(request)` +- `download_target_year(discovery_result)` + +Real source integrations for GHG Protocol, DEFRA/DESNZ, and IPCC EFDB remain +deferred to source-specific implementation tasks. diff --git a/docs/production-packaging-operator-runbook.md b/docs/production-packaging-operator-runbook.md new file mode 100644 index 00000000..cbc577a6 --- /dev/null +++ b/docs/production-packaging-operator-runbook.md @@ -0,0 +1,585 @@ +# Production Packaging And Operator Runbook + +This runbook is the supported operator path for the Python ingestion runtime. +It documents the commands an operator can run today to install, configure, +validate, execute, rerun, stop, and troubleshoot CarbonOps-Parser without +editing Python source files. + +The production ingestion runtime path is the Python operator command. The .NET +solution contributes the parity evidence required by the final verdict: a +scheduled-worker executable baseline, production config validation, PostgreSQL +schema bootstrap/year-state runtime baseline, source-cycle orchestration, +source-specific master/detail insert, Docker PostgreSQL E2E validation, and +Python/.NET persisted parity validation. Its `run-once` command is still a safe +not-yet-implemented placeholder and is not a production ingestion path. + +Project-level production-ready is claimed only in the narrow scope documented +in [Final Project Production-Ready Verdict](final-project-production-ready-verdict.md) +and [Production Parity Contract](production-parity-contract.md). + +The .NET contract/test solution remains available at `src/dotnet/CarbonOps.Parser.sln`. + +## Runtime Surface + +| Surface | Current entrypoint | Production operation status | +| --- | --- | --- | +| Python package | `carbonops-parser` from `pyproject.toml` | Supported for configured PostgreSQL ingestion | +| Source acquisition CLI | `carbonops-source-acquisition` | Supported for local dry-run/source planning checks | +| .NET scheduled-worker baseline | `dotnet run --project src/dotnet/CarbonOps.Parser.Service -- ` | Entrypoint/config-validation, PostgreSQL schema/year-state baseline, source-specific master/detail insert baseline, safe source-cycle preview, Docker PostgreSQL E2E evidence, and persisted parity evidence; `run-once` ingestion not promoted | + +Supported scheduling is cron or manual scheduled execution of the packaged +Python command. There is no daemon, long-running service installer, distributed +lock, or system service wrapper in this repository. + +The .NET scheduled-worker command surface added by PROD-003 is directly +runnable and suitable for future cron/manual scheduling, but `run-once` returns +`ingestion_status=not_implemented` and a non-zero exit code until later .NET +service execution work promotes it. +PROD-009 adds an opt-in Docker PostgreSQL contract-test baseline for all three +local fixture-backed Phase 1 source families. PROD-010 adds an opt-in persisted +PostgreSQL parity baseline for Python and .NET output from the same fixture +families. + +## Safety Modes + +| Mode | Command shape | Purpose | External mutation | +| --- | --- | --- | --- | +| Local fixture/dry-run | `carbonops-parser local-dry-run ...` | Parse deterministic checked-in fixture data and render preview metadata | No | +| Local PostgreSQL smoke | `carbonops-parser real-source-smoke --config config/carbonops.ingestion.example.json --cycles 1` | Validate the packaged Python runtime against an operator-owned local PostgreSQL database | Yes, local DB only | +| Production PostgreSQL | `carbonops-parser run-ingestion --config /etc/carbonops-parser/ingestion.production.json --cycles 1` | Run configured source-family ingestion against the approved production PostgreSQL database | Yes, production DB | + +HTTPS source access is blocked unless the operator explicitly sets +`real_source_smoke.allow_live_source_access` or passes +`--allow-live-source-access` to the smoke command. Production configs must use +reviewed artifact URLs or local artifact paths under `source_years`. + +## Install + +From a clean checkout or packaged deployment directory: + +```bash +python -m pip install -e ".[postgresql]" +carbonops-parser --help +``` + +The `postgresql` extra installs the psycopg binary wrapper used by the runtime. +Do not commit virtual environments, package caches, or machine-local install +artifacts. + +## Configure + +Use JSON for the Python ingestion runtime. Production configuration is split: +non-secret ingestion settings live in an operator-managed JSON file, while +PostgreSQL credentials and connection fields are supplied by environment or the +deployment secret mechanism. + +Example production JSON file: + +```json +{ + "archive_root": "/var/lib/carbonops-parser/raw-archive", + "enabled_source_families": ["ghg_protocol", "defra_desnz", "ipcc_efdb"], + "initial_year": 2024, + "cycle": { + "interval_seconds": 0, + "max_cycles": 1 + }, + "real_source_smoke": { + "allow_live_source_access": false + }, + "source_years": { + "ghg_protocol": { + "2024": { + "artifact_url": "/var/lib/carbonops-parser/sources/ghg_protocol_2024.csv", + "publication_url": "https://", + "title": "GHG Protocol reviewed artifact 2024", + "version_label": "", + "content_type": "text/csv", + "format_hint": "csv" + } + }, + "defra_desnz": { + "2024": { + "artifact_url": "/var/lib/carbonops-parser/sources/defra_desnz_2024.csv", + "publication_url": "https://", + "title": "DEFRA/DESNZ reviewed artifact 2024", + "version_label": "", + "content_type": "text/csv", + "format_hint": "csv" + } + }, + "ipcc_efdb": { + "2024": { + "artifact_url": "/var/lib/carbonops-parser/sources/ipcc_efdb_2024.csv", + "publication_url": "https://", + "title": "IPCC EFDB reviewed artifact 2024", + "version_label": "", + "content_type": "text/csv", + "format_hint": "csv" + } + } + } +} +``` + +A placeholder copy of this shape is checked in at +[../config/carbonops.ingestion.production.example.json](../config/carbonops.ingestion.production.example.json). +It contains no credentials and is not directly runnable until the operator +replaces placeholder artifact paths and source metadata. + +Required runtime environment: + +| Name | Required | Secret | Purpose | +| --- | --- | --- | --- | +| `CARBONOPS_POSTGRESQL_HOST` | Yes, unless DSN is used | No | PostgreSQL host | +| `CARBONOPS_POSTGRESQL_PORT` | Yes | No | PostgreSQL port, normally `5432` | +| `CARBONOPS_POSTGRESQL_DATABASE` | Yes, unless DSN is used | No | Database name | +| `CARBONOPS_POSTGRESQL_USERNAME` | Yes, unless DSN is used | No | Runtime database role | +| `CARBONOPS_POSTGRESQL_PASSWORD` | Yes, unless DSN is used | Yes | Runtime database password supplied externally | +| `CARBONOPS_POSTGRESQL_APPLICATION_NAME` | Yes for production operations | No | PostgreSQL application name, for example `carbonops-parser-prod` | +| `CARBONOPS_POSTGRESQL_SSL_MODE` | Deployment-specific | No | psycopg SSL mode, for example `require` | +| `CARBONOPS_POSTGRESQL_INITIAL_YEAR` | Yes for production operations | No | Initial year for empty year-state tables; keep aligned with JSON `initial_year` | +| `CARBONOPS_POSTGRESQL_DSN` | No | Yes if it embeds credentials | Alternative connection input; avoid in production because it is easier to leak | + +Required JSON keys: + +| Key | Required | Purpose | +| --- | --- | --- | +| `archive_root` | Yes | Operator-managed raw archive directory | +| `enabled_source_families` | Yes | Explicit subset of `ghg_protocol`, `defra_desnz`, `ipcc_efdb` | +| `initial_year` | Yes | First target year when no year-state exists | +| `cycle.max_cycles` | Yes | Use `1` for cron/manual scheduled production runs | +| `cycle.interval_seconds` | Yes | Use `0` for cron/manual scheduled production runs | +| `real_source_smoke.allow_live_source_access` | Yes | Must be explicit; default production recommendation is `false` with staged local artifacts | +| `source_years...artifact_url` | Yes for each enabled family/year | Local path, `file:` URI, `local:` URI, or reviewed HTTPS URL | +| `source_years...publication_url` | Yes | Source publication reference for audit metadata | +| `source_years...title` | Yes | Human-readable source artifact title | +| `source_years...version_label` | Yes | Reviewed version label | +| `source_years...content_type` | Yes | Usually `text/csv` | +| `source_years...format_hint` | Yes | Usually `csv` | + +The only required secret in the split environment path is +`CARBONOPS_POSTGRESQL_PASSWORD`. Provide it through the deployment secret +manager or protected shell environment. Do not put passwords, tokens, private +DSNs, or real credentials in repository files, runbooks, tickets, command +history examples, or logs. + +## Validate + +Run local package and fixture checks first: + +```bash +python -m pytest +git diff --check +python scripts/production_rc_verification.py +carbonops-source-acquisition validate +carbonops-source-acquisition run --dry-run --base-directory ./data/source-acquisition +carbonops-parser local-dry-run \ + --local-path examples/fixtures/defra_desnz_minimal.csv \ + --source-family defra_desnz \ + --source-id defra-desnz-minimal-fixture \ + --content-type text/csv \ + --format-hint csv +dotnet test tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj \ + --configuration Release \ + --no-restore \ + --filter "FullyQualifiedName~ProductionConfigBoundaryTests|FullyQualifiedName~Phase1OperationalDiagnosticsTests|FullyQualifiedName~PostgreSQLRuntimeConfigGateContractTests|FullyQualifiedName~CarbonOpsParserServiceCommandTests" +dotnet test tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj \ + --configuration Release \ + --no-restore \ + --filter "FullyQualifiedName~PostgreSQLRuntimeSchemaAndYearStateTests|FullyQualifiedName~CarbonOpsParserServiceCommandTests" +dotnet test tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj \ + --configuration Release \ + --no-restore \ + --filter "FullyQualifiedName~PostgreSQLSourceSpecificFactorPersistenceTests|FullyQualifiedName~PostgreSQLRuntimeSchemaAndYearStateTests|FullyQualifiedName~CarbonOpsParserServiceCommandTests" +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- help +``` + +The commands above must not require production configuration or credentials. +The focused .NET command is the default release-gate contract subset; the full .NET contract suite is outside the default release gate and remains a separate reviewer/operator validation choice when broader parity evidence is needed. + +Validate production configuration without opening PostgreSQL: + +```bash +export CARBONOPS_POSTGRESQL_HOST='' +export CARBONOPS_POSTGRESQL_PORT='5432' +export CARBONOPS_POSTGRESQL_DATABASE='' +export CARBONOPS_POSTGRESQL_USERNAME='' +export CARBONOPS_POSTGRESQL_PASSWORD='' +export CARBONOPS_POSTGRESQL_APPLICATION_NAME='carbonops-parser-prod' +export CARBONOPS_POSTGRESQL_SSL_MODE='require' +export CARBONOPS_POSTGRESQL_INITIAL_YEAR='2024' + +carbonops-parser validate-ingestion-config \ + --config /etc/carbonops-parser/ingestion.production.json \ + --cycles 1 +``` + +The .NET entrypoint has a separate validation command. It now loads a flat JSON +config file when `--config ` is supplied, loads the process environment, +and lets `CARBONOPS_PARSER_*` environment values override file values. It +validates presence and basic value shape for the expected `.NET` +`CARBONOPS_PARSER_*` keys, reports required key and password presence, does not +connect to PostgreSQL, and does not print secret values: + +```bash +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- validate-config --config /etc/carbonops-parser/dotnet.production.json +``` + +Validate the .NET PostgreSQL runtime baseline without opening PostgreSQL: + +```bash +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- validate-postgresql-runtime --config /etc/carbonops-parser/dotnet.production.json +``` + +Expected baseline result includes `schema_bootstrap_available=True`, +`year_state_available=True`, +`source_specific_master_detail_insert_baseline=True`, +`master_detail_insert_e2e_validated=False`, +`production_ingestion_ready=False`, +`postgresql_connection_opened=False`, +`.net_runtime_production_ready=False`, and +`project_level_production_ready=False`. This PostgreSQL validation command is +not the source-cycle preview command, so it still reports production ingestion +source download and parser orchestration as incomplete. + +Optionally validate the .NET Docker PostgreSQL E2E/idempotency baseline against +a disposable Docker PostgreSQL database. This path is disabled by default and +must use externally supplied credentials: + +```bash +export CARBONOPS_RUN_DOTNET_POSTGRESQL_INTEGRATION=1 +export CARBONOPS_DOTNET_POSTGRESQL_TEST_DSN='' + +dotnet test tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj \ + --configuration Release \ + --no-restore \ + --filter "FullyQualifiedName~DotNetPostgreSQLIntegrationE2ETests" +``` + +The tests may also use split `CARBONOPS_PARSER_POSTGRES_*` settings. They do +not print passwords, DSNs, or connection strings. They validate additive schema +bootstrap, GHG Protocol, DEFRA/DESNZ, and IPCC EFDB local fixture parsing, +source-specific master/detail insert, same-year rerun duplicate skips, +successful year-state progression, `no_available_source_year`, failure +rollback, and redacted diagnostics. They do not make `.NET run-once` a +production ingestion command and do not prove Python/.NET persisted parity. + +Optionally validate Python/.NET persisted PostgreSQL parity against a +disposable database. This path is disabled by default and must use externally +supplied credentials: + +```bash +export CARBONOPS_RUN_PERSISTED_PARITY_VALIDATION=1 +export CARBONOPS_RUN_POSTGRESQL_INTEGRATION=1 +export CARBONOPS_POSTGRESQL_TEST_DSN='' + +dotnet restore src/dotnet/CarbonOps.Parser.sln +python -m pytest -q tests/test_postgresql_persisted_parity_validation.py +``` + +The parity test creates isolated generated schemas, persists the same GHG +Protocol, DEFRA/DESNZ, and IPCC EFDB checked-in fixtures through the Python and +.NET paths, reruns same-year inserts to verify duplicate skips, records +successful `2024` year-state, and compares stable source-specific +master/detail output. It compares source-family identifiers, row counts, +source years, source versions, deterministic external keys, factor fields, +statuses, latest successful year, and next target year. It does not compare +volatile timestamps or generated UUIDs, does not print DSNs or credentials, and +does not perform destructive DB cleanup. + +Preview the .NET source-cycle orchestration baseline without opening PostgreSQL +or writing source-family records: + +```bash +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- preview-source-cycle --config /etc/carbonops-parser/dotnet.production.json +``` + +The preview selects enabled source families from the optional JSON config, +calculates each target year from the year-state contract using the default +initial year `2024` when no successful year exists, checks configured local +artifacts under the narrow `source_artifacts..` config shape, and +hands supported local CSV/text artifacts to the existing normalized parser +contracts. It does not make live network calls, open PostgreSQL, insert +records, or advance year-state. Missing target-year artifacts report +`no_available_source_year`; unsupported artifact shapes report +`parser_not_available`; successful parser handoff reports `parsed` plus +`persistence_not_implemented`. + +Expected validation result: + +```text +status=ready +postgresql_password_configured=True +postgresql_connection_opened=False +secret_values_printed=False +``` + +If validation prints `status=blocked`, fix the named field before opening a DB +connection. + +Raw PostgreSQL connection strings are rejected by the committed configuration +boundary. Use split `.NET` `CARBONOPS_PARSER_*` fields for the .NET entrypoint +and keep production secret values in environment or an operator-managed secret +source rather than committed files. The PROD-004 .NET boundary satisfies only +the config loader/redaction item from the production parity map. The PROD-005 +.NET boundary satisfies only the PostgreSQL schema bootstrap/year-state item. +The PROD-006 .NET boundary satisfies only the source discovery/load/parsing +orchestration item from the production parity map. The PROD-007 .NET boundary +satisfies only the source-specific master/detail insert runtime item from the +production parity map. PROD-010 satisfies the opt-in fixture-backed persisted +parity baseline. .NET production ingestion remains incomplete until service +run-once ingestion execution is promoted by a separately scoped task. + +Validate DB connectivity and schema bootstrap with an isolated local or +pre-production database before production. The integration-test DSN is external +test-runner input and must not be printed: + +```bash +CARBONOPS_RUN_POSTGRESQL_INTEGRATION=1 \ +CARBONOPS_POSTGRESQL_TEST_DSN='' \ +python -m pytest -q \ + tests/test_postgresql_connection_smoke_boundary.py::test_postgresql_opt_in_connection_open_close_smoke \ + tests/test_postgresql_runtime_year_state.py::test_docker_postgresql_schema_bootstrap_and_year_state_integration \ + tests/test_postgresql_source_family_repository.py::test_docker_postgresql_source_specific_master_detail_tables_integration +``` + +## Run + +Local fixture/dry-run mode: + +```bash +carbonops-parser local-dry-run \ + --local-path examples/fixtures/defra_desnz_minimal.csv \ + --source-family defra_desnz \ + --source-id defra-desnz-minimal-fixture \ + --content-type text/csv \ + --format-hint csv \ + --include-postgresql-preview +``` + +Local PostgreSQL smoke mode: + +```bash +export CARBONOPS_POSTGRESQL_HOST='127.0.0.1' +export CARBONOPS_POSTGRESQL_PORT='5432' +export CARBONOPS_POSTGRESQL_DATABASE='carbonops' +export CARBONOPS_POSTGRESQL_USERNAME='carbonops' +export CARBONOPS_POSTGRESQL_PASSWORD='' +export CARBONOPS_POSTGRESQL_APPLICATION_NAME='carbonops-parser-local' +export CARBONOPS_POSTGRESQL_INITIAL_YEAR='2024' + +carbonops-parser real-source-smoke \ + --config config/carbonops.ingestion.example.json \ + --cycles 1 +``` + +Production PostgreSQL mode: + +```bash +export CARBONOPS_POSTGRESQL_HOST='' +export CARBONOPS_POSTGRESQL_PORT='5432' +export CARBONOPS_POSTGRESQL_DATABASE='' +export CARBONOPS_POSTGRESQL_USERNAME='' +export CARBONOPS_POSTGRESQL_PASSWORD='' +export CARBONOPS_POSTGRESQL_APPLICATION_NAME='carbonops-parser-prod' +export CARBONOPS_POSTGRESQL_SSL_MODE='require' +export CARBONOPS_POSTGRESQL_INITIAL_YEAR='2024' + +carbonops-parser run-ingestion \ + --config /etc/carbonops-parser/ingestion.production.json \ + --cycles 1 +``` + +Use `--cycles 1` for production scheduling. The command creates missing Phase 1 +tables additively, ingests the next target year per source family, records +year-state after successful source-family inserts, and reports inserted/skipped +counts. Re-running safely skips duplicate master/detail records and does not +advance year-state for `no_available_source_year` runs. + +.NET scheduled-worker placeholder: + +```bash +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- run-once +``` + +Expected behavior remains fail-closed: `status=blocked`, +`ingestion_status=not_implemented`, `postgresql_connection_opened=False`, and +`records_inserted=0`. This command must not be treated as production ingestion +until later tasks complete .NET service run-once ingestion execution and the +separate promotion of that command. This limitation is outside the final +project-level production-ready scope. + +## PostgreSQL Readiness + +Minimum database privileges for the runtime role: + +- `CONNECT` on the configured database. +- `USAGE` on the target schema. +- `CREATE` on the target schema for additive schema bootstrap. +- `SELECT`, `INSERT`, and `UPDATE` on Phase 1 tables created or owned by the + runtime schema. +- Sequence privileges if the deployment changes table definitions to use + sequences. + +Schema bootstrap is additive and idempotent. It uses `CREATE TABLE IF NOT +EXISTS` and `CREATE INDEX IF NOT EXISTS` for required Phase 1 tables; it does +not drop, truncate, or destructively migrate existing tables. + +Before the first production run, the operator must verify: + +- The target database and schema are correct. +- A backup/restore point exists according to the operator's production policy. +- Monitoring, alerting, retention, and credential rotation are owned outside + this repository. +- The runtime role has the minimum privileges listed above. +- The archive root exists, is writable by the runtime user, and has enough + storage. +- The configured source artifacts exist or reviewed live access is explicitly + enabled. + +After the first run, verify required tables: + +```sql +SELECT table_name +FROM information_schema.tables +WHERE table_schema = current_schema() + AND table_name IN ( + 'source_family_year_states', + 'ghg_emission_factor_masters', + 'ghg_emission_factor_details', + 'defra_emission_factor_masters', + 'defra_emission_factor_details', + 'ipcc_emission_factor_masters', + 'ipcc_emission_factor_details' + ) +ORDER BY table_name; +``` + +Verify GHG Protocol master/detail counts: + +```sql +SELECT 'ghg_emission_factor_masters' AS table_name, count(*) AS records +FROM ghg_emission_factor_masters +UNION ALL +SELECT 'ghg_emission_factor_details', count(*) +FROM ghg_emission_factor_details; +``` + +Verify DEFRA/DESNZ master/detail counts: + +```sql +SELECT 'defra_emission_factor_masters' AS table_name, count(*) AS records +FROM defra_emission_factor_masters +UNION ALL +SELECT 'defra_emission_factor_details', count(*) +FROM defra_emission_factor_details; +``` + +Verify IPCC EFDB master/detail counts: + +```sql +SELECT 'ipcc_emission_factor_masters' AS table_name, count(*) AS records +FROM ipcc_emission_factor_masters +UNION ALL +SELECT 'ipcc_emission_factor_details', count(*) +FROM ipcc_emission_factor_details; +``` + +Verify latest ingested year / year-state: + +```sql +SELECT source_family, max(ingested_year) AS latest_ingested_year +FROM source_family_year_states +GROUP BY source_family +ORDER BY source_family; +``` + +Production database backup/restore, database monitoring, storage monitoring, +credential rotation, and audit-log retention are operator responsibilities +unless a future repository task implements a specific supported integration. + +## Scheduling + +Supported production scheduling is cron or manual scheduled execution of a +single-cycle command. Do not run overlapping production invocations for the same +database/schema unless an external scheduler lock is in place. + +Safe cron shape: + +```cron +SHELL=/bin/sh +CARBONOPS_POSTGRESQL_HOST= +CARBONOPS_POSTGRESQL_PORT=5432 +CARBONOPS_POSTGRESQL_DATABASE= +CARBONOPS_POSTGRESQL_USERNAME= +CARBONOPS_POSTGRESQL_PASSWORD_FILE=/run/secrets/carbonops-postgresql-password +CARBONOPS_POSTGRESQL_APPLICATION_NAME=carbonops-parser-prod +CARBONOPS_POSTGRESQL_SSL_MODE=require +CARBONOPS_POSTGRESQL_INITIAL_YEAR=2024 + +15 4 * * * CARBONOPS_POSTGRESQL_PASSWORD="$(cat "$CARBONOPS_POSTGRESQL_PASSWORD_FILE")" /opt/carbonops-parser/.venv/bin/carbonops-parser run-ingestion --config /etc/carbonops-parser/ingestion.production.json --cycles 1 >> /var/log/carbonops-parser/ingestion.log 2>&1 +``` + +The cron example uses placeholders and an external secret file. Ensure the +secret file and log file permissions are managed by the operator. Logs must not +include the password value. + +## Stop And Rerun + +Stop a running command with the process supervisor, scheduler cancellation, or +terminal interrupt. Normal stop/cancel must not delete archives, schemas, +tables, branches, worktrees, or source artifacts. + +Rerun the same command after fixing an operator-visible issue. Duplicate +master/detail rows are skipped by the source-family repositories. If a run +reports `no_available_source_year`, add the missing configured year artifact or +reviewed live source access before rerunning. + +## Troubleshooting + +Common failures and operator actions: + +| Symptom | Likely cause | Action | +| --- | --- | --- | +| `POSTGRESQL_RUNTIME_CONFIG_MISSING_HOST` or related config issue | Missing DB env | Set the named `CARBONOPS_POSTGRESQL_*` value through the deployment environment | +| Connection failure before startup summary | Bad DB credentials, host, port, SSL mode, network route, or database name | Verify externally with `psql` using redacted operator procedures; rotate secrets if exposure is suspected | +| `archive_root could not be created` or `archive_root must be a directory path` | Missing or invalid archive root | Create the directory and set runtime-user permissions | +| `Unsupported enabled source family` | Unsupported family key | Use only `ghg_protocol`, `defra_desnz`, or `ipcc_efdb` | +| `requires artifact_url` | Missing configured source artifact | Add `source_years...artifact_url` | +| Live HTTPS source access error | HTTPS artifact configured without explicit live opt-in | Stage the artifact locally or explicitly enable reviewed live access | +| `no_available_source_year` | No configured artifact for the target year | Add the target year artifact or confirm the no-op is expected | +| Duplicate rows skipped | Idempotent rerun | Confirm skipped counts match prior successful run | + +When reporting failures, include run ID, source family, target year, status, +issue code, and sanitized message. Do not include passwords, DSNs, tokens, or +private artifact URLs with credentials. + +## Production Validation Checklist + +A Python runtime deployment is ready for the supported operator path only when +every item is pass/fail recorded: + +- PASS/FAIL: Clean install completed with `python -m pip install -e ".[postgresql]"`. +- PASS/FAIL: `carbonops-parser validate-ingestion-config --config --cycles 1` reports `status=ready`. +- PASS/FAIL: DB connectivity passed against an isolated pre-production or approved production target. +- PASS/FAIL: Schema bootstrap created or verified all required Phase 1 tables. +- PASS/FAIL: One source-family smoke passed, or the full three-source smoke passed. +- PASS/FAIL: Idempotent rerun skipped duplicate master/detail rows. +- PASS/FAIL: Redaction check found no passwords, tokens, private DSNs, or secrets in logs and tickets. +- PASS/FAIL: Full Python test baseline passed with `python -m pytest`. +- PASS/FAIL: `git diff --check` passed. +- PASS/FAIL: `git status --short` is clean or contains only the reviewed deployment change. + +Historical release-gate trace retained for reviewer continuity: +Task-ID: OPS-033 +Task-Issue: #500 + +## Related Documents + +- [Python Ingestion Local Runbook](python-ingestion-local-runbook.md) +- [Production Parity Contract](production-parity-contract.md) +- [Real-Source Smoke Mode](real-source-smoke-mode.md) +- [PostgreSQL Runtime Readiness Checklist](postgresql-runtime-readiness-checklist.md) +- [PostgreSQL Opt-In Integration Runbook](postgresql-opt-in-integration-runbook.md) +- [PostgreSQL Phase 1 Schema Contract](postgresql-phase1-schema-contract.md) diff --git a/docs/production-parity-contract.md b/docs/production-parity-contract.md new file mode 100644 index 00000000..42a3443b --- /dev/null +++ b/docs/production-parity-contract.md @@ -0,0 +1,273 @@ +# Production Parity Contract + +This contract defines project-level production readiness for CarbonOps-Parser. +It is documentation only. It does not implement .NET runtime code, change +Python runtime behavior, add credentials, connect to PostgreSQL, download +sources, parse source files, write records, close issues, or broaden the final +verdict beyond the supported Phase 1 scope. + +## Current Verdict + +Project-level production-ready: yes, in the narrow supported scope. + +- Python runtime production path: yes, through the packaged + `carbonops-parser run-ingestion` operator path. +- .NET runtime parity path: yes, through the directly runnable + scheduled-worker service entrypoint, production config loader/redaction + boundary, PostgreSQL schema bootstrap/year-state baseline, source-cycle + orchestration baseline, source-specific master/detail insert baseline, + opt-in Docker PostgreSQL E2E/idempotency validation for all three Phase 1 + source families, and opt-in Python/.NET persisted parity validation. +- Project-level production-ready: yes, for operator-run/scheduled Python + ingestion, PostgreSQL-backed source-specific master/detail persistence for + GHG Protocol, DEFRA/DESNZ, and IPCC EFDB, explicit config/secret handling, + and the .NET parity evidence listed above. + +The .NET `run-once` command remains a fail-closed placeholder and must not be +scheduled as production ingestion unless a future task promotes it. That is not +a blocker for this final verdict because the supported production operation +path is the Python operator runtime and the .NET requirement for this verdict is +runtime parity evidence across the reviewed boundaries. + +## Runtime Evidence Requirement + +A production-ready CarbonOps-Parser release must make the Python operator +runtime and the .NET parity path comparable across the supported boundaries. +The Python runtime is the supported production operation path. The reviewed +.NET path must be: + +- Installable in an operator-owned environment. +- Configurable without committing secrets or raw connection strings. +- Runnable directly for a single ingestion cycle. +- Usable as a service or scheduled worker through documented operator steps. +- Stoppable and rerunnable without destructive cleanup. +- Troubleshootable from redacted structured diagnostics and documented failure + modes. + +The Python runtime currently has this documented operator path. The .NET path +has the scheduled-worker entrypoint shape plus real file/environment config +loading and redaction for `validate-config`, PostgreSQL schema +bootstrap/year-state runtime primitives, safe local source-cycle commands, +source-specific master/detail insert primitives, opt-in Docker PostgreSQL E2E +evidence for three fixture-backed source families, and persisted parity +evidence against the Python path. + +## Equivalent Data Contract + +Python production runs and .NET parity validation must write equivalent parsed +data into the same PostgreSQL schema/model. + +Equivalence requires: + +- The same shared runtime tables and source-family year-state semantics. +- The same source-specific master/detail table families. +- The same PostgreSQL object names, required columns, constraints, indexes, and + idempotency keys once the project-level schema is finalized. +- The same source family identifiers: `ghg_protocol`, `defra_desnz`, and + `ipcc_efdb`. +- The same accepted, skipped, duplicate, validation-failed, conflict, and + no-op count meanings. + +Language-specific code structure may differ, but persisted PostgreSQL behavior +and operator-visible outcomes must not drift. + +## Source Families + +Both runtime paths must support the same Phase 1 source families: + +- GHG Protocol. +- DEFRA/DESNZ. +- IPCC EFDB. + +The Python runtime must discover, download or load, archive, parse, validate, +and persist the selected target year for each enabled source family through the +production contract. The .NET path must validate equivalent source-cycle, +parser handoff, source-specific persistence, idempotency, and persisted output +behavior for those families. A runtime path is not production-parity complete +if it supports only a subset of these source families. + +## Year Selection + +For each enabled source family, both runtimes must select exactly one target +year per cycle. + +Initial-year behavior: + +- If PostgreSQL has no successful year-state data for the source family, select + the configured initial year. +- The default initial year is `2024`. +- The effective initial year must be visible in configuration validation and run + output. + +Next-cycle behavior: + +- If PostgreSQL contains successful imports for the source family, read the + latest successful imported year from the database. +- Select `target_year = latest_successful_imported_year + 1`. +- Update year-state only after the source-family master/detail insert succeeds. + +The runtimes must not silently backfill multiple years, skip ahead, or infer one +source family's target year from another family unless a future reviewed +contract explicitly changes that behavior. + +## No Available Source Year + +Both runtimes must implement the same `no_available_source_year` behavior. + +When the target year has no available source data: + +- The source-family result is a successful no-op, not a hard runtime failure. +- No source records are inserted. +- No latest-year state is advanced. +- The run output includes source family, latest year when present, target year, + no-op reason, and zero attempted/inserted record counts. +- The whole run may complete when every selected source family reports + `no_available_source_year`. + +## Idempotency And Reruns + +Both runtimes must provide equivalent idempotency and rerun behavior. + +- Repeating a completed source-family year with the same source document and + parsed record identities must not create duplicate logical records. +- Duplicate master/detail records must be reported as skipped or already + ingested with deterministic counts. +- Same idempotency key with a different record checksum must be treated as a + conflict unless a future reviewed replacement policy exists. +- Failed transactions must not leave ambiguous partial ingestion state. +- Rerunning after an operator-visible fix must use the same documented command + shape and must not require destructive cleanup. + +## Redaction And Secret Handling + +Python and .NET runtimes must follow the same secret-handling contract. + +- Runtime configuration must keep secrets outside committed files. +- PostgreSQL passwords, tokens, raw DSNs with credentials, private artifact URLs + with credentials, and secret-store values must not be logged. +- Configuration validation and run summaries may report secret presence, but + not secret values. +- Failure diagnostics, tickets, test output, and operator runbooks must use + redacted examples. + +## Operator Expectations + +Both runtimes must document equivalent operator flows: + +- Install. +- Configure. +- Validate configuration without opening PostgreSQL when possible. +- Validate PostgreSQL readiness in an isolated or approved target. +- Run one ingestion cycle directly. +- Schedule as a service or scheduled worker. +- Stop a running cycle without destructive cleanup. +- Rerun safely after fixes. +- Troubleshoot common configuration, PostgreSQL, source availability, + validation, idempotency, and redaction failures. + +The command names and service mechanisms may be runtime-specific, but the +operator-visible capabilities must be equivalent. + +## Production Validation Expectations + +Project-level production validation must include both runtime-specific evidence +and cross-runtime parity evidence: + +- Python production operator validation. +- .NET production operator validation. +- PostgreSQL schema bootstrap validation for both runtimes. +- Initial-year `2024` validation for both runtimes. +- Latest-successful-year to next-year validation for both runtimes. +- `no_available_source_year` validation for both runtimes. +- Idempotent rerun validation for both runtimes. +- Redaction validation for both runtimes. +- Docker PostgreSQL end-to-end tests for the .NET runtime before it is marked + production-ready. +- Python/.NET parity validation proving equivalent persisted rows, counts, + statuses, and operator-visible behavior against the same schema. + +## Persisted Parity Validation Baseline + +PROD-010 adds an opt-in PostgreSQL persisted parity validation path. The default +test suite still does not require PostgreSQL, credentials, Docker, or live +network access. + +When explicitly enabled with an externally supplied PostgreSQL test DSN, the +baseline creates isolated PostgreSQL schemas for the Python and .NET runtime +paths, loads the same checked-in GHG Protocol, DEFRA/DESNZ, and IPCC EFDB +fixtures, persists source-specific master/detail rows, reruns the same +fixture-backed inserts to prove duplicate skips, records successful `2024` +year-state, and compares stable persisted output. The comparison includes + source-family identifiers, latest-year and next-target-year state, + master/detail row counts, source-year/source-version values, master/detail + external keys, factor fields, and statuses. It intentionally excludes + volatile timestamps and generated UUIDs. + +The Python PostgreSQL runtime now persists the same source-family identifiers +required by this contract: `ghg_protocol`, `defra_desnz`, and `ipcc_efdb`, while +preserving the established `ghg_*`, `defra_*`, and `ipcc_*` table families. + +This baseline is parity evidence for the fixture-backed persisted PostgreSQL +surface. It does not make the .NET `run-once` service command production +ingestion ready; PROD-011 uses it as final project-level readiness evidence +inside the narrow supported scope. + +## Follow-Up .NET Task Map + +The implementation sequence for .NET production readiness is: + +1. .NET service/scheduled-worker entrypoint. Satisfied by PROD-003 as an + executable command-surface baseline. +2. .NET production config loader and redaction. Satisfied by PROD-004 for + `validate-config` file/environment loading, deterministic environment + override behavior, fail-closed diagnostics, and redaction only. Later tasks + added PostgreSQL writes and final project-level readiness evidence. +3. .NET PostgreSQL schema bootstrap and year-state. Satisfied by PROD-005 for + additive/idempotent DDL generation, explicit Npgsql runtime boundary, + latest-successful-year lookup, next-year calculation, idempotent successful + year-state recording, and redacted diagnostics only. .NET source + discovery/download/parsing and source-specific master/detail inserts remain + incomplete. +4. .NET source discovery/download/parsing orchestration. Satisfied by PROD-006 + for enabled source-family selection, year-state target-year calculation, + configured local artifact availability checks, normalized parser handoff for + supported local CSV artifacts, and deterministic operator summaries only. + It does not insert source-specific master/detail records, update year-state + after parsing alone, enable uncontrolled network access, or make .NET + production ingestion complete. +5. .NET source-specific master/detail insert. Satisfied by PROD-007 for + parser-output mapping into `ghg_*`, `defra_*`, and `ipcc_*` + master/detail tables, source-family/year transaction scope, duplicate skip + counts, additive `INSERT ... ON CONFLICT DO NOTHING` SQL shape, redacted + database diagnostics, and year-state update only after successful + source-family persistence. This does not make `run-once` production-ready, + does not add uncontrolled network access, and does not complete Docker + PostgreSQL E2E or Python/.NET persisted parity validation. +6. .NET idempotency and rerun behavior E2E. PROD-008 added an opt-in Docker + PostgreSQL contract-test baseline for DEFRA/DESNZ local fixture parsing. + PROD-009 extends that opt-in path to GHG Protocol, DEFRA/DESNZ, and IPCC + EFDB local fixture parsing, first insert, duplicate rerun skips, successful + year-state progression to next target year `2025`, + `no_available_source_year` no-op behavior, failure rollback, and redaction. + This does not make `run-once` production-ready and does not prove + Python/.NET persisted parity. +7. .NET Docker PostgreSQL E2E tests. Satisfied as a fixture-backed, explicit + opt-in Docker PostgreSQL test baseline for all three Phase 1 source families. +8. Python/.NET parity validation. PROD-010 adds an opt-in persisted PostgreSQL + parity baseline for all three Phase 1 source families using local fixtures, + stable source-family identifiers, year-state comparison, idempotent rerun + duplicate-skip comparison, and source-specific master/detail row comparison + without volatile timestamps. It does not make `.NET run-once` production + ingestion ready. +9. Final project production-ready verdict. Satisfied by PROD-011 for the + narrow scope documented in + [Final Project Production-Ready Verdict](final-project-production-ready-verdict.md). + +Each task remained narrow, tested, and reviewable. None of these task outcomes +should be read as source-owner endorsement, certified factor correctness, +managed infrastructure ownership, or uncontrolled live source support. + +## Scope Rule + +Project-level production-ready may be claimed only for the supported scope in +the final verdict. Any broader claim requires a separately scoped review. diff --git a/docs/production-readiness-gap-analysis.md b/docs/production-readiness-gap-analysis.md index c3454c66..6e179674 100644 --- a/docs/production-readiness-gap-analysis.md +++ b/docs/production-readiness-gap-analysis.md @@ -1,14 +1,22 @@ # Production Readiness Gap Analysis -This document records the gap between the current public CarbonOps-Parser artifact and a future implementation that could be reviewed for production use. +This document records the historical gap between the early public +CarbonOps-Parser artifact and the implementation later reviewed for production +use. PROD-002 superseded the earlier project-level production-ready reading. +PROD-011 now issues the final project-level production-ready verdict for the +narrow supported scope. It adds no code, contracts, examples, tests, runtime behavior, source acquisition, persistence, scheduling, configuration loading, unit conversion, factor correctness logic, or deployment workflow. ## Purpose -The current repository is not ready for production use. +The current repository is project-level production-ready only in the narrow +scope documented by +[Final Project Production-Ready Verdict](final-project-production-ready-verdict.md). -This gap analysis gives reviewers and contributors a conservative map of what is missing before any Python or .NET implementation can be considered for production use. It keeps current documentation-first work separate from future implementation tasks. +This gap analysis gives reviewers and contributors a conservative map of the +historical gaps that had to be closed before the final verdict. It keeps +documentation-first work separate from future implementation tasks. The existing artificial examples and skeletons do not prove real-world correctness. Existing smoke tests protect import/export and documentation governance only. Production readiness requires explicit future implementation tasks with narrow scope, tests, and review gates. @@ -48,7 +56,7 @@ Python production hardening should remain deferred until boundary documents and ## .NET Production Readiness Gaps -The .NET path is currently a planned implementation option rather than a parity implementation. +The .NET path now has parity evidence for the reviewed production scope; remaining gaps should be tracked as explicit parity or runtime-promotion review items. The .NET path still needs explicitly scoped future work for: diff --git a/docs/production-readiness-sequencing-roadmap.md b/docs/production-readiness-sequencing-roadmap.md index 99c2a8e7..faa6818d 100644 --- a/docs/production-readiness-sequencing-roadmap.md +++ b/docs/production-readiness-sequencing-roadmap.md @@ -1,12 +1,21 @@ # Production Readiness Sequencing Roadmap -This roadmap orders future work that would be needed before CarbonOps-Parser could be reviewed for production use. +This roadmap records the work sequence used before CarbonOps-Parser was +reviewed for project-level production use. + +PROD-002 supersedes any Python-only interpretation of project-level production +readiness. The Python runtime has a production operator path. PROD-003 through +PROD-010 added the .NET parity evidence, and PROD-011 records the final +project-level production-ready verdict for the narrow supported scope. It is documentation-only. It does not implement production readiness, certify production readiness, or add runtime behavior. ## Purpose -The repository currently has public documentation, contracts, artificial examples, skeletons, and governance smoke tests. Those artifacts help future contributors work in small, reviewable increments, but they do not make the repository ready for production use. +The repository has public documentation, contracts, deterministic fixtures, +governance smoke tests, a Python runtime production operator path, and .NET +parity evidence. PROD-011 is the authoritative final project-level verdict for +the supported scope. This document proposes a safe order for future production readiness work. It does not prove parser, normalization, unit conversion, factor, compliance, legal, or carbon accounting correctness. Real behavior must be added only in future narrow tasks with tests and review gates. @@ -52,7 +61,9 @@ Future tasks may scope: - Public API stability notes. - Test strategy for Python behavior that already has public contracts or artificial examples. -This phase does not make the Python path ready for production use. +This historical phase did not by itself make the Python path ready for +production use. Later Python operator work established the current Python +runtime production path. ### Phase 2: Source Acquisition And Local/Remote Source Boundaries diff --git a/docs/pt-040-runtime-config-gate-parity-review.md b/docs/pt-040-runtime-config-gate-parity-review.md new file mode 100644 index 00000000..b2d8c196 --- /dev/null +++ b/docs/pt-040-runtime-config-gate-parity-review.md @@ -0,0 +1,128 @@ +# PT-040 Parity Review: Runtime Config Gate + +Task-ID: PT-040 +Task-Issue: #357 + +## Scope + +Parity review for the PostgreSQL runtime configuration gate across the Python +and .NET contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/persistence/postgresql_runtime_config_gate.py` +- `tests/test_postgresql_runtime_config_gate.py` +- `src/carbonfactor_parser/persistence/__init__.py` +- `src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGate.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateDecision.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateDescription.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateEvaluator.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/PostgreSQLRuntimeConfigGateContractTests.cs` + +This review did not change runtime behavior, add configuration loading, add +credential loading, connect to PostgreSQL, execute SQL, or modify parser, +database, scheduler, downloader, or source-specific ingestion behavior. + +## Parity Findings + +No blocking parity mismatch was found. + +### Behavior And Contracts + +Python and .NET expose the same gate concepts: + +- caller-provided `PostgreSQLRuntimeConfigGate` metadata +- structured `PostgreSQLRuntimeConfigGateDecision` +- side-effect-free `PostgreSQLRuntimeConfigGateDescription` +- structured `PostgreSQLRuntimeConfigGateIssue` +- pure evaluation/description helpers + +Both implementations keep the gate passive. Evaluation and description do not +load environment variables, read config files, load credentials, open database +connections, create cursors, execute SQL, run migrations, or enable repository +runtime behavior. + +### Naming And Wire Values + +The shared status set is aligned: + +| Concept | Python | .NET | Wire value | +| --- | --- | --- | --- | +| Disabled/default | `DISABLED` | `Disabled` | `disabled` | +| Requested but incomplete | `BLOCKED` | `Blocked` | `blocked` | +| Metadata complete but still not enabled | `NOT_ENABLED` | `NotEnabled` | `not_enabled` | + +.NET includes explicit `ContractWireNames` mappings for these values. Python +uses string enum values matching the same wire names. + +The required future component identifiers are aligned and ordered consistently: + +1. `postgresql_implementation_safety_gate` +2. `postgresql_persistence_options_contract` +3. `explicit_runtime_configuration_opt_in` +4. `approved_secret_source` + +### State Transitions + +The state-transition semantics match: + +- no caller request returns `disabled` +- caller request with incomplete future metadata returns `blocked` +- caller request with all future metadata marked complete returns `not_enabled` + +All three states keep: + +- `config_loading_enabled=False` +- `runtime_enabled=False` +- `loads_environment=False` +- `loads_config_files=False` +- `loads_credentials=False` + +The description surface also keeps `opens_connection=False` and `runs_sql=False` +in both language paths. + +### Error And Issue Semantics + +Issue codes are aligned: + +- `POSTGRESQL_RUNTIME_CONFIG_DISABLED_BY_DEFAULT` +- `POSTGRESQL_RUNTIME_CONFIG_BLOCKED` +- `POSTGRESQL_RUNTIME_CONFIG_NOT_ENABLED` + +Requested states set `field_name`/`FieldName` to `requested`. Default-disabled +states leave the field unset/null. Both implementations use `warning` as the +default issue severity. + +The review found no drift that would require code changes. + +## Validation Performed + +- Reviewed Python runtime config gate implementation and tests. +- Reviewed .NET runtime config gate contract records, evaluator, wire-name + mappings, and tests. +- Compared status values, required component identifiers, passive runtime flags, + transition outcomes, and issue codes/messages. + +## Remaining Risks + +- This is a parity review of current metadata-only gate behavior. It does not + prove future runtime configuration loading behavior because that behavior is + intentionally not implemented. +- Cross-language drift remains possible if a future task updates one gate + surface without synchronized tests and documentation. +- The gate still relies on future safety-gated work before runtime + configuration loading can be enabled. + +## Verdict + +Merge-ready for parity-review scope. + +The Python and .NET runtime config gate surfaces are aligned for behavior, +contracts, naming, wire values, schema shape, state transitions, and issue +semantics. No code changes are required for PT-040. + +Task-ID: PT-040 +Task-Issue: #357 diff --git a/docs/pt-041-source-acquisition-run-repository-contract-parity-review.md b/docs/pt-041-source-acquisition-run-repository-contract-parity-review.md new file mode 100644 index 00000000..aca2d16a --- /dev/null +++ b/docs/pt-041-source-acquisition-run-repository-contract-parity-review.md @@ -0,0 +1,143 @@ +# PT-041 Parity Review: Source Acquisition Run Repository Contract + +Task-ID: PT-041 +Task-Issue: #360 + +## Scope + +Parity review for the source acquisition run repository contract across the +Python and .NET contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/source_acquisition/run_repository_contract.py` +- `tests/test_source_acquisition_run_repository_contract.py` +- `src/carbonfactor_parser/source_acquisition/run_contract.py` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepository.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryPersistResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryPersistStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryRegistry.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryValidationResult.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceAcquisitionRunRepositoryContractTests.cs` + +This review did not add runtime database execution, parser coupling, downloader +coupling, scheduler behavior, production configuration, or destructive +operations. + +## Parity Findings + +No blocking parity mismatch was found. + +### Behavior And Contracts + +Python and .NET expose the same metadata-only repository contract shape: + +- a repository interface/protocol with a human-readable provider name +- a persist operation that accepts source acquisition run results +- a persist result that reports provider name, status, persisted count, and + issues +- a validation result that reports issue collections and validity +- a pure helper/registry function that validates inputs before producing the + persist result + +Both implementations stay runtime-passive. The contract surface does not open +database connections, execute SQL, fetch remote resources, read files, or start +parser execution. + +### Naming And Schema Alignment + +The language-specific casing differs, but the schema concepts align: + +| Concept | Python | .NET | +| --- | --- | --- | +| Provider name | `provider_name` | `ProviderName` | +| Persist status | `status` | `Status` | +| Persisted count | `persisted_count` | `PersistedCount` | +| Issues | `issues` | `Issues` | +| Issue field name | `field_name` | `FieldName` | +| Repository type | `SourceAcquisitionRunRepository` | `ISourceAcquisitionRunRepository` | + +The issue record shape is aligned in both implementations: + +- `code` +- `message` +- `field_name` or `FieldName` +- `severity` with default value `error` + +The persist result shape is also aligned: + +- provider name is echoed back unchanged +- status is deterministic +- persisted count is zero on validation failure +- issues are snapshot/copied into the result rather than exposed as a mutable + caller-owned collection + +### State Transitions + +The persist-result transition semantics match: + +- valid provider name plus valid run instances returns `declared` or + `Declared` +- any validation issue returns `failed_validation` or `FailedValidation` +- validation failure forces persisted count to `0` +- a successful declaration sets persisted count to the number of supplied runs + +Both implementations combine validation issues with any caller-supplied issues +before determining the final status. + +### Error Semantics + +The validation issue codes are aligned: + +- `SOURCE_ACQUISITION_RUN_REPOSITORY_MISSING_PROVIDER_NAME` +- `SOURCE_ACQUISITION_RUN_REPOSITORY_INVALID_RUN` + +The invalid input semantics also align: + +- blank or whitespace provider names are rejected +- invalid run entries are rejected per index +- the field path format matches conceptually as `runs[0]` in Python and + `Runs[0]` in .NET + +The provider-name validation messages are equivalent aside from identifier +casing: + +- Python: `provider_name must be a non-empty string.` +- .NET: `ProviderName must be a non-empty string.` + +The invalid-run validation message is aligned exactly in meaning: + +- `runs must contain SourceAcquisitionRunResult instances.` +- `Runs must contain SourceAcquisitionRunResult instances.` + +## Validation Performed + +- Reviewed the Python repository contract helper and its dedicated tests. +- Reviewed the .NET repository interface, records, registry helper, and its + dedicated tests. +- Compared repository shape, persist-result structure, validation rules, + default issue severity, persisted-count behavior, and runtime-passive + constraints. + +## Remaining Risks + +- The review confirms parity for the repository contract only. It does not + prove parity for downstream runtime repository implementations because those + implementations are intentionally outside this task. +- Cross-language drift remains possible if future changes update one repository + contract surface without synchronized tests or review artifacts. +- .NET currently validates null run entries while Python validates non-contract + objects generically; the resulting contract outcome is still aligned, but the + language runtimes reach that outcome through different type systems. + +## Verdict + +Merge-ready for parity-review scope. + +The Python and .NET source acquisition run repository contract surfaces are +aligned for behavior, contract shape, naming intent, state transitions, and +error semantics. No code change is required for PT-041. + +Task-ID: PT-041 +Task-Issue: #360 diff --git a/docs/pt-042-source-document-repository-contract-parity-review.md b/docs/pt-042-source-document-repository-contract-parity-review.md new file mode 100644 index 00000000..8595064e --- /dev/null +++ b/docs/pt-042-source-document-repository-contract-parity-review.md @@ -0,0 +1,154 @@ +# PT-042 Parity Review: Source Document Repository Contract + +Task-ID: PT-042 +Task-Issue: #363 + +## Scope + +Parity review for the source document repository contract across the Python and +.NET contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/persistence/source_document_repository.py` +- `tests/test_source_document_repository_contract.py` +- `src/carbonfactor_parser/source_acquisition/models.py` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepository.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryPersistResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryPersistStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryRegistry.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryValidationResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentPersistenceRecord.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceDocumentRepositoryContractTests.cs` + +This review did not add runtime database execution, source-specific ingestion, +parser coupling, downloader coupling, scheduler behavior, production +configuration, credentials, or destructive database operations. + +## Parity Findings + +No blocking parity mismatch was found. + +### Behavior And Contracts + +Python and .NET expose the same metadata-only repository contract shape: + +- a repository interface/protocol with a human-readable provider name +- a persist operation that accepts source document persistence records +- a persist result that reports provider name, status, persisted count, and + issues +- a validation result that reports issue collections and validity +- a pure helper/registry function that validates inputs before producing the + persist result + +Both implementations stay runtime-passive. The public contract surface does not +open database connections, execute SQL, fetch remote resources, read files, +perform checksum calculation, or start parser execution. + +### Naming And Schema Alignment + +The language-specific casing differs, but the repository concepts align: + +| Concept | Python | .NET | +| --- | --- | --- | +| Provider name | `provider_name` | `ProviderName` | +| Persist operation | `persist_source_documents` | `PersistSourceDocuments` | +| Persist status | `status` | `Status` | +| Persisted count | `persisted_count` | `PersistedCount` | +| Issues | `issues` | `Issues` | +| Issue field name | `field_name` | `FieldName` | +| Repository type | `SourceDocumentRepository` | `ISourceDocumentRepository` | + +The issue record shape is aligned in both implementations: + +- `code` +- `message` +- `field_name` or `FieldName` +- `severity` with default value `error` + +The persist result shape is also aligned: + +- provider name is echoed back unchanged +- status is deterministic +- persisted count is zero on validation failure +- issues are snapshotted into the result rather than exposed as a mutable + caller-owned collection + +The source document persistence record carried by the repository is intentionally +language-specific in detail because the Python side models the broader local +database mapping shape while the .NET side carries the compact cross-contract +source document reference and checksum shape. For this repository contract, both +sides still align on accepting source document persistence records as metadata +inputs without performing runtime persistence. + +### State Transitions + +The persist-result transition semantics match: + +- valid provider name plus valid source document persistence records returns + `declared` or `Declared` +- any validation issue returns `failed_validation` or `FailedValidation` +- validation failure forces persisted count to `0` +- a successful declaration sets persisted count to the number of supplied + records + +Both implementations combine validation issues with any caller-supplied issues +before determining the final status. + +### Error Semantics + +The validation issue codes are aligned: + +- `SOURCE_DOCUMENT_REPOSITORY_MISSING_PROVIDER_NAME` +- `SOURCE_DOCUMENT_REPOSITORY_INVALID_RECORD` + +The invalid input semantics also align: + +- blank or whitespace provider names are rejected +- invalid record entries are rejected per index +- the field path format matches conceptually as `records[0]` in Python and + `Records[0]` in .NET + +The provider-name validation messages are equivalent aside from identifier +casing: + +- Python: `provider_name must be a non-empty string.` +- .NET: `ProviderName must be a non-empty string.` + +The invalid-record validation messages are aligned in meaning: + +- Python: `records must contain SourceDocumentPersistenceRecord instances.` +- .NET: `Records must contain SourceDocumentPersistenceRecord instances.` + +## Validation Performed + +- Reviewed the Python repository protocol, result helper, validation helper, and + dedicated tests. +- Reviewed the .NET repository interface, records, registry helper, and + dedicated tests. +- Compared repository shape, persist-result structure, validation rules, + default issue severity, persisted-count behavior, record-input expectations, + and runtime-passive constraints. + +## Remaining Risks + +- The review confirms parity for the repository contract only. It does not + prove parity for future runtime repository implementations because those + implementations are intentionally outside this task. +- Cross-language drift remains possible if future changes update one repository + contract surface without synchronized tests or review artifacts. +- .NET currently validates null record entries while Python validates + non-contract objects generically; the resulting contract outcome is aligned, + but the language runtimes reach that outcome through different type systems. + +## Verdict + +Merge-ready for parity-review scope. + +The Python and .NET source document repository contract surfaces are aligned for +behavior, contract shape, naming intent, state transitions, and error semantics. +No source code change is required for PT-042. + +Task-ID: PT-042 +Task-Issue: #363 diff --git a/docs/pt-043-parser-run-repository-contract-parity-review.md b/docs/pt-043-parser-run-repository-contract-parity-review.md new file mode 100644 index 00000000..0e847bf1 --- /dev/null +++ b/docs/pt-043-parser-run-repository-contract-parity-review.md @@ -0,0 +1,149 @@ +# PT-043 Parity Review: Parser Run Repository Contract + +Task-ID: PT-043 +Task-Issue: #366 + +## Scope + +Parity review for the parser run repository contract across the Python and .NET +contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/parsers/run_repository_contract.py` +- `tests/test_parser_run_repository_contract.py` +- `src/carbonfactor_parser/parsers/parser_run_contract.py` +- `src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepository.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryPersistResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryPersistStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryRegistry.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryValidationResult.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/ParserRunRepositoryContractTests.cs` + +This review did not add runtime database execution, source-specific ingestion, +parser execution, downloader coupling, scheduler behavior, production +configuration, credentials, or destructive database operations. + +## Parity Findings + +No blocking parity mismatch was found. + +### Behavior And Contracts + +Python and .NET expose the same metadata-only repository contract shape: + +- a repository interface/protocol with a human-readable provider name +- a persist operation that accepts parser run result metadata +- a persist result that reports provider name, status, persisted count, and + issues +- a validation result that reports issue collections and validity +- a pure helper/registry function that validates inputs before producing the + persist result + +Both implementations stay runtime-passive. The public contract surface does not +open database connections, execute SQL, fetch remote resources, read files, +execute parsers, calculate factors, or perform source-specific ingestion. + +### Naming And Schema Alignment + +The language-specific casing differs, but the repository concepts align: + +| Concept | Python | .NET | +| --- | --- | --- | +| Provider name | `provider_name` | `ProviderName` | +| Persist operation | `persist_runs` | `PersistRuns` | +| Persist status | `status` | `Status` | +| Persisted count | `persisted_count` | `PersistedCount` | +| Issues | `issues` | `Issues` | +| Issue field name | `field_name` | `FieldName` | +| Repository type | `ParserRunRepository` | `IParserRunRepository` | + +The issue record shape is aligned in both implementations: + +- `code` +- `message` +- `field_name` or `FieldName` +- `severity` with default value `error` + +The persist result shape is also aligned: + +- provider name is echoed back unchanged +- status is deterministic +- persisted count is zero on validation failure +- issues are snapshotted into the result rather than exposed as a mutable + caller-owned collection + +The repository input is aligned on parser run result metadata. Python validates +`ParserRunResult` dataclass instances, while .NET accepts `ParserRunResult` +instances and rejects null entries through its nullable enumerable contract. + +### State Transitions + +The persist-result transition semantics match: + +- valid provider name plus valid parser run results returns `declared` or + `Declared` +- any validation issue returns `failed_validation` or `FailedValidation` +- validation failure forces persisted count to `0` +- a successful declaration sets persisted count to the number of supplied runs + +Both implementations combine validation issues with any caller-supplied issues +before determining the final status. + +### Error Semantics + +The validation issue codes are aligned: + +- `PARSER_RUN_REPOSITORY_MISSING_PROVIDER_NAME` +- `PARSER_RUN_REPOSITORY_INVALID_RUN` + +The invalid input semantics also align: + +- blank or whitespace provider names are rejected +- invalid run entries are rejected per index +- the field path format matches conceptually as `runs[0]` in Python and + `Runs[0]` in .NET + +The provider-name validation messages are equivalent aside from identifier +casing: + +- Python: `provider_name must be a non-empty string.` +- .NET: `ProviderName must be a non-empty string.` + +The invalid-run validation messages are aligned in meaning: + +- Python: `runs must contain ParserRunResult instances.` +- .NET: `Runs must contain ParserRunResult instances.` + +## Validation Performed + +- Reviewed the Python repository protocol, result helper, validation helper, and + dedicated tests. +- Reviewed the .NET repository interface, records, registry helper, and + dedicated tests. +- Compared repository shape, persist-result structure, validation rules, + default issue severity, persisted-count behavior, parser-run input + expectations, and runtime-passive constraints. + +## Remaining Risks + +- The review confirms parity for the repository contract only. It does not + prove parity for future runtime repository implementations because those + implementations are intentionally outside this task. +- Cross-language drift remains possible if future changes update one repository + contract surface without synchronized tests or review artifacts. +- .NET currently validates null run entries while Python validates non-contract + objects generically; the resulting contract outcome is aligned, but the + language runtimes reach that outcome through different type systems. + +## Verdict + +Merge-ready for parity-review scope. + +The Python and .NET parser run repository contract surfaces are aligned for +behavior, contract shape, naming intent, state transitions, and error semantics. +No source code change is required for PT-043. + +Task-ID: PT-043 +Task-Issue: #366 diff --git a/docs/pt-044-source-family-repository-contract-parity-review.md b/docs/pt-044-source-family-repository-contract-parity-review.md new file mode 100644 index 00000000..9ec070a6 --- /dev/null +++ b/docs/pt-044-source-family-repository-contract-parity-review.md @@ -0,0 +1,198 @@ +# PT-044 Parity Review: Source-Family Master/Detail Repository Contract + +Task-ID: PT-044 + +Task-Issue: #369 + +## Scope + +Parity review for the source-family master/detail repository contract across +the Python and .NET contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/persistence/source_family_repository.py` +- `tests/test_source_family_repository_contract.py` +- `src/carbonfactor_parser/persistence/postgresql_schema_catalog.py` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamily.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRegistry.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyMasterRecord.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyDetailRecord.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepository.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryPersistResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryPersistStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryRegistry.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryTableNames.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryValidationResult.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceFamilyRepositoryContractTests.cs` + +This review did not add runtime database execution, source-specific ingestion, +parser coupling, downloader coupling, scheduler behavior, production +configuration, credentials, destructive database operations, or source/test +changes. + +## Parity Findings + +No blocking parity mismatch was found. + +### Behavior And Contracts + +Python and .NET expose the same metadata-only repository contract shape: + +- a repository interface/protocol with a human-readable provider name +- a persist operation that accepts source-family master and detail records +- a persist result that reports provider name, status, persisted master count, + persisted detail count, and issues +- a validation result that reports issue collections and validity +- a pure helper/registry function that validates inputs before producing the + persist result +- a helper for source-family-owned master/detail table names + +Both implementations stay runtime-passive. The public contract surface does not +open database connections, execute SQL, fetch remote resources, read files, +calculate factors, run parsers, or perform source-specific ingestion. + +### Naming And Schema Alignment + +The language-specific casing differs, but the repository concepts align: + +| Concept | Python | .NET | +| --- | --- | --- | +| Provider name | `provider_name` | `ProviderName` | +| Persist operation | `persist_source_family_records` | `PersistSourceFamilyRecords` | +| Master records input | `master_records` | `MasterRecords` | +| Detail records input | `detail_records` | `DetailRecords` | +| Persist status | `status` | `Status` | +| Persisted master count | `persisted_master_count` | `PersistedMasterCount` | +| Persisted detail count | `persisted_detail_count` | `PersistedDetailCount` | +| Issues | `issues` | `Issues` | +| Issue field name | `field_name` | `FieldName` | +| Repository type | `SourceFamilyRepository` | `ISourceFamilyRepository` | +| Table names helper | `source_family_repository_table_names` | `GetTableNames` | + +The source-family record shapes align by concept: + +| Concept | Python | .NET | +| --- | --- | --- | +| Source family | `source_family` | `SourceFamily` | +| Master id | `source_family_master_id` | `SourceFamilyMasterId` | +| Source document id | `source_document_id` | `SourceDocumentId` | +| Master external key | `master_external_key` | `MasterExternalKey` | +| Effective range | `effective_from`, `effective_to` | `EffectiveFrom`, `EffectiveTo` | +| Detail id | `source_family_detail_id` | `SourceFamilyDetailId` | +| Detail external key | `detail_external_key` | `DetailExternalKey` | +| Factor value and unit | `factor_value`, `factor_unit` | `FactorValue`, `FactorUnit` | +| Lifecycle status | `lifecycle_status` | `LifecycleStatus` | +| Record checksum | `record_checksum_sha256` | `RecordChecksumSha256` | +| Timestamps | `created_at`, `updated_at` | `CreatedAt`, `UpdatedAt` | + +The source-family vocabulary is aligned to the same Phase 1 families: + +- GHG Protocol: Python `ghg`, .NET `GhgProtocol` +- DEFRA/DESNZ: Python `defra`, .NET `DefraDesnz` +- IPCC EFDB: Python `ipcc`, .NET `IpccEfdb` + +The table-name mapping is aligned: + +| Source family | Master table | Detail table | +| --- | --- | --- | +| GHG Protocol | `ghg_emission_factor_masters` | `ghg_emission_factor_details` | +| DEFRA/DESNZ | `defra_emission_factor_masters` | `defra_emission_factor_details` | +| IPCC EFDB | `ipcc_emission_factor_masters` | `ipcc_emission_factor_details` | + +The issue record shape is aligned in both implementations: + +- `code` +- `message` +- `field_name` or `FieldName` +- `severity` with default value `error` + +The persist result shape is also aligned: + +- provider name is echoed back unchanged +- status is deterministic +- persisted master and detail counts are zero on validation failure +- successful declaration counts are based on the supplied master/detail inputs +- issues are snapshotted into the result rather than exposed as a mutable + caller-owned collection + +### State Transitions + +The persist-result transition semantics match: + +- valid provider name plus valid master/detail records returns `declared` or + `Declared` +- any validation issue returns `failed_validation` or `FailedValidation` +- validation failure forces both persisted counts to `0` +- a successful declaration sets persisted counts to the supplied master and + detail record counts + +Both implementations combine validation issues with any caller-supplied issues +before determining the final status. + +The master/detail relationship rule is also aligned: each detail record must +reference a declared master record in the same source family. + +### Error Semantics + +The validation issue codes are aligned: + +- `SOURCE_FAMILY_REPOSITORY_MISSING_PROVIDER_NAME` +- `SOURCE_FAMILY_REPOSITORY_INVALID_MASTER_RECORD` +- `SOURCE_FAMILY_REPOSITORY_INVALID_DETAIL_RECORD` +- `SOURCE_FAMILY_REPOSITORY_INVALID_SOURCE_FAMILY` +- `SOURCE_FAMILY_REPOSITORY_MISSING_REQUIRED_FIELD` +- `SOURCE_FAMILY_REPOSITORY_DETAIL_MASTER_NOT_DECLARED` + +The invalid input semantics also align: + +- blank or whitespace provider names are rejected +- invalid master and detail entries are rejected per index +- unsupported source-family values are rejected +- required master/detail fields must be non-empty strings +- detail records cannot point at a missing or cross-family master record +- the field path format matches conceptually as `master_records[0]` / + `detail_records[0]` in Python and `MasterRecords[0]` / + `DetailRecords[0]` in .NET + +The validation messages are equivalent aside from identifier casing and +language-specific type-system wording. .NET validates null entries explicitly, +while Python validates non-contract objects generically; the observable contract +outcome is aligned. + +## Validation Performed + +- Reviewed the Python repository protocol, master/detail record contracts, + result helper, validation helper, table-name helper, and dedicated tests. +- Reviewed the .NET repository interface, master/detail records, registry + helper, source-family enum, table-name record, validation result, and + dedicated tests. +- Compared repository shape, master/detail record naming, source-family + vocabulary, table-name mapping, persist-result structure, validation rules, + default issue severity, persisted-count behavior, master/detail reference + checks, state transitions, and runtime-passive constraints. + +## Remaining Risks + +- The review confirms parity for the repository contract only. It does not + prove parity for future runtime repository implementations because those + implementations are intentionally outside this task. +- Cross-language drift remains possible if future changes update one repository + contract surface without synchronized tests or review artifacts. +- Python accepts string source-family values that coerce into the `SourceFamily` + enum, while .NET accepts enum values and rejects undefined enum values. The + supported source-family vocabulary and validation outcome are aligned, but + the language runtimes reach that outcome through different type systems. + +## Verdict + +Merge-ready for parity-review scope. + +The Python and .NET source-family master/detail repository contract surfaces are +aligned for behavior, contract shape, naming intent, schema/table mapping, state +transitions, and error semantics. No source code change is required for PT-044. + +Task-ID: PT-044 + +Task-Issue: #369 diff --git a/docs/pt-045-ghg-source-discovery-runtime-boundary-parity-review.md b/docs/pt-045-ghg-source-discovery-runtime-boundary-parity-review.md new file mode 100644 index 00000000..49ac8b30 --- /dev/null +++ b/docs/pt-045-ghg-source-discovery-runtime-boundary-parity-review.md @@ -0,0 +1,190 @@ +# PT-045 Parity Review: GHG Source Discovery Runtime Boundary + +Task-ID: PT-045 + +Task-Issue: #372 + +## Scope + +Parity review for the GHG source discovery runtime boundary across the Python +and .NET contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/source_acquisition/ghg_source_discovery_boundary.py` +- `tests/test_ghg_source_discovery_boundary.py` +- `src/carbonfactor_parser/source_acquisition/contract_api.py` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryBoundary.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryRequest.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDocumentCandidate.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryValidationResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryMode.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgSourceDiscoveryBoundaryTests.cs` + +This review did not add source-specific ingestion, parser execution, downloader +execution, scheduler behavior, runtime database execution, production +configuration, credentials, destructive database operations, or source +discovery network access. + +## Parity Findings + +One Python-side result-validation gap was found and fixed during this review: + +- Python now rejects declared GHG discovery results that include issue metadata, + matching the .NET `GHG_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES` + validation rule. +- Python now rejects undefined result statuses, matching the .NET + `GHG_SOURCE_DISCOVERY_RESULT_INVALID_STATUS` validation rule. + +No remaining blocking parity mismatch was found. + +### Behavior And Contracts + +Both implementations expose the same runtime-passive boundary shape: + +- deterministic request creation +- deterministic metadata-only result creation +- a GHG-only source document candidate declaration +- request, candidate, and result validation helpers +- structured issue metadata with code, message, field name, and default error + severity +- result-level runtime guard flags for network, download, parse, database + writes, SQL, and scheduler behavior + +The public boundary remains declarative. Neither implementation fetches +references, downloads files, opens databases, executes SQL, schedules work, +starts parser execution, or performs production source discovery. + +### Naming And Schema Alignment + +The language-specific casing differs, but the contract concepts align: + +| Concept | Python | .NET | +| --- | --- | --- | +| Source family | `source_family` | `SourceFamily` | +| Source key | `source_key` | `SourceKey` | +| Discovery reference | `discovery_reference_uri` | `DiscoveryReferenceUri` | +| Mode | `mode` | `Mode` | +| Runtime side-effect gates | `allow_*` / `no_*` | `Allow*` / `No*` | +| Candidate id | `candidate_id` | `CandidateId` | +| Reference URI | `reference_uri` | `ReferenceUri` | +| Artifact kind | `artifact_kind` | `ArtifactKind` | +| Candidate ids projection | `candidate_ids` | `CandidateIds` | +| Issue field name | `field_name` | `FieldName` | + +The canonical wire values align: + +| Concept | Wire value | +| --- | --- | +| Source family/source key | `ghg_protocol` | +| Discovery mode | `runtime_passive` | +| Declared status | `declared` | +| Invalid status | `invalid` | +| Discovery reference URI | `discovery://ghg_protocol/acquisition` | +| Artifact kind | `discovery` | + +The deterministic candidate id aligns as +`ghg_source_discovery_candidate_001_ghg_protocol`. + +Python and .NET use different implementation provenance labels in +`version_label`/`VersionLabel` (`py045_ghg_discovery_boundary` and +`dn045_ghg_discovery_boundary`). The field shape and validation semantics are +aligned, and the label difference is non-blocking because the runtime boundary +uses it only as metadata. + +### State Transitions + +The state transitions now match: + +- valid runtime-passive request plus valid candidate returns `declared` +- invalid request returns `invalid`, zero candidates, and request validation + issues +- invalid candidate metadata returns `invalid`, zero candidates, and candidate + validation issues +- result validation rejects enabled side-effect flags +- result validation rejects declared results with issue metadata +- result validation rejects undefined result status values +- invalid results require issue metadata + +### Error Semantics + +Request validation issue codes align: + +- `GHG_SOURCE_DISCOVERY_MISSING_SOURCE_KEY` +- `GHG_SOURCE_DISCOVERY_MISSING_REFERENCE_URI` +- `GHG_SOURCE_DISCOVERY_SOURCE_FAMILY_MISMATCH` +- `GHG_SOURCE_DISCOVERY_SOURCE_KEY_MISMATCH` +- `GHG_SOURCE_DISCOVERY_UNSUPPORTED_MODE` +- `GHG_SOURCE_DISCOVERY_NETWORK_NOT_ALLOWED` +- `GHG_SOURCE_DISCOVERY_DOWNLOAD_NOT_ALLOWED` +- `GHG_SOURCE_DISCOVERY_PARSE_NOT_ALLOWED` +- `GHG_SOURCE_DISCOVERY_DATABASE_WRITES_NOT_ALLOWED` +- `GHG_SOURCE_DISCOVERY_SCHEDULER_NOT_ALLOWED` + +Candidate validation issue codes align: + +- `GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_SOURCE_KEY` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_CANDIDATE_ID` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_TITLE` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_REFERENCE_URI` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_ARTIFACT_KIND` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_BLANK_CONTENT_TYPE` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_BLANK_EXTENSION` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_BLANK_CHECKSUM_SHA256` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_BLANK_VERSION_LABEL` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_BLANK_DISCOVERED_AT_LABEL` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_INVALID_DOCUMENT_YEAR` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_INVALID_REPORTING_YEAR` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_SOURCE_FAMILY_MISMATCH` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_SOURCE_KEY_MISMATCH` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_ARTIFACT_KIND_MISMATCH` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_UNSUPPORTED_STATUS` +- `GHG_SOURCE_DISCOVERY_CANDIDATE_DOWNLOAD_NOT_ALLOWED` + +Result validation issue codes align for the shared observable contract: + +- `GHG_SOURCE_DISCOVERY_RESULT_INVALID_STATUS` +- `GHG_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED` +- `GHG_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES` +- `GHG_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH` +- `GHG_SOURCE_DISCOVERY_RESULT_MISSING_INVALID_ISSUES` + +.NET additionally validates null request/candidate/result inputs and undefined +`SourceFamily` enum values. Python reaches the same supported contract outcome +through dataclass and string-field validation, so that difference is +language-runtime-specific rather than a blocking parity issue. + +## Validation Performed + +- Reviewed the Python GHG source discovery boundary and dedicated tests. +- Reviewed the .NET GHG source discovery boundary, record types, enum wire + names, and dedicated tests. +- Compared contract shape, deterministic metadata, field naming, wire values, + side-effect gates, state transitions, validation issue codes, and + runtime-passive constraints. + +## Remaining Risks + +- The review confirms parity for the runtime-passive discovery boundary only. + It does not validate future runtime discovery, downloader, parser, scheduler, + or persistence implementations. +- Cross-language drift remains possible if future changes update one GHG + boundary without synchronized tests and parity review. +- Python and .NET preserve different implementation provenance labels in the + candidate version metadata. + +## Verdict + +Commit-ready for parity-review scope. + +The Python and .NET GHG source discovery runtime boundary contract surfaces are +aligned for behavior, contract shape, naming intent, schema alignment, state +transitions, and error semantics after the Python result-validation parity fix. + +Task-ID: PT-045 + +Task-Issue: #372 diff --git a/docs/pt-046-ghg-source-download-execution-parity-review.md b/docs/pt-046-ghg-source-download-execution-parity-review.md new file mode 100644 index 00000000..7fd21312 --- /dev/null +++ b/docs/pt-046-ghg-source-download-execution-parity-review.md @@ -0,0 +1,224 @@ +# PT-046 Parity Review: GHG Source Download Execution + +Task-ID: PT-046 + +Task-Issue: #375 + +## Scope + +Parity review for GHG source download execution across the Python and .NET +contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/source_acquisition/ghg_source_download_execution_boundary.py` +- `tests/test_ghg_source_download_execution_boundary.py` +- `src/carbonfactor_parser/source_acquisition/contract_api.py` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionBoundary.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionRequest.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadedArtifact.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionValidationResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadTransportResponse.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgSourceDownloadExecutionBoundaryTests.cs` + +This review did not add runtime database execution, production credentials, +source-specific ingestion outside the existing GHG boundary, parser coupling, +scheduler behavior, destructive database operations, or new source acquisition +runtime behavior. + +## Post-OPS-032 Status + +The original PT-046 review found blocking parity drift. OPS-032 / PR #512 was +merged to address that drift in the Python and .NET GHG source download +execution boundaries and focused tests. + +After OPS-032, no remaining blocking parity mismatch was found for the current +runtime-passive GHG source download execution contract. + +## Parity Findings + +### Behavior And Contracts + +Python and .NET expose the same high-level execution boundary: + +- an explicit request created from a GHG discovery candidate +- an injected transport callback rather than an owned downloader client +- validation before transport execution +- fail-closed blocking for unsafe or non-opted-in requests +- checksum calculation before file persistence +- local artifact metadata on successful downloads +- result side-effect flags that keep parsing, SQL, database writes, and + scheduler execution out of this boundary + +The public status vocabulary is aligned: + +| Concept | Python | .NET | Wire name | +| --- | --- | --- | --- | +| Blocked before execution | `BLOCKED` | `Blocked` | `blocked` | +| Download persisted | `DOWNLOADED` | `Downloaded` | `downloaded` | +| Execution failed | `FAILED` | `Failed` | `failed` | + +The request, response, artifact, issue, validation-result, and execution-result +records are aligned by concept, with language-appropriate casing differences. + +### Naming And Schema Alignment + +The request fields align by intent: + +| Concept | Python | .NET | +| --- | --- | --- | +| Source family | `source_family` | `SourceFamily` | +| Source key | `source_key` | `SourceKey` | +| Candidate identity | `candidate_id`, `candidate_title` | `CandidateId`, `CandidateTitle` | +| Source URI | `source_reference_uri` | `SourceReferenceUri` | +| Artifact kind | `artifact_kind` | `ArtifactKind` | +| Target path | `target_root`, `target_relative_path` | `TargetRoot`, `TargetRelativePath` | +| Explicit execution flags | `allow_download_execution`, `allow_file_write`, `allow_network`, `allow_overwrite` | `AllowDownloadExecution`, `AllowFileWrite`, `AllowNetwork`, `AllowOverwrite` | +| Forbidden side-effect flags | `allow_parse`, `allow_database_writes`, `allow_scheduler` | `AllowParse`, `AllowDatabaseWrites`, `AllowScheduler` | +| Optional metadata | `content_type`, `extension`, `expected_checksum_sha256`, `document_year`, `reporting_year`, `version_label` | `ContentType`, `Extension`, `ExpectedChecksumSha256`, `DocumentYear`, `ReportingYear`, `VersionLabel` | + +The result and artifact fields align by intent: + +| Concept | Python | .NET | +| --- | --- | --- | +| Status | `status` | `Status` | +| Original request | `request` | `Request` | +| Artifact metadata | `artifact` | `Artifact` | +| Issues | `issues` | `Issues` | +| Downloaded convenience property | `downloaded` | `Downloaded` | +| Side-effect guards | `no_parse`, `no_database_writes`, `no_sql`, `no_scheduler` | `NoParse`, `NoDatabaseWrites`, `NoSql`, `NoScheduler` | +| Artifact id and local path | `artifact_id`, `local_path` | `ArtifactId`, `LocalPath` | +| Checksum and size | `checksum_sha256`, `size_bytes` | `ChecksumSha256`, `SizeBytes` | + +The GHG source identity is conceptually aligned, but represented through +different type systems: + +- Python uses string values: `ghg_protocol`. +- .NET uses `SourceFamily.GhgProtocol` plus a string source key of + `ghg_protocol`. + +### State Transitions + +The primary transition model is aligned: + +- invalid requests return `blocked` without invoking transport +- unsafe target paths return `blocked` +- transport exceptions return `failed` +- missing transport responses return `failed` +- missing transport content returns `failed` +- non-byte transport content is rejected in Python and unrepresentable through + the .NET `byte[]` contract except through null-content validation +- blank transport metadata is rejected consistently +- empty transport content returns `failed` +- checksum mismatch returns `failed` before writing a file +- successful writes return `downloaded` with artifact metadata +- blocked and failed result validation requires diagnostic issue metadata +- successful results expose no issues and preserve side-effect guard flags + +Both implementations block existing targets when overwrite is not explicitly +allowed, reject direct parser/database/scheduler opt-in flags, reject insecure +HTTP, require network opt-in for HTTPS references, and require target paths to +stay under an absolute target root. + +### Error Semantics + +Validation issue codes are aligned for the shared observable contract, +including: + +- `GHG_SOURCE_DOWNLOAD_MISSING_SOURCE_KEY` +- `GHG_SOURCE_DOWNLOAD_MISSING_CANDIDATE_ID` +- `GHG_SOURCE_DOWNLOAD_MISSING_CANDIDATE_TITLE` +- `GHG_SOURCE_DOWNLOAD_MISSING_SOURCE_REFERENCE_URI` +- `GHG_SOURCE_DOWNLOAD_MISSING_ARTIFACT_KIND` +- `GHG_SOURCE_DOWNLOAD_MISSING_TARGET_ROOT` +- `GHG_SOURCE_DOWNLOAD_MISSING_TARGET_RELATIVE_PATH` +- `GHG_SOURCE_DOWNLOAD_SOURCE_FAMILY_MISMATCH` +- `GHG_SOURCE_DOWNLOAD_SOURCE_KEY_MISMATCH` +- `GHG_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE` +- `GHG_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE` +- `GHG_SOURCE_DOWNLOAD_EXECUTION_NOT_ALLOWED` +- `GHG_SOURCE_DOWNLOAD_FILE_WRITE_NOT_ALLOWED` +- `GHG_SOURCE_DOWNLOAD_PARSE_NOT_ALLOWED` +- `GHG_SOURCE_DOWNLOAD_DATABASE_WRITES_NOT_ALLOWED` +- `GHG_SOURCE_DOWNLOAD_SCHEDULER_NOT_ALLOWED` +- `GHG_SOURCE_DOWNLOAD_NETWORK_NOT_ALLOWED` +- `GHG_SOURCE_DOWNLOAD_INSECURE_HTTP_NOT_ALLOWED` +- `GHG_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI` +- `GHG_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME` +- `GHG_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI` +- `GHG_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_ABSOLUTE` +- `GHG_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_ABSOLUTE` +- `GHG_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_URI` +- `GHG_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE` +- `GHG_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE` +- `GHG_SOURCE_DOWNLOAD_TARGET_EXISTS` +- `GHG_SOURCE_DOWNLOAD_TRANSPORT_FAILED` +- `GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING` +- `GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT` +- `GHG_SOURCE_DOWNLOAD_RESPONSE_EMPTY_CONTENT` +- `GHG_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE` +- `GHG_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI` +- `GHG_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH` +- `GHG_SOURCE_DOWNLOAD_WRITE_FAILED` +- `GHG_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED` +- `GHG_SOURCE_DOWNLOAD_RESULT_MISSING_ARTIFACT` +- `GHG_SOURCE_DOWNLOAD_RESULT_UNEXPECTED_ARTIFACT` +- `GHG_SOURCE_DOWNLOAD_RESULT_MISSING_ISSUES` + +OPS-032 specifically addressed the prior blocking drift around discovery +reference diagnostics, blocked/failed result diagnostics, transport response +validation, blank response metadata validation, and malformed URI diagnostics. + +### Target-Path Safety Semantics + +Both implementations reject parent traversal, absolute relative paths, URI-like +target paths, existing final symlinks, and existing target files unless +overwrite is allowed. + +Python continues to use directory-relative write hardening with `O_NOFOLLOW` +where platform support allows it. .NET performs containment and symlink checks +before and after transport and now has focused coverage for parent symlink swap +during transport. The runtime hardening mechanisms are language-specific, but +the observable contract remains fail-closed for the covered escape scenarios. + +## Validation Performed + +- Reviewed Python GHG source download request creation, request validation, + execution, safe target preparation, transport response validation, artifact + validation, result validation, and dedicated tests. +- Reviewed .NET GHG source download request creation, request validation, + execution, target path validation, transport response validation, result + validation, wire-name mapping, record shapes, and dedicated tests. +- Compared behavior, contracts, naming, schema alignment, state transitions, + error semantics, side-effect guard flags, source identity, URI handling, + target path safety, checksum failure behavior, overwrite behavior, and public + test coverage after OPS-032. + +## Remaining Risks + +- This review confirms parity for the current GHG source download execution + boundary only. It does not validate future source-specific ingestion, + parser execution, scheduler behavior, or runtime database writes. +- Python and .NET use different platform mechanisms for target-path hardening; + the covered observable contract is aligned, but absolute implementation + equivalence is not expected across runtimes. +- Cross-language drift remains possible if future GHG download execution + changes update one runtime without synchronized tests and parity review. + +## Verdict + +Merge-ready for parity-review scope. + +The Python and .NET GHG source download execution surfaces are aligned for the +current runtime-passive contract shape, naming intent, status vocabulary, +explicit opt-in flow, state transitions, diagnostic issue semantics, +transport-response validation, malformed URI diagnostics, and side-effect +boundaries after OPS-032. + +Task-ID: PT-046 + +Task-Issue: #375 diff --git a/docs/pt-047-defra-source-discovery-runtime-boundary-parity-review.md b/docs/pt-047-defra-source-discovery-runtime-boundary-parity-review.md new file mode 100644 index 00000000..e1b0410f --- /dev/null +++ b/docs/pt-047-defra-source-discovery-runtime-boundary-parity-review.md @@ -0,0 +1,191 @@ +# PT-047 Parity Review: DEFRA Source Discovery Runtime Boundary + +Task-ID: PT-047 + +Task-Issue: #378 + +## Scope + +Parity review for the DEFRA source discovery runtime boundary across the +Python and .NET contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/source_acquisition/defra_source_discovery_boundary.py` +- `tests/test_defra_source_discovery_boundary.py` +- `src/carbonfactor_parser/source_acquisition/contract_api.py` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryBoundary.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryRequest.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDocumentCandidate.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryValidationResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryMode.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraSourceDiscoveryBoundaryTests.cs` + +This review did not add source-specific ingestion, parser execution, downloader +execution, scheduler behavior, runtime database execution, production +configuration, credentials, destructive database operations, or source +discovery network access. + +## Parity Findings + +One Python-side result-validation gap was found and fixed during this review: + +- Python now rejects declared DEFRA discovery results that include issue + metadata, matching the .NET `DEFRA_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES` + validation rule. +- Python now rejects undefined result statuses, matching the .NET + `DEFRA_SOURCE_DISCOVERY_RESULT_INVALID_STATUS` validation rule. + +No remaining blocking parity mismatch was found. + +### Behavior And Contracts + +Both implementations expose the same runtime-passive boundary shape: + +- deterministic request creation +- deterministic metadata-only result creation +- a DEFRA-only source document candidate declaration +- request, candidate, and result validation helpers +- structured issue metadata with code, message, field name, and default error + severity +- result-level runtime guard flags for network, download, parse, database + writes, SQL, and scheduler behavior + +The public boundary remains declarative. Neither implementation fetches +references, downloads files, opens databases, executes SQL, schedules work, +starts parser execution, or performs production source discovery. + +### Naming And Schema Alignment + +The language-specific casing differs, but the contract concepts align: + +| Concept | Python | .NET | +| --- | --- | --- | +| Source family | `source_family` | `SourceFamily` | +| Source key | `source_key` | `SourceKey` | +| Discovery reference | `discovery_reference_uri` | `DiscoveryReferenceUri` | +| Mode | `mode` | `Mode` | +| Runtime side-effect gates | `allow_*` / `no_*` | `Allow*` / `No*` | +| Candidate id | `candidate_id` | `CandidateId` | +| Reference URI | `reference_uri` | `ReferenceUri` | +| Artifact kind | `artifact_kind` | `ArtifactKind` | +| Candidate ids projection | `candidate_ids` | `CandidateIds` | +| Issue field name | `field_name` | `FieldName` | + +The canonical wire values align: + +| Concept | Wire value | +| --- | --- | +| Source family/source key | `defra_desnz` | +| Discovery mode | `runtime_passive` | +| Declared status | `declared` | +| Invalid status | `invalid` | +| Discovery reference URI | `discovery://defra_desnz/homepage` | +| Artifact kind | `discovery` | + +The deterministic candidate id aligns as +`defra_source_discovery_candidate_001_defra_desnz`. + +Python and .NET use different implementation provenance labels in +`version_label`/`VersionLabel` (`py047_defra_discovery_boundary` and +`dn047_defra_discovery_boundary`). The field shape and validation semantics are +aligned, and the label difference is non-blocking because the runtime boundary +uses it only as metadata. + +### State Transitions + +The state transitions now match: + +- valid runtime-passive request plus valid candidate returns `declared` +- invalid request returns `invalid`, zero candidates, and request validation + issues +- invalid candidate metadata returns `invalid`, zero candidates, and candidate + validation issues +- result validation rejects enabled side-effect flags +- result validation rejects declared results with issue metadata +- result validation rejects undefined result status values +- invalid results require issue metadata + +### Error Semantics + +Request validation issue codes align: + +- `DEFRA_SOURCE_DISCOVERY_MISSING_SOURCE_KEY` +- `DEFRA_SOURCE_DISCOVERY_MISSING_REFERENCE_URI` +- `DEFRA_SOURCE_DISCOVERY_SOURCE_FAMILY_MISMATCH` +- `DEFRA_SOURCE_DISCOVERY_SOURCE_KEY_MISMATCH` +- `DEFRA_SOURCE_DISCOVERY_UNSUPPORTED_MODE` +- `DEFRA_SOURCE_DISCOVERY_NETWORK_NOT_ALLOWED` +- `DEFRA_SOURCE_DISCOVERY_DOWNLOAD_NOT_ALLOWED` +- `DEFRA_SOURCE_DISCOVERY_PARSE_NOT_ALLOWED` +- `DEFRA_SOURCE_DISCOVERY_DATABASE_WRITES_NOT_ALLOWED` +- `DEFRA_SOURCE_DISCOVERY_SCHEDULER_NOT_ALLOWED` + +Candidate validation issue codes align: + +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_SOURCE_KEY` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_CANDIDATE_ID` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_TITLE` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_REFERENCE_URI` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_ARTIFACT_KIND` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_BLANK_CONTENT_TYPE` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_BLANK_EXTENSION` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_BLANK_CHECKSUM_SHA256` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_BLANK_VERSION_LABEL` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_BLANK_DISCOVERED_AT_LABEL` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_INVALID_DOCUMENT_YEAR` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_INVALID_REPORTING_YEAR` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_SOURCE_FAMILY_MISMATCH` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_SOURCE_KEY_MISMATCH` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_ARTIFACT_KIND_MISMATCH` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_UNSUPPORTED_STATUS` +- `DEFRA_SOURCE_DISCOVERY_CANDIDATE_DOWNLOAD_NOT_ALLOWED` + +Result validation issue codes align for the shared observable contract: + +- `DEFRA_SOURCE_DISCOVERY_RESULT_INVALID_STATUS` +- `DEFRA_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED` +- `DEFRA_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES` +- `DEFRA_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH` +- `DEFRA_SOURCE_DISCOVERY_RESULT_MISSING_INVALID_ISSUES` + +.NET additionally validates null request/candidate/result inputs and undefined +`SourceFamily` enum values. Python reaches the same supported contract outcome +through dataclass and string-field validation, so that difference is +language-runtime-specific rather than a blocking parity issue. + +## Validation Performed + +- Reviewed the Python DEFRA source discovery boundary and dedicated tests. +- Reviewed the .NET DEFRA source discovery boundary, record types, enum wire + names, and dedicated tests. +- Compared contract shape, deterministic metadata, field naming, wire values, + side-effect gates, state transitions, validation issue codes, and + runtime-passive constraints. + +## Remaining Risks + +- The review confirms parity for the runtime-passive discovery boundary only. + It does not validate future runtime discovery, downloader, parser, scheduler, + or persistence implementations. +- Cross-language drift remains possible if future changes update one DEFRA + boundary without synchronized tests and parity review. +- Python and .NET preserve different implementation provenance labels in the + candidate version metadata. + +## Verdict + +Commit-ready for parity-review scope. + +The Python and .NET DEFRA source discovery runtime boundary contract surfaces +are aligned for behavior, contract shape, naming intent, schema alignment, +state transitions, and error semantics after the Python result-validation +parity fix. + +Task-ID: PT-047 + +Task-Issue: #378 diff --git a/docs/pt-048-defra-source-download-execution-parity-review.md b/docs/pt-048-defra-source-download-execution-parity-review.md new file mode 100644 index 00000000..4b94139a --- /dev/null +++ b/docs/pt-048-defra-source-download-execution-parity-review.md @@ -0,0 +1,250 @@ +# PT-048 Parity Review: DEFRA Source Download Execution + +Task-ID: PT-048 + +Task-Issue: #381 + +## Scope + +Parity review for DEFRA source download execution across the Python and .NET +contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/source_acquisition/defra_source_download_execution_boundary.py` +- `tests/test_defra_source_download_execution_boundary.py` +- `src/carbonfactor_parser/source_acquisition/contract_api.py` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionBoundary.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionRequest.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadedArtifact.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionValidationResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadTransportResponse.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraSourceDownloadExecutionBoundaryTests.cs` + +This review did not add source-specific ingestion beyond the existing DEFRA +download boundary, parser execution, scheduler behavior, runtime database +execution, production configuration, credentials, destructive database +operations, or source discovery network access. + +Allowed files for this implementation were constrained to: + +- `src/carbonfactor_parser/source_acquisition/defra_source_download_execution_boundary.py` +- `tests/test_defra_source_download_execution_boundary.py` +- `docs/pt-048-defra-source-download-execution-parity-review.md` + +## Parity Findings + +Python-side validation gaps were found and fixed during this review: + +- Python now classifies malformed source reference URIs with + `DEFRA_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI`, matching .NET for + malformed `://` inputs and hostless HTTP(S) references. +- Python now distinguishes a missing transport response from missing transport + content with `DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING` and + `DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT`. +- Python now rejects blank transport response `content_type` and `final_uri` + metadata with the .NET-aligned response metadata issue codes. +- Python now rejects undefined result statuses with + `DEFRA_SOURCE_DOWNLOAD_RESULT_INVALID_STATUS`, matching the .NET result + validation rule. + +No remaining blocking parity mismatch was found. + +### Behavior And Contracts + +Both implementations expose the same explicit execution boundary shape: + +- a request created from DEFRA discovery candidate metadata +- explicit opt-in flags for download execution and file writes +- an injected transport callback instead of an owned downloader client +- validation before transport execution +- fail-closed blocking for unsafe or non-opted-in requests +- checksum calculation before file persistence +- local artifact metadata on successful downloads +- result side-effect flags that keep parsing, SQL, database writes, and + scheduler execution out of this boundary + +The public status vocabulary is aligned: + +| Concept | Python | .NET | Wire name | +| --- | --- | --- | --- | +| Blocked before execution | `BLOCKED` | `Blocked` | `blocked` | +| Download persisted | `DOWNLOADED` | `Downloaded` | `downloaded` | +| Execution failed | `FAILED` | `Failed` | `failed` | + +The request, transport response, artifact, issue, validation-result, and +execution-result records are aligned by concept, with language-appropriate +casing differences. + +### Naming And Schema Alignment + +The request fields align by intent: + +| Concept | Python | .NET | +| --- | --- | --- | +| Source family | `source_family` | `SourceFamily` | +| Source key | `source_key` | `SourceKey` | +| Candidate identity | `candidate_id`, `candidate_title` | `CandidateId`, `CandidateTitle` | +| Source URI | `source_reference_uri` | `SourceReferenceUri` | +| Artifact kind | `artifact_kind` | `ArtifactKind` | +| Target path | `target_root`, `target_relative_path` | `TargetRoot`, `TargetRelativePath` | +| Explicit execution flags | `allow_download_execution`, `allow_file_write`, `allow_network`, `allow_overwrite` | `AllowDownloadExecution`, `AllowFileWrite`, `AllowNetwork`, `AllowOverwrite` | +| Forbidden side-effect flags | `allow_parse`, `allow_database_writes`, `allow_scheduler` | `AllowParse`, `AllowDatabaseWrites`, `AllowScheduler` | +| Optional metadata | `content_type`, `extension`, `expected_checksum_sha256`, `document_year`, `reporting_year`, `version_label` | `ContentType`, `Extension`, `ExpectedChecksumSha256`, `DocumentYear`, `ReportingYear`, `VersionLabel` | + +The result and artifact fields align by intent: + +| Concept | Python | .NET | +| --- | --- | --- | +| Status | `status` | `Status` | +| Original request | `request` | `Request` | +| Artifact metadata | `artifact` | `Artifact` | +| Issues | `issues` | `Issues` | +| Downloaded convenience property | `downloaded` | `Downloaded` | +| Side-effect guards | `no_parse`, `no_database_writes`, `no_sql`, `no_scheduler` | `NoParse`, `NoDatabaseWrites`, `NoSql`, `NoScheduler` | +| Artifact id and local path | `artifact_id`, `local_path` | `ArtifactId`, `LocalPath` | +| Checksum and size | `checksum_sha256`, `size_bytes` | `ChecksumSha256`, `SizeBytes` | + +The DEFRA source identity is conceptually aligned: + +- Python uses string values: `defra_desnz`. +- .NET uses `SourceFamily.DefraDesnz` plus a string source key of + `defra_desnz`. + +Python and .NET use different implementation provenance labels in test fixture +metadata (`py048_mock_download` and `dn048_mock_download`). The field shape and +validation semantics are aligned, and the label difference is non-blocking +because the boundary treats it only as metadata. + +### State Transitions + +The primary transition model is aligned: + +- invalid requests return `blocked` without invoking transport +- default DEFRA discovery candidates are not downloadable and remain blocked +- unsafe target paths return `blocked` +- transport exceptions return `failed` +- missing transport responses return `failed` +- missing transport content returns `failed` +- non-byte transport content is rejected in Python and unrepresentable through + the .NET `byte[]` contract except through null-content validation +- blank transport response metadata is rejected consistently +- empty transport content returns `failed` +- checksum mismatch returns `failed` before writing a file +- successful writes return `downloaded` with artifact metadata +- blocked and failed result validation requires diagnostic issue metadata +- successful results expose no issues and preserve side-effect guard flags + +Both implementations block existing targets when overwrite is not explicitly +allowed, reject direct parser/database/scheduler opt-in flags, reject insecure +HTTP, require network opt-in for HTTPS references, and require target paths to +stay under an absolute target root. + +### Error Semantics + +Validation issue codes are aligned for the shared observable contract, +including: + +- `DEFRA_SOURCE_DOWNLOAD_MISSING_SOURCE_KEY` +- `DEFRA_SOURCE_DOWNLOAD_MISSING_CANDIDATE_ID` +- `DEFRA_SOURCE_DOWNLOAD_MISSING_CANDIDATE_TITLE` +- `DEFRA_SOURCE_DOWNLOAD_MISSING_SOURCE_REFERENCE_URI` +- `DEFRA_SOURCE_DOWNLOAD_MISSING_ARTIFACT_KIND` +- `DEFRA_SOURCE_DOWNLOAD_MISSING_TARGET_ROOT` +- `DEFRA_SOURCE_DOWNLOAD_MISSING_TARGET_RELATIVE_PATH` +- `DEFRA_SOURCE_DOWNLOAD_SOURCE_FAMILY_MISMATCH` +- `DEFRA_SOURCE_DOWNLOAD_SOURCE_KEY_MISMATCH` +- `DEFRA_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE` +- `DEFRA_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE` +- `DEFRA_SOURCE_DOWNLOAD_EXECUTION_NOT_ALLOWED` +- `DEFRA_SOURCE_DOWNLOAD_FILE_WRITE_NOT_ALLOWED` +- `DEFRA_SOURCE_DOWNLOAD_PARSE_NOT_ALLOWED` +- `DEFRA_SOURCE_DOWNLOAD_DATABASE_WRITES_NOT_ALLOWED` +- `DEFRA_SOURCE_DOWNLOAD_SCHEDULER_NOT_ALLOWED` +- `DEFRA_SOURCE_DOWNLOAD_NETWORK_NOT_ALLOWED` +- `DEFRA_SOURCE_DOWNLOAD_INSECURE_HTTP_NOT_ALLOWED` +- `DEFRA_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI` +- `DEFRA_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME` +- `DEFRA_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI` +- `DEFRA_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_ABSOLUTE` +- `DEFRA_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_ABSOLUTE` +- `DEFRA_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_URI` +- `DEFRA_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE` +- `DEFRA_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE` +- `DEFRA_SOURCE_DOWNLOAD_TARGET_EXISTS` +- `DEFRA_SOURCE_DOWNLOAD_TRANSPORT_FAILED` +- `DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING` +- `DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT` +- `DEFRA_SOURCE_DOWNLOAD_RESPONSE_EMPTY_CONTENT` +- `DEFRA_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE` +- `DEFRA_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI` +- `DEFRA_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH` +- `DEFRA_SOURCE_DOWNLOAD_WRITE_FAILED` +- `DEFRA_SOURCE_DOWNLOAD_RESULT_INVALID_STATUS` +- `DEFRA_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED` +- `DEFRA_SOURCE_DOWNLOAD_RESULT_MISSING_ARTIFACT` +- `DEFRA_SOURCE_DOWNLOAD_RESULT_UNEXPECTED_ARTIFACT` +- `DEFRA_SOURCE_DOWNLOAD_RESULT_MISSING_ISSUES` + +.NET additionally validates null request/result inputs and undefined enum +values. Python reaches the same supported contract outcome through dataclass +construction and string-field validation, so that difference is +language-runtime-specific rather than a blocking parity issue. + +### Target-Path Safety Semantics + +Both implementations reject parent traversal, absolute relative paths, URI-like +target paths, existing final symlinks, and existing target files unless +overwrite is allowed. + +Python uses directory-relative write hardening with `O_NOFOLLOW` and fails +closed when required platform flags are unavailable. .NET performs containment +and symlink checks around the write path and has focused coverage for parent +symlink swap during transport. The runtime hardening mechanisms are +language-specific, but the observable contract remains fail-closed for the +covered escape scenarios. + +## Validation Performed + +- Reviewed Python DEFRA source download request creation, request validation, + execution, safe target preparation, transport response validation, artifact + validation, result validation, public API export coverage, and dedicated + tests. +- Reviewed .NET DEFRA source download request creation, request validation, + execution, target path validation, transport response validation, result + validation, wire-name mapping, record shapes, and dedicated tests. +- Compared behavior, contracts, naming, schema alignment, state transitions, + error semantics, side-effect guard flags, source identity, URI handling, + target path safety, checksum failure behavior, overwrite behavior, and public + test coverage. + +## Remaining Risks + +- This review confirms parity for the current DEFRA source download execution + boundary only. It does not validate future source-specific ingestion, + parser execution, scheduler behavior, or runtime database writes. +- Python and .NET use different platform mechanisms for target-path hardening; + the covered observable contract is aligned, but absolute implementation + equivalence is not expected across runtimes. +- Cross-language drift remains possible if future DEFRA download execution + changes update one runtime without synchronized tests and parity review. +- Local `pytest` execution was blocked because the active Python environment + does not have `pytest` installed. + +## Verdict + +Commit-ready for parity-review scope. + +The Python and .NET DEFRA source download execution surfaces are aligned for +the current explicit execution contract shape, naming intent, status +vocabulary, explicit opt-in flow, state transitions, diagnostic issue +semantics, transport-response validation, malformed URI diagnostics, and +side-effect boundaries after the Python parity fixes in this task. + +Task-ID: PT-048 + +Task-Issue: #381 diff --git a/docs/pt-049-ipcc-source-discovery-runtime-boundary-parity-review.md b/docs/pt-049-ipcc-source-discovery-runtime-boundary-parity-review.md new file mode 100644 index 00000000..0a4eb83f --- /dev/null +++ b/docs/pt-049-ipcc-source-discovery-runtime-boundary-parity-review.md @@ -0,0 +1,198 @@ +# PT-049 Parity Review: IPCC Source Discovery Runtime Boundary + +Task-ID: PT-049 + +Task-Issue: #384 + +## Scope + +Parity review for the IPCC source discovery runtime boundary across the +Python and .NET contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/source_acquisition/ipcc_source_discovery_boundary.py` +- `tests/test_ipcc_source_discovery_boundary.py` +- `src/carbonfactor_parser/source_acquisition/contract_api.py` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryBoundary.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryRequest.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDocumentCandidate.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryValidationResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryMode.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccSourceDiscoveryBoundaryTests.cs` + +This review did not add source-specific ingestion, parser execution, +downloader execution, scheduler behavior, runtime database execution, +production configuration, credentials, destructive database operations, or +source discovery network access. + +Allowed files for this implementation were constrained to: + +- `src/carbonfactor_parser/source_acquisition/ipcc_source_discovery_boundary.py` +- `tests/test_ipcc_source_discovery_boundary.py` +- `docs/pt-049-ipcc-source-discovery-runtime-boundary-parity-review.md` + +## Parity Findings + +Python-side result-validation gaps were found and fixed during this review: + +- Python now rejects declared IPCC discovery results that include issue + metadata, matching the .NET + `IPCC_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES` validation rule. +- Python now rejects undefined result statuses with + `IPCC_SOURCE_DISCOVERY_RESULT_INVALID_STATUS`, matching the .NET result + validation rule. + +No remaining blocking parity mismatch was found. + +### Behavior And Contracts + +Both implementations expose the same runtime-passive boundary shape: + +- deterministic request creation +- deterministic metadata-only result creation +- an IPCC-only source document candidate declaration +- request, candidate, and result validation helpers +- structured issue metadata with code, message, field name, and default error + severity +- result-level runtime guard flags for network, download, parse, database + writes, SQL, and scheduler behavior + +The public boundary remains declarative. Neither implementation fetches +references, downloads files, opens databases, executes SQL, schedules work, +starts parser execution, or performs production source discovery. + +### Naming And Schema Alignment + +The language-specific casing differs, but the contract concepts align: + +| Concept | Python | .NET | +| --- | --- | --- | +| Source family | `source_family` | `SourceFamily` | +| Source key | `source_key` | `SourceKey` | +| Discovery reference | `discovery_reference_uri` | `DiscoveryReferenceUri` | +| Mode | `mode` | `Mode` | +| Runtime side-effect gates | `allow_*` / `no_*` | `Allow*` / `No*` | +| Candidate id | `candidate_id` | `CandidateId` | +| Reference URI | `reference_uri` | `ReferenceUri` | +| Artifact kind | `artifact_kind` | `ArtifactKind` | +| Candidate ids projection | `candidate_ids` | `CandidateIds` | +| Issue field name | `field_name` | `FieldName` | + +The canonical wire values align: + +| Concept | Wire value | +| --- | --- | +| Source family/source key | `ipcc_efdb` | +| Discovery mode | `runtime_passive` | +| Declared status | `declared` | +| Invalid status | `invalid` | +| Discovery reference URI | `discovery://ipcc_efdb/homepage` | +| Artifact kind | `discovery` | + +The deterministic candidate id aligns as +`ipcc_source_discovery_candidate_001_ipcc_efdb`. + +Python and .NET use different implementation provenance labels in +`version_label`/`VersionLabel` (`py049_ipcc_discovery_boundary` and +`dn049_ipcc_discovery_boundary`). The field shape and validation semantics are +aligned, and the label difference is non-blocking because the runtime boundary +uses it only as metadata. + +### State Transitions + +The state transitions now match: + +- valid runtime-passive request plus valid candidate returns `declared` +- invalid request returns `invalid`, zero candidates, and request validation + issues +- invalid candidate metadata returns `invalid`, zero candidates, and candidate + validation issues +- result validation rejects enabled side-effect flags +- result validation rejects declared results with issue metadata +- result validation rejects undefined result status values +- invalid results require issue metadata + +### Error Semantics + +Request validation issue codes align: + +- `IPCC_SOURCE_DISCOVERY_MISSING_SOURCE_KEY` +- `IPCC_SOURCE_DISCOVERY_MISSING_REFERENCE_URI` +- `IPCC_SOURCE_DISCOVERY_SOURCE_FAMILY_MISMATCH` +- `IPCC_SOURCE_DISCOVERY_SOURCE_KEY_MISMATCH` +- `IPCC_SOURCE_DISCOVERY_UNSUPPORTED_MODE` +- `IPCC_SOURCE_DISCOVERY_NETWORK_NOT_ALLOWED` +- `IPCC_SOURCE_DISCOVERY_DOWNLOAD_NOT_ALLOWED` +- `IPCC_SOURCE_DISCOVERY_PARSE_NOT_ALLOWED` +- `IPCC_SOURCE_DISCOVERY_DATABASE_WRITES_NOT_ALLOWED` +- `IPCC_SOURCE_DISCOVERY_SCHEDULER_NOT_ALLOWED` + +Candidate validation issue codes align: + +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_SOURCE_KEY` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_CANDIDATE_ID` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_TITLE` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_REFERENCE_URI` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_ARTIFACT_KIND` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_BLANK_CONTENT_TYPE` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_BLANK_EXTENSION` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_BLANK_CHECKSUM_SHA256` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_BLANK_VERSION_LABEL` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_BLANK_DISCOVERED_AT_LABEL` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_INVALID_DOCUMENT_YEAR` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_INVALID_REPORTING_YEAR` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_SOURCE_FAMILY_MISMATCH` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_SOURCE_KEY_MISMATCH` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_ARTIFACT_KIND_MISMATCH` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_UNSUPPORTED_STATUS` +- `IPCC_SOURCE_DISCOVERY_CANDIDATE_DOWNLOAD_NOT_ALLOWED` + +Result validation issue codes align for the shared observable contract: + +- `IPCC_SOURCE_DISCOVERY_RESULT_INVALID_STATUS` +- `IPCC_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED` +- `IPCC_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES` +- `IPCC_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH` +- `IPCC_SOURCE_DISCOVERY_RESULT_MISSING_INVALID_ISSUES` + +.NET additionally validates null request/candidate/result inputs and undefined +`SourceFamily` enum values. Python reaches the same supported contract outcome +through dataclass and string-field validation, so that difference is +language-runtime-specific rather than a blocking parity issue. + +## Validation Performed + +- Reviewed the Python IPCC source discovery boundary and dedicated tests. +- Reviewed the .NET IPCC source discovery boundary, record types, enum wire + names, and dedicated tests. +- Compared contract shape, deterministic metadata, field naming, wire values, + side-effect gates, state transitions, validation issue codes, and + runtime-passive constraints. + +## Remaining Risks + +- The review confirms parity for the runtime-passive discovery boundary only. + It does not validate future runtime discovery, downloader, parser, + scheduler, or persistence implementations. +- Cross-language drift remains possible if future changes update one IPCC + boundary without synchronized tests and parity review. +- Python and .NET preserve different implementation provenance labels in the + candidate version metadata. + +## Verdict + +Commit-ready for parity-review scope. + +The Python and .NET IPCC source discovery runtime boundary contract surfaces +are aligned for behavior, contract shape, naming intent, schema alignment, +state transitions, and error semantics after the Python result-validation +parity fix. + +Task-ID: PT-049 + +Task-Issue: #384 diff --git a/docs/pt-053-ipcc-source-download-execution-parity-review.md b/docs/pt-053-ipcc-source-download-execution-parity-review.md new file mode 100644 index 00000000..61ba12ae --- /dev/null +++ b/docs/pt-053-ipcc-source-download-execution-parity-review.md @@ -0,0 +1,123 @@ +# PT-053 Parity Review: IPCC Source Download Execution + +Task-ID: PT-053 + +Task-Issue: #478 + +## Scope + +Parity review for IPCC EFDB source download execution across the Python and +.NET contract surfaces. + +Reviewed files: + +- `src/carbonfactor_parser/source_acquisition/ipcc_source_download_execution_boundary.py` +- `tests/test_ipcc_source_download_execution_boundary.py` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionBoundary.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionRequest.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionStatus.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadedArtifact.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionIssue.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionValidationResult.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadTransportResponse.cs` +- `src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs` +- `tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccSourceDownloadExecutionBoundaryTests.cs` +- `tests/fixtures/parity/ipcc_source_download_execution_expectations.json` + +This review did not add live network calls, production database credentials, +parser execution, scheduler behavior, runtime database execution, destructive +database operations, branch cleanup, or worktree cleanup. + +## Fixture Coverage + +Python and .NET now read the same deterministic fixture: + +- `tests/fixtures/parity/ipcc_source_download_execution_expectations.json` + +The shared fixture covers: + +- successful download metadata and checksum capture +- existing known document idempotency +- checksum mismatch failure before file persistence +- default discovery candidate blocked before transport +- blank transport response metadata failure + +The fixture is local-only and uses injected mock transport payloads. + +## Parity Findings + +Python and .NET are aligned for the shared IPCC EFDB download execution +contract shape: + +- explicit request creation from IPCC discovery candidate metadata +- explicit opt-in for download execution and file writes +- injected transport callback instead of owned downloader behavior +- validation before transport execution +- fail-closed blocking for unsafe or non-opted-in requests +- checksum calculation before file persistence +- artifact metadata capture on successful or already-known local documents +- side-effect guard flags for parse, SQL, database writes, and scheduler work + +The status vocabulary is aligned for blocked, downloaded, and failed outcomes. +.NET additionally exposes `already_known` for idempotent existing documents. +Python preserves its existing public API by returning `downloaded` with +`reused_existing=true` on the artifact. The shared fixture documents and tests +that as an intentional current difference. + +## Metadata And Naming + +Common metadata expectations are aligned: + +- source family/key: `ipcc_efdb` +- candidate id/title +- source reference URI +- artifact kind +- content type and extension +- document year, reporting year, and version label +- checksum SHA-256 and size in bytes +- local path/original filename semantics +- issue code diagnostics + +Language-specific representation differences remain non-blocking: + +- Python uses string source-family values; .NET uses `SourceFamily.IpccEfdb`. +- Python records `retrieved_at_label`; .NET records `RetrievedAtUtc`. +- Python records `storage_identity` and `reused_existing`; .NET records an + `AlreadyKnown` result property/status. + +## Error Semantics + +The shared fixture confirms aligned issue codes for: + +- `IPCC_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE` +- `IPCC_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE` +- `IPCC_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH` +- `IPCC_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE` +- `IPCC_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI` + +Existing dedicated tests in both runtimes also cover unsafe URI schemes, +target path validation, symlink safety, transport failure, missing/empty +transport responses, side-effect flag validation, and result validation. + +## Remaining Risks + +- This review confirms the current IPCC EFDB source download execution boundary + only. It does not validate parser execution, scheduler behavior, production + database writes, or future source-specific ingestion. +- Python and .NET use different target-path hardening mechanisms. The covered + observable contract is aligned, but implementation-level equivalence is not + expected across runtimes. +- Existing-document idempotency remains intentionally represented differently: + Python returns `downloaded` plus `reused_existing=true`; .NET returns + `already_known`. +- Cross-language drift remains possible if future IPCC download changes update + only one runtime without synchronized fixture coverage. + +## Verdict + +Commit-ready for parity-review scope. + +Task-ID: PT-053 + +Task-Issue: #478 diff --git a/docs/python-ingestion-local-runbook.md b/docs/python-ingestion-local-runbook.md new file mode 100644 index 00000000..7c6e6728 --- /dev/null +++ b/docs/python-ingestion-local-runbook.md @@ -0,0 +1,208 @@ +# Python Ingestion Local Runbook + +This runbook packages the Python ingestion path for a local PostgreSQL run. It +uses checked-in local CSV fixtures for GHG Protocol, DEFRA/DESNZ, and IPCC EFDB +so a user can clone the repository, start Docker PostgreSQL, and run the +3-source ingestion cycle without writing or editing Python files. + +This document does not add .NET parity, production credentials, production +source URLs, legal/compliance claims, carbon-accounting correctness claims, or +source-owner correctness claims. + +## Install + +From a fresh checkout: + +```bash +git clone CarbonOps-Parser +cd CarbonOps-Parser +python -m pip install -e ".[postgresql]" +``` + +The package installs the `carbonops-parser` command from +[pyproject.toml](../pyproject.toml). The PostgreSQL extra installs the local +`psycopg` driver dependency used by the runtime. + +For the controlled live/configured artifact smoke path, see +[Real-Source Smoke Mode](real-source-smoke-mode.md). + +## Start Docker PostgreSQL + +Start an isolated local PostgreSQL container: + +```bash +docker run --name carbonops-postgres \ + -e POSTGRES_USER=carbonops \ + -e POSTGRES_PASSWORD=carbonops_local_password \ + -e POSTGRES_DB=carbonops \ + -p 5432:5432 \ + -d postgres:16 +``` + +If you need to restart a stopped container: + +```bash +docker start carbonops-postgres +``` + +If you need a clean database after a previous local test run: + +```bash +docker rm -f carbonops-postgres +``` + +Then run the `docker run` command again. + +## Configure + +Use the checked-in example config: +[config/carbonops.ingestion.example.json](../config/carbonops.ingestion.example.json). + +The config file contains archive settings, enabled source families, cycle +settings, and local fixture paths for source years `2024`, `2025`, and `2026`. +It does not contain a PostgreSQL password or any other secret. Runtime +PostgreSQL settings come from environment variables. + +The `source_years` entries are explicit artifact configuration. They preserve +the local fixture path for all three source families. Without explicit +configuration, GHG Protocol and IPCC EFDB return `no_available_source_year`; +DEFRA/DESNZ can additionally discover reviewed GOV.UK publication pages for +mapped years only when live source access is explicitly enabled. See +[Source Discovery](source-discovery.md) for the live availability boundary. + +Set local environment variables: + +```bash +export CARBONOPS_POSTGRESQL_HOST=127.0.0.1 +export CARBONOPS_POSTGRESQL_PORT=5432 +export CARBONOPS_POSTGRESQL_DATABASE=carbonops +export CARBONOPS_POSTGRESQL_USERNAME=carbonops +export CARBONOPS_POSTGRESQL_PASSWORD=carbonops_local_password +export CARBONOPS_POSTGRESQL_APPLICATION_NAME=carbonops-parser-local +``` + +Validate the local ingestion config without opening PostgreSQL: + +```bash +carbonops-parser validate-ingestion-config --config config/carbonops.ingestion.example.json --cycles 1 +``` + +## Run Ingestion + +Run the packaged Python ingestion command: + +```bash +carbonops-parser run-ingestion --config config/carbonops.ingestion.example.json +``` + +The dedicated real-source smoke command uses the same configured artifacts and +PostgreSQL writes, but makes the smoke intent explicit: + +```bash +carbonops-parser real-source-smoke --config config/carbonops.ingestion.example.json --cycles 1 +``` + +HTTPS source access is disabled by default. To run against reviewed HTTPS +artifact URLs, set `real_source_smoke.allow_live_source_access` to `true` in the +config or pass `--allow-live-source-access`. + +For a single-line shell command without exported variables: + +```bash +CARBONOPS_POSTGRESQL_HOST=127.0.0.1 CARBONOPS_POSTGRESQL_PORT=5432 CARBONOPS_POSTGRESQL_DATABASE=carbonops CARBONOPS_POSTGRESQL_USERNAME=carbonops CARBONOPS_POSTGRESQL_PASSWORD=carbonops_local_password CARBONOPS_POSTGRESQL_APPLICATION_NAME=carbonops-parser-local carbonops-parser run-ingestion --config config/carbonops.ingestion.example.json +``` + +The example config sets `max_cycles` to `4`, so one command demonstrates the +full local cycle behavior: + +- Cycle 1 targets `2024` for all three source families. +- Cycle 2 reads the latest ingested year from PostgreSQL and targets `2025`. +- Cycle 3 targets `2026`. +- Cycle 4 targets `2027`; because the example config has no `2027` source + artifact, each family returns `no_available_source_year`. + +The `2027` cycle is a safe no-op. It does not insert source records and does not +advance `source_family_year_states`. + +## Verify Records + +Use `psql` inside the Docker container. + +Check source-specific master/detail table counts: + +```bash +docker exec -e PGPASSWORD=carbonops_local_password carbonops-postgres \ + psql -U carbonops -d carbonops -c " +SELECT 'ghg_emission_factor_masters' AS table_name, count(*) AS records FROM ghg_emission_factor_masters +UNION ALL SELECT 'ghg_emission_factor_details', count(*) FROM ghg_emission_factor_details +UNION ALL SELECT 'defra_emission_factor_masters', count(*) FROM defra_emission_factor_masters +UNION ALL SELECT 'defra_emission_factor_details', count(*) FROM defra_emission_factor_details +UNION ALL SELECT 'ipcc_emission_factor_masters', count(*) FROM ipcc_emission_factor_masters +UNION ALL SELECT 'ipcc_emission_factor_details', count(*) FROM ipcc_emission_factor_details +ORDER BY table_name;" +``` + +After a clean 4-cycle run, each source-specific table should have local fixture +records for `2024`, `2025`, and `2026`. + +Check year state: + +```bash +docker exec -e PGPASSWORD=carbonops_local_password carbonops-postgres \ + psql -U carbonops -d carbonops -c " +SELECT source_family, max(ingested_year) AS latest_ingested_year +FROM source_family_year_states +GROUP BY source_family +ORDER BY source_family;" +``` + +Expected latest ingested year after a clean 4-cycle run: + +```text +defra | 2026 +ghg | 2026 +ipcc | 2026 +``` + +Confirm that the unavailable `2027` cycle did not insert source-specific +records: + +```bash +docker exec -e PGPASSWORD=carbonops_local_password carbonops-postgres \ + psql -U carbonops -d carbonops -c " +SELECT 'ghg_masters_2027' AS check_name, count(*) AS records FROM ghg_emission_factor_masters WHERE source_year = 2027 +UNION ALL SELECT 'defra_masters_2027', count(*) FROM defra_emission_factor_masters WHERE source_year = 2027 +UNION ALL SELECT 'ipcc_masters_2027', count(*) FROM ipcc_emission_factor_masters WHERE source_year = 2027 +UNION ALL SELECT 'ghg_details_2027', count(*) FROM ghg_emission_factor_details d JOIN ghg_emission_factor_masters m USING (ghg_emission_factor_master_id) WHERE m.source_year = 2027 +UNION ALL SELECT 'defra_details_2027', count(*) FROM defra_emission_factor_details d JOIN defra_emission_factor_masters m USING (defra_emission_factor_master_id) WHERE m.source_year = 2027 +UNION ALL SELECT 'ipcc_details_2027', count(*) FROM ipcc_emission_factor_details d JOIN ipcc_emission_factor_masters m USING (ipcc_emission_factor_master_id) WHERE m.source_year = 2027 +ORDER BY check_name;" +``` + +Each `2027` count should be `0`. + +## Accepted Risks And Non-Claims + +- The example source artifacts are deterministic local fixtures, not official + source-owner files. +- The Docker password is a local example value only. Do not use it for shared, + hosted, or production databases. +- The command creates and writes Phase 1 tables in the connected local + PostgreSQL database. Use an isolated database or reset the Docker container + before repeat demos. +- Re-running against the same database is idempotent for duplicate source + rows, but the latest-year state will already be advanced from the previous + run. +- The Python runner is the only packaged ingestion path covered here. This task + does not implement .NET runtime parity. +- The local run demonstrates packaging, configuration, source-specific + master/detail persistence, and cycle behavior. Use + [Production Packaging And Operator Runbook](production-packaging-operator-runbook.md) + for production operation. + +## Related Documents + +- [Production E2E Ingestion Readiness Contract](production-e2e-ingestion-readiness-contract.md) +- [PH-018 Real Production-Ready Ingestion Contract](ph-018-real-production-ready-ingestion-contract.md) +- [Source Discovery](source-discovery.md) +- [PostgreSQL Phase 1 Schema Contract](postgresql-phase1-schema-contract.md) diff --git a/docs/real-source-smoke-mode.md b/docs/real-source-smoke-mode.md new file mode 100644 index 00000000..a0fa2d34 --- /dev/null +++ b/docs/real-source-smoke-mode.md @@ -0,0 +1,117 @@ +# Real-Source Smoke Mode + +Real-source smoke mode is the controlled Python path for checking configured +GHG Protocol, DEFRA/DESNZ, and IPCC EFDB artifacts against the PostgreSQL +source-family master/detail tables. It uses the same adapters as the configured +cycle runner and keeps the local fixture path unchanged. + +This mode does not add credentials, scheduler behavior, .NET parity, +production factor correctness claims, compliance claims, legal claims, or +source-owner correctness claims. + +## Command + +Use the dedicated command with an explicit JSON config: + +```bash +carbonops-parser real-source-smoke --config config/carbonops.ingestion.example.json --cycles 1 +``` + +Validate the same config without opening PostgreSQL first: + +```bash +carbonops-parser validate-ingestion-config --config config/carbonops.ingestion.example.json --cycles 1 +``` + +The command supports local files by default. Local artifact references may be +plain paths, `file:` URIs, or `local:` paths. HTTPS source access is blocked +unless the operator opts in with either: + +```bash +carbonops-parser real-source-smoke --config path/to/real-source-smoke.json --allow-live-source-access +``` + +or a config flag: + +```json +{ + "real_source_smoke": { + "allow_live_source_access": true + } +} +``` + +## Config Shape + +Each live or local artifact must be configured explicitly under `source_years`. +The runner supports the canonical source families `ghg_protocol`, +`defra_desnz`, and `ipcc_efdb`. + +```json +{ + "archive_root": "./data/raw", + "enabled_source_families": ["ghg_protocol", "defra_desnz", "ipcc_efdb"], + "initial_year": 2024, + "cycle": {"max_cycles": 1}, + "real_source_smoke": {"allow_live_source_access": false}, + "source_years": { + "ghg_protocol": { + "2024": { + "artifact_url": "examples/fixtures/ingestion/ghg_protocol_2024.csv", + "publication_url": "examples/fixtures/ingestion/ghg_protocol_2024.csv", + "title": "Configured GHG Protocol artifact 2024", + "version_label": "configured-2024", + "content_type": "text/csv", + "format_hint": "csv" + } + } + } +} +``` + +For HTTPS artifacts, set `artifact_url` to the reviewed artifact URL and opt in +to live access. DEFRA/DESNZ can use configured artifact URLs directly; when the +reviewed DEFRA/DESNZ publication fallback is used, live access is also required. + +## Output + +Each source line reports: + +- `download_status`: `downloaded`, `failed`, or `not_run`. +- `parse_status`: `parsed`, `failed`, `not_run`, or `no_rows`. +- `master_inserted` and `detail_inserted`. +- `master_skipped` and `detail_skipped` for idempotent duplicate rows. + +The summary line reports aggregate `no_available_source_year`, parsed row, +inserted, and skipped duplicate counts. Error messages are user-readable and +redacted before they are printed. + +## Docker PostgreSQL Smoke + +Start PostgreSQL as described in +[Python Ingestion Local Runbook](python-ingestion-local-runbook.md), export the +PostgreSQL environment variables, then run: + +```bash +carbonops-parser real-source-smoke --config config/carbonops.ingestion.example.json --cycles 1 +``` + +Verify source-specific master/detail rows: + +```bash +docker exec -e PGPASSWORD=carbonops_local_password carbonops-postgres \ + psql -U carbonops -d carbonops -c " +SELECT 'ghg_emission_factor_masters' AS table_name, count(*) AS records FROM ghg_emission_factor_masters +UNION ALL SELECT 'ghg_emission_factor_details', count(*) FROM ghg_emission_factor_details +UNION ALL SELECT 'defra_emission_factor_masters', count(*) FROM defra_emission_factor_masters +UNION ALL SELECT 'defra_emission_factor_details', count(*) FROM defra_emission_factor_details +UNION ALL SELECT 'ipcc_emission_factor_masters', count(*) FROM ipcc_emission_factor_masters +UNION ALL SELECT 'ipcc_emission_factor_details', count(*) FROM ipcc_emission_factor_details +ORDER BY table_name;" +``` + +Re-running the same smoke against the same database should report skipped +duplicates instead of duplicate inserts for already persisted source rows. + +For production scheduling and the `run-ingestion` command, see +[Production Packaging And Operator Runbook](production-packaging-operator-runbook.md). diff --git a/docs/release-notes-draft.md b/docs/release-notes-draft.md new file mode 100644 index 00000000..3d8d46cf --- /dev/null +++ b/docs/release-notes-draft.md @@ -0,0 +1,45 @@ +# Release Notes Draft + +## CarbonOps-Parser Public Alpha/Review Release + +Status: draft for first public alpha/review release. + +This draft is for a public review release of CarbonOps-Parser, a climate-tech data infrastructure project for auditable carbon accounting emission factor ingestion and validation. The release is intended to make the repository understandable and inspectable for reviewers interested in GHG Protocol, DEFRA/DESNZ, IPCC EFDB, Python, .NET, and PostgreSQL boundaries. + +## Highlights + +- Public project positioning for auditable carbon factor ingestion, validation, diagnostics, and PostgreSQL readiness. +- Phase 1 documentation for supported source families: GHG Protocol, DEFRA/DESNZ, and IPCC EFDB. +- Python implementation slices for local source acquisition boundaries, parser contracts, normalization handoff, persistence input preparation, PostgreSQL previews, and local dry-run composition. +- .NET contract records and tests for shared Phase 1 concepts, including parser, acquisition, validation, diagnostics, and PostgreSQL readiness boundaries. +- PostgreSQL schema descriptors, DDL preview, bootstrap/readiness contracts, disabled execution adapters, and opt-in integration boundaries. +- Deterministic local examples and fixtures, including a DEFRA/DESNZ-style dry-run fixture that requires no network, database, credentials, or production services. + +## What Reviewers Can Try Safely + +From a local checkout, reviewers can install the Python package, run the Python tests, and execute the local dry-run fixture described in the README: + +```bash +python -m pip install -e . +python -m pytest +carbonops-parser local-dry-run \ + --local-path examples/fixtures/defra_desnz_minimal.csv \ + --source-family defra_desnz \ + --source-id defra-desnz-minimal-fixture \ + --content-type text/csv \ + --format-hint csv +``` + +The quickstart is non-destructive. It does not execute SQL, connect to PostgreSQL, perform source downloads, call live source endpoints, run a scheduler, or require credentials. + +## Known Limits + +- No production carbon-accounting correctness, compliance correctness, legal correctness, source-owner correctness, or factor correctness is claimed. +- Live source availability and full upstream source coverage are not proven by the default local checks. +- PostgreSQL runtime writes remain disabled, preview-only, or opt-in depending on the boundary being exercised. +- Scheduler and production deployment behavior are documented as boundaries and roadmap items, not promoted production guarantees. +- GHG Protocol, DEFRA/DESNZ, and IPCC EFDB support should be read as Phase 1 ingestion and parser boundary support, not complete source-owner coverage. + +## Suggested Public Release Notes Summary + +CarbonOps-Parser is ready for public alpha/review as a local-safe, documentation-forward repository for carbon emissions data ingestion infrastructure. It provides searchable public documentation, deterministic examples, Python and .NET contract surfaces, PostgreSQL readiness boundaries, and a conservative roadmap for expanding carbon accounting emission factor ingestion without overclaiming production correctness. diff --git a/docs/roadmap.md b/docs/roadmap.md index 11cc7920..8e3d6b0b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,8 +1,8 @@ # Roadmap -This roadmap organizes the path from the documentation baseline to a `v0.1.0` reference release. +This roadmap organizes the path from the current Phase 1 ingestion foundation to a `v0.1.0` reference release. -## Sprint 1: Documentation Baseline +## Sprint 1: Documentation Foundation - Add public project positioning. - Document architecture, configuration, database model, background job model, and source support. @@ -43,6 +43,23 @@ This roadmap organizes the path from the documentation baseline to a `v0.1.0` re - Add database startup check design. - Add background schedule skeleton. +## Production Parity Sequence + +Project-level production-ready is claimed only in the narrow scope documented +by the final verdict. The Python runtime has a production operator path; the +.NET runtime contributes parity evidence through the sequence below: + +- Add .NET service/scheduled-worker entrypoint. Added by PROD-003 as a + scheduled-worker command-surface baseline only. +- Add .NET production config loader and redaction. +- Add .NET PostgreSQL schema bootstrap and year-state. +- Add .NET source discovery/download/parsing orchestration. +- Add .NET source-specific master/detail insert. +- Add .NET idempotency and rerun behavior. +- Add .NET Docker PostgreSQL E2E tests. +- Add Python/.NET parity validation. +- Record the final project production-ready verdict. Completed by PROD-011. + ## Sprint 7: GHG Protocol and IPCC Preparation - Add source discovery outputs for GHG Protocol. diff --git a/docs/rv-041-source-acquisition-repository-contracts-review.md b/docs/rv-041-source-acquisition-repository-contracts-review.md new file mode 100644 index 00000000..4fbb72eb --- /dev/null +++ b/docs/rv-041-source-acquisition-repository-contracts-review.md @@ -0,0 +1,162 @@ +# RV-041 Review: Source Acquisition Repository Contracts + +Task-ID: RV-041 +Task-Issue: #386 + +## Executive Summary + +This review-only checkpoint covers the source acquisition repository contract +surface after the PT-041, PT-042, and PT-044 parity reviews. The reviewed +contracts remain metadata-only and runtime-passive. No blocker was found for the +next review checkpoint. + +No product or runtime source code was changed. This task did not add +credentials, raw connection strings, live source endpoint calls, database +execution, destructive database operations, branch deletion, worktree deletion, +issue closure, PR merge activity, or source-specific ingestion behavior. + +## Scope Reviewed + +This review covered documentation and existing contract surfaces for repository +contracts related to source acquisition persistence boundaries: + +- source acquisition run repository contract +- source document repository contract +- source-family master/detail repository contract +- cross-language Python and .NET parity-review conclusions for those contracts +- safety posture of the reviewed contracts as non-executing declarations + +Out of scope: + +- new runtime repository implementation +- source-specific downloader or parser behavior +- production database connectivity +- migrations or destructive database operations +- scheduler, retry, or orchestration execution behavior +- production readiness claims beyond this review checkpoint + +## Source Acquisition Repository Contracts Reviewed + +The following existing review artifacts were used as the primary contract +inputs: + +- `docs/pt-041-source-acquisition-run-repository-contract-parity-review.md` +- `docs/pt-042-source-document-repository-contract-parity-review.md` +- `docs/pt-044-source-family-repository-contract-parity-review.md` + +The corresponding contract areas are: + +- Python source acquisition run repository: + `src/carbonfactor_parser/source_acquisition/run_repository_contract.py` +- Python source document repository: + `src/carbonfactor_parser/persistence/source_document_repository.py` +- Python source-family repository: + `src/carbonfactor_parser/persistence/source_family_repository.py` +- .NET source acquisition run repository contract records and registry under + `src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepository*.cs` +- .NET source document repository contract records and registry under + `src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepository*.cs` +- .NET source-family repository contract records, registry, and table-name + helper under `src/dotnet/CarbonOps.Parser.Contracts/SourceFamily*.cs` + +The reviewed contracts share the same broad repository declaration model: + +- provider-name validation +- persist-operation shape +- deterministic persist-result status +- persisted-count reporting +- validation issue collection +- failed-validation behavior that reports zero persisted records +- metadata-only registry/helper functions + +## Python/.NET Readiness Observations + +Python and .NET readiness is acceptable for this review checkpoint. + +The parity reviews confirm aligned contract intent across the Python and .NET +surfaces for: + +- repository interface/protocol shape +- issue record shape and default severity +- validation-result shape +- deterministic declaration vs failed-validation status transitions +- persisted-count handling +- source-family table-name mapping for Phase 1 source families +- runtime-passive behavior + +Language-specific differences remain expected and accepted: + +- Python uses snake_case identifiers while .NET uses PascalCase identifiers. +- Python validates non-contract objects generically while .NET validates null or + invalid typed entries through its type system. +- Python can coerce supported source-family values into enum values while .NET + receives enum values directly and rejects undefined values. + +These differences do not block the review checkpoint because the observable +contract outcomes remain aligned. + +## Contract Consistency Assessment + +The reviewed repository contracts are consistent enough for the next review +checkpoint. + +The source acquisition run, source document, and source-family repository +contracts all follow the same pattern: + +- no runtime persistence is performed by the contract helper itself +- validation happens before declared persistence results are returned +- validation failure prevents nonzero persisted counts +- provider names are required and echoed in results +- issue collections are reported as contract data +- caller-supplied issues are preserved when determining final status + +The PT-041, PT-042, and PT-044 reviews found no blocking parity mismatch across +Python and .NET. Their combined findings support a consistent repository +contract model for source acquisition persistence boundaries. + +## Safety Assessment + +### No Production Credentials + +No production credentials were added, reviewed, required, or referenced. This +review document does not include secrets, tokens, passwords, usernames, or raw +connection strings. + +### No Live Source Endpoint Calls + +No live source endpoints were called. The reviewed repository contracts are +metadata-only persistence boundaries and do not fetch remote resources. + +### No Runtime DB Execution + +No runtime database execution was performed. The reviewed contracts define +declaration and validation shapes only; they do not open database connections or +execute SQL. + +### No Destructive DB Operations + +No destructive database operations were performed or added. This review did not +run migrations, truncate data, drop objects, delete records, or issue database +commands. + +## Remaining Risks + +- This review confirms repository contract consistency only; it does not prove + future runtime repository implementation correctness. +- Cross-language drift remains possible if future changes update Python or .NET + contract surfaces without synchronized tests and review artifacts. +- Execution-path behavior remains intentionally deferred until a separate scoped + task introduces runtime persistence with explicit safety gates. +- Source-specific acquisition correctness, parser correctness, factor + correctness, compliance interpretation, and carbon-accounting correctness are + outside this review. + +## Verdict + +ready for next review checkpoint + +The reviewed source acquisition repository contracts are consistent, +runtime-passive, and aligned with the safety boundaries required for RV-041. + +Task-ID: RV-041 +Task-Issue: #386 diff --git a/docs/rv-042-ghg-download-execution-readiness-review.md b/docs/rv-042-ghg-download-execution-readiness-review.md new file mode 100644 index 00000000..b9678cf7 --- /dev/null +++ b/docs/rv-042-ghg-download-execution-readiness-review.md @@ -0,0 +1,111 @@ +# RV-042 GHG Download Execution Readiness Review + +## Executive Summary + +This review covers GHG Protocol source download execution readiness after the +PT-046 parity checkpoint. The reviewed surface remains documentation and +contract oriented: GHG download execution is explicit, caller-driven, and +guarded against parser execution, scheduler behavior, SQL generation, and +runtime database writes. + +No blocker was found for the next review checkpoint. The GHG download execution +boundary is ready to proceed with the accepted limitation that it is not a +production downloader, live endpoint validation, or database execution review. + +## Scope Reviewed + +- PT-046 GHG source download execution parity review. +- Python GHG source download execution request, validation, execution result, + artifact, issue, and transport-response contracts. +- Python contract API exports for the GHG download execution boundary. +- Focused Python tests covering explicit opt-in behavior, unsafe input + blocking, injected transport behavior, checksum handling, path safety, + side-effect flags, and runtime-passive import behavior. +- .NET GHG source download execution contract and focused parity test surface + as summarized by PT-046. + +This review did not modify product or runtime source code. + +## GHG Download Execution Readiness Assessment + +The GHG download execution boundary is ready for the next review checkpoint +within its current contract scope. + +The boundary requires explicit request flags before execution: + +- `allow_download_execution` +- `allow_file_write` +- `allow_network` for HTTPS references +- `allow_overwrite` when replacing an existing target is intended + +The execution path is caller-driven through an injected transport callback. The +boundary does not own source-specific HTTP clients, production retry behavior, +authentication, release scheduling, parser execution, normalization, or database +persistence. Successful execution produces local artifact metadata with source +identity, candidate identity, local path, checksum, size, content metadata, and +version/year metadata where available. + +Validation remains fail-closed for missing required fields, non-GHG source +identity, non-downloadable candidates, discovery references, insecure or unsafe +URI schemes, missing network opt-in, unsafe target paths, symlink escapes, +existing targets without overwrite opt-in, failed transport responses, empty +content, blank response metadata, checksum mismatches, and write failures. + +## Contract And Boundary Consistency Notes + +- Source identity is consistently scoped to `ghg_protocol`. +- The public status model remains `blocked`, `downloaded`, and `failed`. +- Blocked and failed results require diagnostic issues. +- Downloaded results require artifact metadata. +- Non-downloaded results must not include artifact metadata. +- Result side-effect guard flags preserve no parser execution, no database + writes, no SQL behavior, and no scheduler behavior. +- Python and .NET contract surfaces are aligned by PT-046 for request shape, + result shape, artifact metadata, issue diagnostics, status vocabulary, + transport-response validation, and observable target-path safety behavior. +- The contract API exports the GHG download execution boundary without adding a + broader ingestion or orchestration dependency. + +## Safety Assessment + +No production credentials are introduced or required by this review. The GHG +download execution request does not contain raw connection strings, database +credentials, source credentials, or environment-backed secret loading. + +No live source endpoint calls were made for this review. The reviewed execution +boundary uses caller-provided transport and the focused tests use local, +deterministic mock-style payloads. + +No runtime database execution was performed or added. The boundary exposes +`allow_database_writes` only as a forbidden side-effect flag and validates that +it remains disabled. + +No destructive database operations were performed or added. The reviewed +boundary does not generate SQL, open database sessions, run migrations, truncate +tables, delete records, or otherwise mutate runtime database state. + +## Remaining Risks + +- This review confirms readiness only for the current GHG download execution + contract and review checkpoint; it does not validate production live-source + availability, endpoint stability, retry behavior, rate limiting, caching, or + authentication. +- The boundary can write local files when explicitly opted in, so future callers + still need integration-level controls for target roots, retention policy, + operator approval, and artifact lifecycle. +- Parser execution, normalization, scheduler integration, and persistence + remain outside this review and require their own readiness checks before any + production workflow claim. +- Python and .NET use language-specific filesystem hardening mechanisms; PT-046 + found the observable contract aligned, but future changes can reintroduce + cross-runtime drift without synchronized tests. +- The review did not execute live network, parser, scheduler, or database + workflows by design, so operational readiness for those systems remains + unproven. + +## Verdict + +ready for next review checkpoint + +Task-ID: RV-042 +Task-Issue: #387 diff --git a/docs/rv-043-defra-download-execution-readiness-review.md b/docs/rv-043-defra-download-execution-readiness-review.md new file mode 100644 index 00000000..85c073e6 --- /dev/null +++ b/docs/rv-043-defra-download-execution-readiness-review.md @@ -0,0 +1,114 @@ +# RV-043 DEFRA Download Execution Readiness Review + +## Executive Summary + +This review covers DEFRA/DESNZ source download execution readiness after the +PT-048 parity checkpoint. The reviewed surface remains documentation and +contract oriented: DEFRA/DESNZ download execution is explicit, caller-driven, +and guarded against parser execution, scheduler behavior, SQL generation, and +runtime database writes. + +No blocker was found for the next review checkpoint. The DEFRA/DESNZ download +execution boundary is ready to proceed with the accepted limitation that this +review does not validate production live-source availability, production +downloader behavior, or runtime database execution. + +## Scope Reviewed + +- PT-048 DEFRA source download execution parity review. +- Python DEFRA source download execution request, validation, execution result, + artifact, issue, and transport-response contracts. +- Python contract API exports for the DEFRA download execution boundary. +- Focused Python tests covering explicit opt-in behavior, non-downloadable + discovery candidates, unsafe input blocking, injected transport behavior, + checksum handling, path safety, side-effect flags, and runtime-passive import + behavior. +- .NET DEFRA source download execution contract and focused parity test surface + as summarized by PT-048. + +This review did not modify product or runtime source code. + +## DEFRA/DESNZ Download Execution Readiness Assessment + +The DEFRA/DESNZ download execution boundary is ready for the next review +checkpoint within its current contract scope. + +The boundary requires explicit request flags before execution: + +- `allow_download_execution` +- `allow_file_write` +- `allow_network` for HTTPS references +- `allow_overwrite` when replacing an existing target is intended + +The execution path is caller-driven through an injected transport callback. The +boundary does not own source-specific HTTP clients, production retry behavior, +authentication, release scheduling, parser execution, normalization, or +database persistence. Successful execution produces local artifact metadata +with source identity, candidate identity, local path, checksum, size, content +metadata, and version/year metadata where available. + +Validation remains fail-closed for missing required fields, non-DEFRA source +identity, non-downloadable candidates, discovery references, insecure or unsafe +URI schemes, missing network opt-in, unsafe target paths, symlink escapes, +existing targets without overwrite opt-in, failed transport responses, empty +content, blank response metadata, checksum mismatches, and write failures. + +## Contract And Boundary Consistency Notes + +- Source identity is consistently scoped to `defra_desnz`. +- The public status model remains `blocked`, `downloaded`, and `failed`. +- Blocked and failed results require diagnostic issues. +- Downloaded results require artifact metadata. +- Non-downloaded results must not include artifact metadata. +- Result side-effect guard flags preserve no parser execution, no database + writes, no SQL behavior, and no scheduler behavior. +- Python and .NET contract surfaces are aligned by PT-048 for request shape, + result shape, artifact metadata, issue diagnostics, status vocabulary, + transport-response validation, malformed URI diagnostics, and observable + target-path safety behavior. +- The contract API exports the DEFRA download execution boundary without adding + a broader ingestion, parser, scheduler, or database dependency. + +## Safety Assessment + +No production credentials are introduced or required by this review. The +DEFRA/DESNZ download execution request does not contain raw connection strings, +database credentials, source credentials, or environment-backed secret loading. + +No live source endpoint calls were made for this review. The reviewed execution +boundary uses caller-provided transport and the focused tests use local, +deterministic mock-style payloads. + +No runtime database execution was performed or added. The boundary exposes +`allow_database_writes` only as a forbidden side-effect flag and validates that +it remains disabled. + +No destructive database operations were performed or added. The reviewed +boundary does not generate SQL, open database sessions, run migrations, +truncate tables, delete records, or otherwise mutate runtime database state. + +## Remaining Risks + +- This review confirms readiness only for the current DEFRA/DESNZ download + execution contract and review checkpoint; it does not validate production + live-source availability, endpoint stability, retry behavior, rate limiting, + caching, or authentication. +- The boundary can write local files when explicitly opted in, so future callers + still need integration-level controls for target roots, retention policy, + operator approval, and artifact lifecycle. +- Parser execution, normalization, scheduler integration, and persistence + remain outside this review and require their own readiness checks before any + production workflow claim. +- Python and .NET use language-specific filesystem hardening mechanisms; PT-048 + found the observable contract aligned, but future changes can reintroduce + cross-runtime drift without synchronized tests. +- The review did not execute live network, parser, scheduler, or database + workflows by design, so operational readiness for those systems remains + unproven. + +## Verdict + +ready for next review checkpoint + +Task-ID: RV-043 +Task-Issue: #388 diff --git a/docs/rv-050-parser-source-download-production-readiness-review.md b/docs/rv-050-parser-source-download-production-readiness-review.md new file mode 100644 index 00000000..10f9a25b --- /dev/null +++ b/docs/rv-050-parser-source-download-production-readiness-review.md @@ -0,0 +1,92 @@ +# RV-050 Parser And Source Download Production Readiness Review + +## Summary + +This review covers the completed parser/source-download expansion for GHG +Protocol, DEFRA/DESNZ, and IPCC EFDB normalized output paths. The reviewed +contract surface is coherent enough for the next persistence-integration task: +source acquisition can describe discovered/downloaded artifacts, those artifacts +can be bridged into parser input metadata, the parser content boundaries produce +source identity and provenance fields, and persistence input already requires +normalized records to carry source identity before database execution. + +No blocking contract mismatch was found that requires a code fix in this review +scope. + +## Reviewed Scope + +- Source discovery and download boundaries for GHG Protocol, DEFRA/DESNZ, and + IPCC EFDB. +- Source download artifact metadata and source-artifact to parser-input bridge + contracts. +- Phase 1 parser adapter registry and parser input artifact contracts. +- GHG Protocol, DEFRA/DESNZ, and IPCC EFDB content parser normalized output + paths. +- Parser execution to normalization handoff and normalization input boundaries. +- Persistence input boundary and PostgreSQL preview/runtime safety gates. +- Existing parity notes for PT-046, PT-048, and PT-053. + +## Readiness Findings + +The source families use stable source identities across acquisition, parser +input, parser execution, normalized raw fields, and persistence preparation: + +- `ghg_protocol` +- `defra_desnz` +- `ipcc_efdb` + +The source-download execution boundaries remain explicitly opt-in and +side-effect guarded. They reject unsafe paths and non-opted-in network/file +write behavior, preserve checksum/provenance metadata on successful artifact +creation, and keep parser execution, SQL, database writes, and scheduler +behavior out of the download boundary. + +The source-artifact to parser-input bridge preserves local artifact reference, +source family/key, parser key, content metadata, checksum, document year, and +reporting year without reading files or calling external systems. That is enough +metadata for persistence integration to identify source lineage and artifact +provenance after parser execution. + +The parser content boundaries for GHG Protocol, DEFRA/DESNZ normalized +extraction rows, and IPCC EFDB produce deterministic normalized raw fields with +`source_family`, `source_id`, source year/version, factor identity/value/unit, +source-specific categorization fields, provenance artifact reference, checksum +metadata, row number, and stable master/detail external keys. + +The persistence input boundary is appropriately fail-closed for this stage: it +requires normalized records to include `source_family` and `source_id`, rejects +mixed source identities in a single persistence input, and does not connect to +or write to a database. + +## Known Limitations + +- The reviewed source-download paths are contract and local-test ready; they do + not prove live-source availability, live network reliability, or production + downloader scheduling. +- GHG Protocol and IPCC EFDB have parser normalized content paths and parity + fixtures, but no source-specific normalization mapper to convert their parser + raw payloads into `NormalizationResult` records for persistence input yet. +- DEFRA/DESNZ has a local dry-run path through normalization and persistence + input, but the existing mapper is still scoped as a minimal fixture/extraction + mapper and does not claim source correctness. +- The normalized parser outputs are deterministic local CSV-style extraction + contracts, not complete production parsers for arbitrary upstream workbooks, + PDFs, web pages, or downloaded archives. +- The download execution boundaries use injected transport/local fixtures in + tests; they do not own production HTTP retry, rate limiting, authentication, + caching, or release packaging behavior. +- Persistence integration remains preview/contract oriented. Runtime database + execution is guarded separately and was not executed in this review. +- Cross-language drift remains possible if Python and .NET source-download or + parser contracts are changed without synchronized parity fixtures/tests. + +## Verdict + +Merge-ready for RV-050 review scope. + +The parser/source-download contracts are coherent enough for persistence +integration to proceed, with the limitations above treated as follow-on scope +rather than blockers for this review. + +Task-ID: RV-050 +Task-Issue: #479 diff --git a/docs/rv-051-parsed-factor-persistence-production-readiness-review.md b/docs/rv-051-parsed-factor-persistence-production-readiness-review.md new file mode 100644 index 00000000..93176adc --- /dev/null +++ b/docs/rv-051-parsed-factor-persistence-production-readiness-review.md @@ -0,0 +1,75 @@ +# RV-051 Parsed Factor Persistence Production Readiness Review + +## Summary + +This review covers the parsed factor persistence writer contracts added after +PT-054. The reviewed path is coherent enough for full ingestion orchestration to +begin wiring parser output into the source-family repository boundary: Python +and .NET both map parsed normalized rows into source-family master/detail +records, preserve source document identity, validate required factor fields, +deduplicate identical identities, reject conflicting duplicates, and return +declared/failed/no-records outcomes without enabling PostgreSQL runtime writes. + +One narrow contract fix was made during this review: the Python public API test +now explicitly covers the parsed factor persistence writer types and both build +and persist functions. + +## Reviewed Scope + +- Python parsed factor persistence writer boundary. +- .NET parsed factor persistence writer contract boundary. +- Shared parity fixture for fallback persistence identity and checksums. +- Source-family repository validation contracts used by the writer. +- PostgreSQL repository and runtime execution safety gates. +- Persistence package public exports. + +## Readiness Findings + +The writer has a stable in-memory command boundary. Both language surfaces build +source-family master/detail records with deterministic identifiers, external +keys, record checksums, lifecycle status, and timestamp labels. + +The Python writer accepts both `ParsedRawRecordPayload` and +`ParserNormalizedOutputBatch`; the .NET writer accepts +`ParserNormalizedOutputBatch`. Both surfaces validate required source document +identity, factor value, and factor unit before calling the repository. + +Duplicate handling is explicit. Identical master/detail identities are +deduplicated and counted; conflicting identities produce validation issues and +block repository submission. + +The repository handoff remains protocol-level. The fake/source-family repository +tests verify attempted and persisted counts, while the concrete PostgreSQL +repository still returns unsupported and does not connect, run SQL, write +records, start transactions, or load credentials. + +The shared parity fixture confirms cross-language fallback persistence intent +for source document id, master/detail ids, external keys, checksums, and default +timestamps. + +## Known Limitations + +- This is not an end-to-end ingestion orchestration path. No scheduler, + downloader, parser runner, repository runtime execution, or database write path + is added here. +- PostgreSQL runtime persistence remains intentionally disabled and unsupported. +- The writer maps only the current source-family master/detail contract fields; + production database migrations, conflict actions, transaction behavior, retry + policy, observability, and rollback verification remain future work. +- Python and .NET do not accept identical input shapes: Python also accepts raw + parsed payloads, while .NET currently covers normalized output batches. +- The default timestamp label is deterministic review metadata, not a production + clock value. +- Source correctness, carbon-accounting correctness, and upstream data coverage + are not claimed by this review. + +## Verdict + +Merge-ready for RV-051 review scope. + +The parsed factor persistence writer behavior is coherent enough for the next +full ingestion orchestration task, with the limitations above treated as +follow-on scope rather than blockers for this review. + +Task-ID: RV-051 +Task-Issue: #483 diff --git a/docs/rv-052-phase1-ingestion-orchestrator-production-readiness-review.md b/docs/rv-052-phase1-ingestion-orchestrator-production-readiness-review.md new file mode 100644 index 00000000..322ee6ce --- /dev/null +++ b/docs/rv-052-phase1-ingestion-orchestrator-production-readiness-review.md @@ -0,0 +1,109 @@ +# RV-052 Phase 1 Ingestion Orchestrator Production Readiness Review + +## Summary + +This review covers the Phase 1 ingestion orchestrator contracts across Python +and .NET before service and scheduler hardening. The reviewed contract surface +is coherent enough for service/runtime integration planning to proceed: +source-family selection is explicit, execution is runtime-injected and +sequential by default, run state is captured in top-level and per-family +results, partial failures are reported deterministically, and PostgreSQL runtime +readiness remains fail-closed before source execution. + +No blocking contract mismatch was found that requires a code fix in this review +scope. + +## Reviewed Scope + +- Python Phase 1 ingestion orchestrator request, dependency, runtime, result, + summary, and failure contracts. +- Python orchestrator tests for source-family selection, duplicate selection, + sequential execution, PostgreSQL readiness blocking, deterministic stage + failures, and partial family failure semantics. +- .NET Phase 1 ingestion orchestrator request, dependency, runtime, result, and + failure contracts. +- .NET orchestrator tests for explicit source selection, duplicate selection, + sequential execution, bounded-parallel blocking, PostgreSQL runtime config + blocking, and partial family failure semantics. +- Related persistence, parser, source acquisition, and PostgreSQL runtime safety + contracts used by the orchestrator boundary. + +## Readiness Findings + +Source-family selection is explicit and limited to the Phase 1 source families: +`ghg_protocol`, `defra_desnz`, and `ipcc_efdb`. Python accepts stable aliases +such as `ghg`, `defra`, and `ipcc` before normalizing to canonical family keys; +.NET accepts the `SourceFamily` enum and filters duplicate selections. Both +surfaces require at least one valid family and avoid running unselected +families. + +Run state is structured enough for a service adapter to observe orchestration +outcomes. Python records top-level run status, selected families, per-family +stage-specific statuses, summary counts, persisted counts, and structured +failure details. .NET records top-level run status, selected families, +per-family completed/failed status, aggregate counts, persisted counts, and +structured stage failures. + +Retry and idempotency posture is intentionally conservative. The orchestrators +do not implement retry loops, background scheduling, queue claims, distributed +locks, or automatic replay. Duplicate source-family selections collapse to one +execution in both languages. Repository and parsed-factor persistence contracts +remain responsible for attempted/persisted counts and validation failures, while +database idempotency, conflict policy, transaction retry, and rollback behavior +remain deferred to scoped runtime persistence work. + +Partial failure semantics are deterministic. A failing family produces a +per-family failure result and does not prevent later selected families from +running in sequential order. Top-level status becomes +`completed_with_failures`/`CompletedWithFailures` when at least one family +completes and at least one fails, and `failed`/`Failed` when no selected family +completes. Repository, parser, acquisition, and runtime exception stages are +reported through structured failure records. + +Runtime readiness remains guarded. Python can block before source execution +when PostgreSQL runtime config is not ready or schema bootstrap reports missing +required tables. .NET blocks before source execution when PostgreSQL runtime +configuration is requested but not enabled. Both surfaces keep production +credentials, environment loading, scheduler behavior, and destructive database +operations outside this orchestrator review. + +## Known Limitations + +- The orchestrators are contract/runtime-injection boundaries, not production + services. No scheduler, worker lifecycle, service installer, queue consumer, + cancellation, lease, distributed lock, or deployment packaging behavior is + added or validated here. +- Retry behavior is not implemented. Production retry policy, backoff, + dead-letter handling, and replay safety remain future service/runtime scope. +- End-to-end idempotency is not proven at the orchestrator level. Duplicate + family selection is idempotent, but database conflict policy, transaction + rollback, version/hash replay handling, and source document uniqueness remain + persistence/runtime responsibilities. +- Python and .NET have different internal runtime shapes: Python separates + discovery, download, and parse calls; .NET combines discovery/download before + normalization. This is acceptable for the current review because both expose + equivalent observable orchestration outcomes, but future parity work should + preserve cross-language stage semantics when runtime adapters harden. +- Python has a schema-bootstrap readiness input on the orchestrator request; + .NET currently gates PostgreSQL runtime config but does not model the same + schema-bootstrap report on the orchestrator request. This is a known runtime + integration gap, not a merge blocker for the contract review because .NET + schema bootstrap execution is still separately scoped. +- .NET per-family status is coarser than Python's stage-specific family status. + Stage details are still present in structured failure records, so service + integration can classify failures, but exact enum parity is not present. +- Parser/source/downloader implementations remain fixture or boundary scoped in + tests. This review does not prove live upstream availability, source + correctness, carbon-accounting correctness, or arbitrary production document + handling. + +## Verdict + +Merge-ready for RV-052 review scope. + +The Phase 1 ingestion orchestrator behavior is coherent enough for +service/runtime integration to proceed, with the limitations above treated as +follow-on hardening scope rather than blockers for this review. + +Task-ID: RV-052 +Task-Issue: #488 diff --git a/docs/rv-053-service-host-production-readiness-review.md b/docs/rv-053-service-host-production-readiness-review.md new file mode 100644 index 00000000..5d0a2a1e --- /dev/null +++ b/docs/rv-053-service-host-production-readiness-review.md @@ -0,0 +1,119 @@ +# RV-053 Service Host Production Readiness Review + +## Summary + +This review covers the Python and .NET Phase 1 service host contracts for +scheduled ingestion before observability, deployment packaging, and release +hardening. The reviewed surface is coherent enough for production hardening to +proceed: startup validation is fail-closed, schema-bootstrap readiness is +checked before scheduled source execution, only sequential scheduled execution +is accepted, overlapping trigger attempts are skipped, shutdown stops new work +while allowing active work to finish, and runner failures release the local +overlap guard. + +No blocking contract mismatch was found that requires a code fix in this review +scope. + +## Reviewed Scope + +- Python `Phase1ScheduledIngestionServiceHost`, startup config validation, + schema-bootstrap handoff, trigger lifecycle, shutdown handling, and dedicated + tests. +- .NET `Phase1ScheduledIngestionServiceHost`, startup config validation, + schema-bootstrap handoff, trigger lifecycle, shutdown handling, and dedicated + tests. +- Phase 1 ingestion orchestrator readiness behavior used by the service host. +- PostgreSQL runtime config and schema-bootstrap boundary contracts used during + service-host startup and orchestrator requests. +- Background job model and database startup documentation that define the + deferred scheduler, lock, and startup expectations. + +## Readiness Findings + +Startup validation is explicit and fail-closed. Both language surfaces require +at least one Phase 1 source family, a non-empty run ID prefix, a positive +schedule interval, sequential execution, single parallelism, valid PostgreSQL +options, and an explicit `password_set`/`PasswordSet` credential availability +signal. Invalid configuration leaves the host blocked and prevents scheduled +orchestrator execution. + +Bootstrap behavior is ordered before source execution. The host invokes a +schema-bootstrap checker only after local configuration validation succeeds. +When required Phase 1 tables are reported missing and `fail_on_missing_schema` +is enabled, startup returns blocked readiness and scheduled runs are not +accepted. The default bootstrap checker remains passive in the current +contracts: it describes table readiness and intent without opening a connection +or running SQL. + +Scheduled execution is intentionally narrow. Both implementations accept only +sequential Phase 1 ingestion and build one orchestrator request per accepted +trigger with the selected source families, run ID, runtime config gate, and +startup schema-bootstrap report. The host does not introduce source-specific +ingestion, downloader behavior, parser behavior, database writes, production +credentials, or release packaging. + +Overlapping-run prevention is coherent for a single service-host instance. A +local synchronization guard marks a run active before invoking the orchestrator. +Nested or concurrent trigger attempts observe the active run and return a +structured skipped-already-running result rather than starting another +orchestrator request. Runner exceptions still release the guard in `finally` +logic and return the host to ready unless shutdown was requested. + +Shutdown semantics are explicit enough for hardening. A shutdown request sets a +host-level shutdown flag. If no run is active, the host stops immediately and +later trigger attempts are skipped. If a run is active, the host reports +shutdown-requested, rejects new trigger attempts, and transitions to stopped +after the active runner unwinds. The current contract does not interrupt active +work. + +Operational failure modes are surfaced structurally. Startup failures produce +`blocked` startup results with issue codes and field names. Trigger-before-start, +trigger-while-running, and trigger-after-shutdown each return deterministic +skipped results with issue metadata. Orchestrator runner exceptions propagate to +the caller while still cleaning up host lifecycle state. + +Python and .NET are aligned at the observable service-host level. Status enums, +startup result shape, scheduled run result shape, sequential-only validation, +bootstrap blocking, overlap prevention, shutdown transitions, run ID sequencing, +and runner-error cleanup are represented in both implementations. Naming differs +where each language follows local conventions, but no blocking behavioral drift +was found. + +## Known Limitations + +- The service host is a synchronous contract boundary, not a deployed + production worker. It does not own OS service registration, container entry + points, deployment packaging, health endpoints, or process supervision. +- The schedule interval is validated but not executed by an actual timer, + cron loop, queue consumer, or hosted-service runtime in this scope. +- Overlap prevention is local to one host instance. There is no distributed + lease, database-backed lock acquisition, stale-lock cleanup, lock renewal, or + cross-process single-instance guarantee yet. +- Shutdown is graceful only at the host boundary. It does not pass cancellation + tokens or cancellation signals into discovery, download, parser, + normalization, or persistence runtime adapters. +- Bootstrap behavior is still passive by default. Real PostgreSQL connection + checks, schema creation, migration execution, retries, and rollback behavior + remain separate hardening scope. +- Operational observability remains deferred. The contracts expose structured + results and issues, but they do not emit logs, metrics, traces, alerts, + readiness probes, or run-history events. +- Retry, backoff, dead-letter handling, replay policy, and idempotent recovery + after process failure are not implemented at the service-host layer. +- Production credential loading is intentionally outside this review. The host + validates that credential availability was confirmed without storing or + resolving secrets itself. +- Source correctness, parser correctness, normalization correctness, unit + conversion correctness, compliance/legal interpretation, and carbon-accounting + correctness are not assessed by this service-host review. + +## Verdict + +Merge-ready for RV-053 review scope. + +The Phase 1 service host and scheduled execution behavior are coherent enough +for production hardening work to proceed, with the limitations above treated as +follow-on scope rather than blockers for this review. + +Task-ID: RV-053 +Task-Issue: #492 diff --git a/docs/rv-054-observability-and-diagnostics-production-readiness-review.md b/docs/rv-054-observability-and-diagnostics-production-readiness-review.md new file mode 100644 index 00000000..77c42e3a --- /dev/null +++ b/docs/rv-054-observability-and-diagnostics-production-readiness-review.md @@ -0,0 +1,127 @@ +# RV-054 Observability And Diagnostics Production Readiness Review + +## Summary + +This review covers Phase 1 observability and operational diagnostics across +Python and .NET before final production hardening and release packaging. The +shared diagnostic payload contracts are mostly coherent: sensitive PostgreSQL +runtime values are redacted, orchestrator events carry run and correlation +identity, per-family diagnostics include source-family context, run status is +reported at family and orchestrator levels, and structured failure records carry +reason codes. + +The review found no evidence of raw credential logging in the tested diagnostic +helpers. The previously blocking .NET service-host diagnostic gap and +Python/.NET orchestrator event-name drift have been resolved, so the PR is now +merge-ready for this review scope. This does not claim a complete production +observability stack; it confirms the contract-level diagnostics needed for the +next hardening increment. + +## Reviewed Scope + +- Python `phase1_observability` helpers for redaction, event serialization, + PostgreSQL option summaries, orchestrator request summaries, per-family result + summaries, and run-level result summaries. +- Python Phase 1 orchestrator operational log emission and tests. +- Python Phase 1 service-host startup and scheduled-run diagnostic emission and + tests. +- .NET `Phase1OperationalDiagnostics` helpers for redaction, event + serialization, PostgreSQL option summaries, orchestrator request summaries, + per-family result summaries, and run-level result summaries. +- .NET Phase 1 orchestrator operational event sink and tests. +- .NET Phase 1 service-host operational event sink and lifecycle diagnostic + tests. +- Shared `phase1_operational_diagnostics_expectations.json` parity fixture. +- Prior RV-052 and RV-053 findings for orchestrator and service-host readiness. + +## Readiness Findings + +Redaction is explicit and tested in both language surfaces. PostgreSQL host, +database, username, application name, DSN, connection string, URI, database URL, +password, token, secret, and credential-shaped fields are redacted before being +serialized into diagnostics. Connection URI user-info and assignment-style +secret fragments are also removed from diagnostic strings. The diagnostic +summaries expose `password_set` as a boolean capability signal without exposing +the password value. + +Correlation is present for orchestrator diagnostics. Python and .NET emit +`correlation_id` and `run_id` in orchestrator start, per-family completion, and +run completion payloads using shared event names: +`phase1_ingestion_orchestrator_started`, +`phase1_source_family_completed`, and +`phase1_ingestion_orchestrator_completed`. This is enough to connect a selected +Phase 1 orchestrator run with its per-family diagnostic records when the +sink/logger is wired by a host. + +Run-status reporting is structured. Python reports top-level statuses such as +`completed`, `completed_with_failures`, `failed`, and `not_executable`, plus +stage-specific family statuses such as `failed_parser` or +`failed_source_document_persistence`. .NET reports the same top-level run +statuses and coarser family statuses of `completed`, `failed`, and `skipped`. +The coarser .NET family status is acceptable only because failure `stage` and +`code` remain present in each structured failure record. + +Source-family context is consistently included in diagnostic payloads. Request +diagnostics include selected source families; per-family diagnostics include +`source_family` and `source_key`; document summaries include source family, +source key, document ID, and safe checksum; failure summaries include source +family, source key, field name, stage, severity, and code. + +Failure reason codes are present and deterministic enough for triage. Python +and .NET orchestrator failures carry structured codes rather than free-text-only +messages, and tested diagnostics preserve those codes while redacting sensitive +message fragments. + +## Resolved Blocking Mismatches + +- The .NET service host now accepts an optional operational event sink and emits + structured lifecycle diagnostics for startup, startup result, scheduled-run + start, scheduled-run completion, and skipped scheduled runs: + `phase1_service_host_starting`, `phase1_service_host_started`, + `phase1_service_host_scheduled_run_started`, + `phase1_service_host_scheduled_run_completed`, and + `phase1_service_host_scheduled_run_skipped`. +- Python and .NET orchestrator event names are now aligned on + `phase1_ingestion_orchestrator_started` and + `phase1_ingestion_orchestrator_completed`. +- The shared parity fixture now represents both orchestrator event-name parity + and service-host lifecycle event-name parity, and .NET tests assert the + service-host sink path and PostgreSQL-sensitive value redaction. + +## Known Limitations + +- Diagnostics are structured JSON/log-contract helpers, not a full + observability stack. Metrics, traces, alert rules, dashboards, SLOs, + retention policy, and centralized log ingestion are not implemented here. +- Python logs directly through the Phase 1 logger; .NET uses optional event + sinks. Production log pipeline wiring remains separate from this contract + work. +- Service-host diagnostics summarize lifecycle outcomes, but they do not add + distributed scheduler identity, process identity, deployment version, host + name, retry attempt, lock owner, or cancellation context. +- Distributed scheduler identity is not implemented. +- Distributed lock or lease behavior is not implemented. +- Runtime database execution was not performed. Redaction was validated through + deterministic contract tests and local summaries only. +- Diagnostic checks do not prove live source availability, parser correctness, + source correctness, carbon-accounting correctness, deployment packaging, or + release packaging. +- Redaction is pattern-based. It protects the expected credential fields and + common inline secret forms, but production logging policy should still avoid + passing arbitrary raw exception payloads or connection strings into diagnostic + messages. + +## Verdict + +Merge-ready for the RV-054 contract-level observability and diagnostics review +scope. + +The Python and .NET diagnostic payload helpers are coherent enough to preserve +safe redaction, correlation identity, run status, source-family context, and +failure reason codes at the orchestrator and service-host levels. The earlier +blocking .NET service-host diagnostic gap, cross-language event-name drift, and +fixture coverage gap are fixed. The known limitations above remain out of scope +for this task and should be handled by later production hardening work. + +Task-ID: RV-054 +Task-Issue: #496 diff --git a/docs/source-acquisition-boundary.md b/docs/source-acquisition-boundary.md index e818726e..e462d17a 100644 --- a/docs/source-acquisition-boundary.md +++ b/docs/source-acquisition-boundary.md @@ -10,7 +10,7 @@ Source acquisition is the future boundary responsible for obtaining or locating The current repository has local and artificial examples that help demonstrate contract shape and deterministic workflows. Existing local/artificial examples do not imply production source acquisition coverage, real source coverage, parser correctness for real external sources, normalization correctness, factor correctness, legal/compliance interpretation, official carbon accounting correctness, or readiness for production use. -This document records the concepts that future tasks should define before implementation begins. +This document records the concepts that source-acquisition tasks should preserve or refine as implementation evolves. ## Current Baseline diff --git a/docs/source-discovery.md b/docs/source-discovery.md index b0029740..363ef5c1 100644 --- a/docs/source-discovery.md +++ b/docs/source-discovery.md @@ -24,6 +24,47 @@ Python should own the first source discovery tooling. It is practical for early The discovery outputs should inform both implementation paths. The .NET implementation should not depend on Python discovery code at runtime. -## Implementation Boundary +## Python Source-Year Availability Contract -This documentation baseline does not add parser implementation, download logic, or source discovery scripts. Those should be added in later tasks. +The packaged Python ingestion runner asks each source-family adapter for one +target year at a time. Discovery returns a structured +`source_year_available` or `no_available_source_year` result before any +download, parse, insert, or year-state update is attempted. + +When discovery returns `no_available_source_year`, the runner performs a safe +no-op for that source family and year. It does not download an artifact, parse +content, insert records, or advance `source_family_year_states`. + +When discovery returns `source_year_available`, the result includes a +download-ready artifact reference plus metadata such as publication URL, title, +version label, content type, format hint, and the discovery strategy used. + +## Source Family Behavior + +| Source family | Availability behavior | Notes | +| --- | --- | --- | +| DEFRA/DESNZ | Configured artifact URL first; otherwise GOV.UK publication page flat-file link discovery for years in the reviewed availability map. | The default map currently includes `2024` and `2025` GOV.UK publication pages. `2026` is unavailable unless an explicit `source_years.defra_desnz.2026.artifact_url` is configured or the reviewed default map is updated. | +| GHG Protocol | Explicit configured artifact URL required. | This boundary does not assume a stable public year-index for GHG Protocol artifacts. Configure `source_years.ghg_protocol..artifact_url` for local fixtures or reviewed source artifacts. | +| IPCC EFDB | Explicit configured artifact URL required. | This boundary does not assume a stable public year-index artifact contract for IPCC EFDB. Configure `source_years.ipcc_efdb..artifact_url` for local fixtures or reviewed source artifacts. | + +The checked-in local ingestion example config keeps deterministic fixture +behavior for `2024`, `2025`, and `2026` across all three source families by +providing explicit `source_years` artifact entries. + +## DEFRA/DESNZ GOV.UK Discovery + +DEFRA/DESNZ live discovery is intentionally narrow. For a mapped target year, +the adapter reads the configured GOV.UK publication page and selects the first +link whose visible label contains `flat` and whose URL starts with +`https://assets.publishing.service.gov.uk/`. + +Discovery failures return user-readable unavailable metadata with redacted +transport details. They are not treated as successful availability, and they do +not trigger download or downstream ingestion steps. + +## Non-Claims + +This discovery contract does not claim source-owner correctness, factor +correctness, legal correctness, compliance correctness, or production carbon +accounting correctness. It only defines conservative target-year availability +behavior for the current Python ingestion boundary. diff --git a/docs/source-ingestion-boundaries.md b/docs/source-ingestion-boundaries.md index 5e0d7995..a4997fee 100644 --- a/docs/source-ingestion-boundaries.md +++ b/docs/source-ingestion-boundaries.md @@ -119,9 +119,9 @@ Metadata should support idempotency, traceability, and operational review. It sh The Python and .NET implementations are independent implementation options for the same conceptual workflow. -Python is planned first because it is better suited for early source discovery, file handling, spreadsheet inspection, and parser experimentation. +Python is the active first runtime path because it is well suited for source discovery, file handling, spreadsheet inspection, parser experimentation, and the current operator-run ingestion workflow. -The .NET implementation should aim for conceptual parity later. It should not define a different product scope, source boundary, or assurance model. +The .NET implementation should preserve conceptual parity within the reviewed scope and track future parity changes through explicit review tasks. It should not define a different product scope, source boundary, or assurance model. Shared documentation should describe common concepts. Implementation-specific details should stay within `src/python` or `src/dotnet`. diff --git a/docs/task-breakdown.md b/docs/task-breakdown.md index 87634f25..ed8b98f8 100644 --- a/docs/task-breakdown.md +++ b/docs/task-breakdown.md @@ -2,9 +2,9 @@ This task list tracks the documentation and implementation path from CO-001 through CO-033. -## Sprint 1: Documentation Baseline +## Sprint 1: Documentation Foundation -- CO-001: Add project documentation baseline. +- CO-001: Add initial project documentation foundation. - CO-002: Review public wording and README navigation. - CO-003: Add initial release note skeleton. - CO-004: Add examples directory notes for future service files. diff --git a/examples/README.md b/examples/README.md index e69de29b..612e6c88 100644 --- a/examples/README.md +++ b/examples/README.md @@ -0,0 +1,36 @@ +# Examples + +This directory contains deterministic, local-only examples for CarbonOps-Parser. They are intended for reviewers who want to inspect carbon accounting source ingestion boundaries, parser handoffs, normalization summaries, validation issues, and PostgreSQL preview metadata without making network calls or connecting to a database. + +## Fixture Quickstart Entry Point + +The current public quickstart fixture is: + +- [fixtures/defra_desnz_minimal.csv](fixtures/defra_desnz_minimal.csv) - a minimal DEFRA/DESNZ-style CSV fixture used by the local dry-run CLI in the repository README. + +Run it from the repository root after installing the package in editable mode: + +```bash +carbonops-parser local-dry-run \ + --local-path examples/fixtures/defra_desnz_minimal.csv \ + --source-family defra_desnz \ + --source-id defra-desnz-minimal-fixture \ + --content-type text/csv \ + --format-hint csv +``` + +This command is a non-destructive dry run. It does not download source files, call GHG Protocol, DEFRA/DESNZ, or IPCC EFDB endpoints, connect to PostgreSQL, execute SQL, write records, or make production carbon emissions correctness claims. + +## Example Categories + +| Category | Files | Purpose | +| --- | --- | --- | +| Parser and fixture flow | `defra_desnz_parser_usage_example.py`, `fixture_parser_pipeline_example.py`, `example_in_memory_parser_usage.py`, `parser_result_contract_example.py` | Show local parser contracts and fixture-oriented parser behavior. | +| Source acquisition handoff | `example_acquisition_artifact_parser_input_mapping.py`, `local_source_fixture_discovery_example.py`, `parser_input_mapping_example.py` | Show how already-known local source metadata can be mapped toward parser input boundaries. | +| Normalization and summaries | `normalization_contract_example.py`, `parser_normalization_handoff_example.py`, `example_normalization_result_summary_usage.py`, `example_artificial_normalization_executor_usage.py`, `example_artificial_normalization_summary_builder_usage.py` | Show deterministic normalization and summary contracts. | +| Source adapter contracts | `source_adapter_registry_example.py`, `source_adapter_static_configuration_example.py`, `source_adapter_summary_example.py` | Show source adapter registry and metadata helpers. | +| Artificial examples | `example_artificial_fixture_parser_usage.py`, `example_artificial_in_memory_manifest_usage.py`, `example_artificial_source_acquisition_validation_pipeline.py` | Exercise artificial or placeholder source shapes for tests and documentation. These are explicitly not real source integrations. | + +## Placeholder Policy + +Future examples may add GHG Protocol, DEFRA/DESNZ, or IPCC EFDB slices only when a task explicitly scopes them and includes deterministic local fixtures. Placeholder examples must be labelled as placeholders and must not imply live source support, production carbon accounting correctness, compliance correctness, legal correctness, source-owner correctness, or factor correctness. diff --git a/examples/fixtures/ingestion/defra_desnz_2024.csv b/examples/fixtures/ingestion/defra_desnz_2024.csv new file mode 100644 index 00000000..062702de --- /dev/null +++ b/examples/fixtures/ingestion/defra_desnz_2024.csv @@ -0,0 +1,2 @@ +source_year,source_version,category,subcategory,activity,factor_id,factor_name,factor_value,unit,greenhouse_gas,provenance +2024,local-defra-2024,Energy,Electricity,Generated,DEFRA-2024-ELEC,Electricity generated,0.20705,kWh,CO2e,local fixture row 1 diff --git a/examples/fixtures/ingestion/defra_desnz_2025.csv b/examples/fixtures/ingestion/defra_desnz_2025.csv new file mode 100644 index 00000000..9ed09c1a --- /dev/null +++ b/examples/fixtures/ingestion/defra_desnz_2025.csv @@ -0,0 +1,2 @@ +source_year,source_version,category,subcategory,activity,factor_id,factor_name,factor_value,unit,greenhouse_gas,provenance +2025,local-defra-2025,Energy,Electricity,Generated,DEFRA-2025-ELEC,Electricity generated,0.19842,kWh,CO2e,local fixture row 1 diff --git a/examples/fixtures/ingestion/defra_desnz_2026.csv b/examples/fixtures/ingestion/defra_desnz_2026.csv new file mode 100644 index 00000000..63477947 --- /dev/null +++ b/examples/fixtures/ingestion/defra_desnz_2026.csv @@ -0,0 +1,2 @@ +source_year,source_version,category,subcategory,activity,factor_id,factor_name,factor_value,unit,greenhouse_gas,provenance +2026,local-defra-2026,Energy,Electricity,Generated,DEFRA-2026-ELEC,Electricity generated,0.19011,kWh,CO2e,local fixture row 1 diff --git a/examples/fixtures/ingestion/ghg_protocol_2024.csv b/examples/fixtures/ingestion/ghg_protocol_2024.csv new file mode 100644 index 00000000..b828529e --- /dev/null +++ b/examples/fixtures/ingestion/ghg_protocol_2024.csv @@ -0,0 +1,2 @@ +record_type,source_year,source_version,factor_id,factor_name,factor_value,unit,category,subcategory,scope,gas,provenance_note +emission_factor,2024,local-ghg-2024,GHG-2024-ELEC,Grid electricity,0.233,kg CO2e/kWh,Stationary combustion,Electricity,Scope 2,CO2e,local fixture row 1 diff --git a/examples/fixtures/ingestion/ghg_protocol_2025.csv b/examples/fixtures/ingestion/ghg_protocol_2025.csv new file mode 100644 index 00000000..c129e4e5 --- /dev/null +++ b/examples/fixtures/ingestion/ghg_protocol_2025.csv @@ -0,0 +1,2 @@ +record_type,source_year,source_version,factor_id,factor_name,factor_value,unit,category,subcategory,scope,gas,provenance_note +emission_factor,2025,local-ghg-2025,GHG-2025-ELEC,Grid electricity,0.221,kg CO2e/kWh,Stationary combustion,Electricity,Scope 2,CO2e,local fixture row 1 diff --git a/examples/fixtures/ingestion/ghg_protocol_2026.csv b/examples/fixtures/ingestion/ghg_protocol_2026.csv new file mode 100644 index 00000000..7aee8695 --- /dev/null +++ b/examples/fixtures/ingestion/ghg_protocol_2026.csv @@ -0,0 +1,2 @@ +record_type,source_year,source_version,factor_id,factor_name,factor_value,unit,category,subcategory,scope,gas,provenance_note +emission_factor,2026,local-ghg-2026,GHG-2026-ELEC,Grid electricity,0.214,kg CO2e/kWh,Stationary combustion,Electricity,Scope 2,CO2e,local fixture row 1 diff --git a/examples/fixtures/ingestion/ipcc_efdb_2024.csv b/examples/fixtures/ingestion/ipcc_efdb_2024.csv new file mode 100644 index 00000000..5ff22d55 --- /dev/null +++ b/examples/fixtures/ingestion/ipcc_efdb_2024.csv @@ -0,0 +1,2 @@ +record_type,source_year,source_version,factor_id,factor_name,factor_value,unit,category,subcategory,ipcc_sector,gas,region,technology,provenance +emission_factor,2024,local-ipcc-2024,IPCC-2024-ENERGY-CO2,Stationary combustion CO2,56.1,t CO2/TJ,Energy,Stationary combustion,1A,CO2,Global,Default,local fixture row 1 diff --git a/examples/fixtures/ingestion/ipcc_efdb_2025.csv b/examples/fixtures/ingestion/ipcc_efdb_2025.csv new file mode 100644 index 00000000..51fff58c --- /dev/null +++ b/examples/fixtures/ingestion/ipcc_efdb_2025.csv @@ -0,0 +1,2 @@ +record_type,source_year,source_version,factor_id,factor_name,factor_value,unit,category,subcategory,ipcc_sector,gas,region,technology,provenance +emission_factor,2025,local-ipcc-2025,IPCC-2025-ENERGY-CO2,Stationary combustion CO2,55.8,t CO2/TJ,Energy,Stationary combustion,1A,CO2,Global,Default,local fixture row 1 diff --git a/examples/fixtures/ingestion/ipcc_efdb_2026.csv b/examples/fixtures/ingestion/ipcc_efdb_2026.csv new file mode 100644 index 00000000..4677e2cd --- /dev/null +++ b/examples/fixtures/ingestion/ipcc_efdb_2026.csv @@ -0,0 +1,2 @@ +record_type,source_year,source_version,factor_id,factor_name,factor_value,unit,category,subcategory,ipcc_sector,gas,region,technology,provenance +emission_factor,2026,local-ipcc-2026,IPCC-2026-ENERGY-CO2,Stationary combustion CO2,55.4,t CO2/TJ,Energy,Stationary combustion,1A,CO2,Global,Default,local fixture row 1 diff --git a/scripts/agent_dispatch_handoff_reporter.py b/scripts/agent_dispatch_handoff_reporter.py index 33d435bf..8a26775a 100644 --- a/scripts/agent_dispatch_handoff_reporter.py +++ b/scripts/agent_dispatch_handoff_reporter.py @@ -21,6 +21,11 @@ from pathlib import Path from typing import Callable, Sequence +try: + from scripts.agent_task_status import replace_task_status +except ModuleNotFoundError: # pragma: no cover - direct script execution path + from agent_task_status import replace_task_status # type: ignore[no-redef] + DEFAULT_REPOSITORY = "ktalpay/CarbonOps-Parser" BASE_BRANCH = "develop" @@ -126,6 +131,7 @@ class HandoffPackage: safety_constraints: tuple[str, ...] human_action_needed: str artifact_path: Path | None = None + issue_labels: tuple[str, ...] = () @dataclass(frozen=True) @@ -134,6 +140,7 @@ class ClaimResult: removed_label: str added_label: str command: tuple[str, ...] + body_command: tuple[str, ...] = () @dataclass(frozen=True) @@ -697,6 +704,7 @@ def build_handoff_package( human_action_needed=( "Review this handoff report, then start the appropriate local Codex lane task manually." ), + issue_labels=issue.labels, ) @@ -939,6 +947,7 @@ def build_report(outcome: SelectionOutcome) -> str: f"- Selected issue: #{outcome.claim_result.issue_number}", f"- Removed label: `{outcome.claim_result.removed_label}`", f"- Added label: `{outcome.claim_result.added_label}`", + "- Updated issue body `Status:` field to `in-progress`.", "- Lane, agent, and type labels were left unchanged.", "", ) @@ -1106,6 +1115,7 @@ def write_prompt_artifact(package: HandoffPackage, artifact_dir: Path) -> Handof safety_constraints=package.safety_constraints, human_action_needed=package.human_action_needed, artifact_path=artifact_path, + issue_labels=package.issue_labels, ) @@ -1132,6 +1142,7 @@ def locate_or_generate_prompt_artifact( safety_constraints=package.safety_constraints, human_action_needed=package.human_action_needed, artifact_path=artifact_path, + issue_labels=package.issue_labels, ) return write_prompt_artifact(package, artifact_dir) @@ -2217,6 +2228,7 @@ def build_run_once_report(outcome: RunOnceOutcome) -> str: f"- Selected issue: #{outcome.claim_result.issue_number}", f"- Removed label: `{outcome.claim_result.removed_label}`", f"- Added label: `{outcome.claim_result.added_label}`", + "- Updated issue body `Status:` field to `in-progress`.", "- Lane, agent, and type labels were left unchanged.", "", ) @@ -2373,25 +2385,94 @@ def require_issue(issue: Issue | None) -> Issue: return issue +class ClaimStatusClient: + def __init__(self, repository: str, runner: CommandRunner) -> None: + self.repository = repository + self.runner = runner + self.commands: list[tuple[str, ...]] = [] + + def add_label(self, issue_number: int, label: str) -> None: + command = ( + "gh", + "issue", + "edit", + str(issue_number), + "--repo", + self.repository, + "--add-label", + label, + ) + self.commands.append(command) + self.runner(command) + + def remove_label(self, issue_number: int, label: str) -> None: + command = ( + "gh", + "issue", + "edit", + str(issue_number), + "--repo", + self.repository, + "--remove-label", + label, + ) + self.commands.append(command) + self.runner(command) + + def replace_status_labels( + self, + issue_number: int, + old_statuses: Sequence[str], + new_status: str, + ) -> None: + command = ( + "gh", + "issue", + "edit", + str(issue_number), + "--repo", + self.repository, + *tuple( + part + for label in old_statuses + if label != new_status + for part in ("--remove-label", label) + ), + "--add-label", + new_status, + ) + self.commands.append(command) + self.runner(command) + + def edit_body(self, issue_number: int, body: str) -> None: + command = ( + "gh", + "issue", + "edit", + str(issue_number), + "--repo", + self.repository, + "--body", + body, + ) + self.commands.append(command) + self.runner(command) + + def claim_selected_issue( repository: str, package: HandoffPackage, runner: CommandRunner = subprocess_runner, ) -> ClaimResult: - command = ( - "gh", - "issue", - "edit", - str(package.selected_issue_number), - "--repo", - repository, - "--remove-label", - "status:ready", - "--add-label", - "status:in-progress", - ) + client = ClaimStatusClient(repository, runner) try: - runner(command) + replace_task_status( + client, + issue_number=package.selected_issue_number, + labels=package.issue_labels or ("status:ready",), + body=package.issue_body, + new_status="status:in-progress", + ) except subprocess.CalledProcessError as exc: stderr = exc.stderr.strip() if exc.stderr else "" detail = stderr or str(exc) @@ -2407,7 +2488,8 @@ def claim_selected_issue( issue_number=package.selected_issue_number, removed_label="status:ready", added_label="status:in-progress", - command=command, + command=client.commands[0], + body_command=client.commands[-1] if len(client.commands) > 1 else (), ) diff --git a/scripts/agent_task_status.py b/scripts/agent_task_status.py new file mode 100644 index 00000000..943390f5 --- /dev/null +++ b/scripts/agent_task_status.py @@ -0,0 +1,125 @@ +"""Shared task issue status mutation helpers for local agent automation.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol, Sequence + + +SUPPORTED_TASK_STATUS_LABELS = ( + "status:blocked", + "status:ready", + "status:in-progress", + "status:merged", + "status:needs-attention", +) + + +class TaskStatusClient(Protocol): + def add_label(self, issue_number: int, label: str) -> None: ... + + def remove_label(self, issue_number: int, label: str) -> None: ... + + def edit_body(self, issue_number: int, body: str) -> None: ... + + +@dataclass(frozen=True) +class TaskStatusReplacement: + issue_number: int + old_statuses: tuple[str, ...] + new_status: str + old_body: str + new_body: str + + +def status_labels(labels: Sequence[str]) -> tuple[str, ...]: + return tuple(label for label in labels if label.startswith("status:")) + + +def bare_status(status_label: str) -> str: + if status_label not in SUPPORTED_TASK_STATUS_LABELS: + raise ValueError(f"Unsupported status label: {status_label}") + return status_label.removeprefix("status:") + + +def sync_status_body(body: str, status_value: str) -> str: + lines = body.splitlines() + keep_trailing_newline = body.endswith("\n") + replacement_line = f"Status: {status_value}" + + for index, line in enumerate(lines): + if line.lstrip().lower().startswith("status:"): + existing_indent = line[: len(line) - len(line.lstrip())] + lines[index] = f"{existing_indent}{replacement_line}" + updated = "\n".join(lines) + return updated + ("\n" if keep_trailing_newline else "") + + insert_at = status_insert_index(lines) + lines.insert(insert_at, replacement_line) + updated = "\n".join(lines) + return updated + ("\n" if keep_trailing_newline else "") + + +def status_insert_index(lines: Sequence[str]) -> int: + if not lines: + return 0 + + task_id_index: int | None = None + lane_index: int | None = None + last_metadata_index: int | None = None + for index, line in enumerate(lines): + stripped = line.strip() + if not stripped: + break + if ":" not in stripped: + break + key = stripped.split(":", 1)[0].strip().lower() + last_metadata_index = index + if key in {"task id", "task-id"}: + task_id_index = index + elif key == "lane": + lane_index = index + + if lane_index is not None: + return lane_index + 1 + if task_id_index is not None: + return task_id_index + 1 + if last_metadata_index is not None: + return last_metadata_index + 1 + return 0 + + +def replace_task_status( + client: TaskStatusClient, + *, + issue_number: int, + labels: Sequence[str], + body: str, + new_status: str, +) -> TaskStatusReplacement: + status_value = bare_status(new_status) + old_statuses = status_labels(labels) + new_body = sync_status_body(body, status_value) + + replace_labels = getattr(client, "replace_status_labels", None) + if callable(replace_labels): + if old_statuses != (new_status,): + replace_labels(issue_number, old_statuses, new_status) + else: + if new_status not in labels: + client.add_label(issue_number, new_status) + + for label in old_statuses: + if label != new_status: + client.remove_label(issue_number, label) + + if new_body != body: + client.edit_body(issue_number, new_body) + + return TaskStatusReplacement( + issue_number=issue_number, + old_statuses=old_statuses, + new_status=new_status, + old_body=body, + new_body=new_body, + ) diff --git a/scripts/agent_task_watcher.py b/scripts/agent_task_watcher.py new file mode 100644 index 00000000..4dae146b --- /dev/null +++ b/scripts/agent_task_watcher.py @@ -0,0 +1,464 @@ +#!/usr/bin/env python3 +"""Process merged task PRs and maintain deterministic task status labels.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from typing import Callable, Sequence + +try: + from scripts.agent_task_status import ( + SUPPORTED_TASK_STATUS_LABELS, + TaskStatusReplacement, + replace_task_status, + ) +except ModuleNotFoundError: # pragma: no cover - direct script execution path + from agent_task_status import ( # type: ignore[no-redef] + SUPPORTED_TASK_STATUS_LABELS, + TaskStatusReplacement, + replace_task_status, + ) + +SUPPORTED_STATUS_LABELS = SUPPORTED_TASK_STATUS_LABELS + +TASK_ID_PATTERN = re.compile(r"\b([A-Za-z]+-[0-9][0-9A-Za-z-]*)\b") +TASK_ID_FOOTER_PATTERN = re.compile( + r"^[ \t]*Task-ID:[ \t]*([A-Za-z]+-[0-9][0-9A-Za-z-]*)[ \t]*$", + re.IGNORECASE | re.MULTILINE, +) +TASK_ISSUE_FOOTER_PATTERN = re.compile( + r"^[ \t]*Task-Issue:[ \t]*#?([0-9]+)[ \t]*$", + re.IGNORECASE | re.MULTILINE, +) + + +CommandRunner = Callable[[Sequence[str]], str] + + +@dataclass(frozen=True) +class Issue: + number: int + title: str + body: str + labels: tuple[str, ...] + state: str = "OPEN" + + +@dataclass(frozen=True) +class PullRequest: + number: int + title: str + body: str + merged: bool + + +StatusReplacement = TaskStatusReplacement + + +@dataclass(frozen=True) +class WatcherResult: + task_id: str + task_issue_number: int + task_status: StatusReplacement + updated_downstream: tuple[StatusReplacement, ...] + skipped_downstream: tuple[str, ...] + + +class WatcherError(RuntimeError): + """Raised for expected watcher failures with clear diagnostics.""" + + +class CommandError(WatcherError): + def __init__(self, command: Sequence[str], returncode: int, message: str) -> None: + super().__init__(f"Command failed ({returncode}): {' '.join(command)}\n{message}") + + +def run_command(command: Sequence[str]) -> str: + completed = subprocess.run( + list(command), + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + message = completed.stderr.strip() or completed.stdout.strip() + raise CommandError(command, completed.returncode, message) + return completed.stdout + + +def labels_from_gh(raw_labels: object) -> tuple[str, ...]: + if not isinstance(raw_labels, list): + return () + labels: list[str] = [] + for raw_label in raw_labels: + if isinstance(raw_label, dict) and isinstance(raw_label.get("name"), str): + labels.append(raw_label["name"]) + elif isinstance(raw_label, str): + labels.append(raw_label) + return tuple(labels) + + +def parse_issue(raw_issue: dict[str, object]) -> Issue: + return Issue( + number=int(raw_issue["number"]), + title=str(raw_issue.get("title") or ""), + body=str(raw_issue.get("body") or ""), + labels=labels_from_gh(raw_issue.get("labels")), + state=str(raw_issue.get("state") or "OPEN"), + ) + + +def parse_pr(raw_pr: dict[str, object]) -> PullRequest: + merged_at = str(raw_pr.get("mergedAt") or "") + merged_value = raw_pr.get("merged") + return PullRequest( + number=int(raw_pr["number"]), + title=str(raw_pr.get("title") or ""), + body=str(raw_pr.get("body") or ""), + merged=bool(merged_value) or bool(merged_at), + ) + + +class GitHubClient: + def __init__(self, repo: str, runner: CommandRunner = run_command) -> None: + self.repo = repo + self.runner = runner + + def view_pr(self, pr_number: int) -> PullRequest: + output = self.runner( + ( + "gh", + "pr", + "view", + str(pr_number), + "--repo", + self.repo, + "--json", + "number,title,body,mergedAt", + ) + ) + try: + raw_pr = json.loads(output) + except json.JSONDecodeError as exc: + raise WatcherError(f"gh pr view returned invalid JSON for PR #{pr_number}.") from exc + if not isinstance(raw_pr, dict): + raise WatcherError(f"gh pr view returned unexpected JSON for PR #{pr_number}.") + return parse_pr(raw_pr) + + def view_issue(self, issue_number: int) -> Issue: + output = self.runner( + ( + "gh", + "issue", + "view", + str(issue_number), + "--repo", + self.repo, + "--json", + "number,title,body,labels,state", + ) + ) + try: + raw_issue = json.loads(output) + except json.JSONDecodeError as exc: + raise WatcherError(f"gh issue view returned invalid JSON for issue #{issue_number}.") from exc + if not isinstance(raw_issue, dict): + raise WatcherError(f"gh issue view returned unexpected JSON for issue #{issue_number}.") + return parse_issue(raw_issue) + + def find_issue_for_task(self, task_id: str) -> int | None: + output = self.runner( + ( + "gh", + "issue", + "list", + "--repo", + self.repo, + "--state", + "all", + "--search", + f"{task_id} in:title", + "--json", + "number,title", + "--limit", + "100", + ) + ) + try: + raw_issues = json.loads(output) + except json.JSONDecodeError as exc: + raise WatcherError(f"gh issue list returned invalid JSON while resolving {task_id}.") from exc + if not isinstance(raw_issues, list): + raise WatcherError(f"gh issue list returned unexpected JSON while resolving {task_id}.") + prefix = f"[{task_id}]" + matches = [ + int(issue["number"]) + for issue in raw_issues + if isinstance(issue, dict) and str(issue.get("title") or "").startswith(prefix) + ] + if len(matches) == 1: + return matches[0] + return None + + def add_label(self, issue_number: int, label: str) -> None: + self.runner(("gh", "issue", "edit", str(issue_number), "--repo", self.repo, "--add-label", label)) + + def remove_label(self, issue_number: int, label: str) -> None: + self.runner(("gh", "issue", "edit", str(issue_number), "--repo", self.repo, "--remove-label", label)) + + def edit_body(self, issue_number: int, body: str) -> None: + self.runner(("gh", "issue", "edit", str(issue_number), "--repo", self.repo, "--body", body)) + + def comment(self, issue_number: int, body: str) -> None: + self.runner(("gh", "issue", "comment", str(issue_number), "--repo", self.repo, "--body", body)) + + +def parse_task_id(value: str) -> str | None: + footer_matches = TASK_ID_FOOTER_PATTERN.findall(value) + if footer_matches: + return footer_matches[-1].upper() + title_match = TASK_ID_PATTERN.search(value) + if title_match: + return title_match.group(1).upper() + return None + + +def parse_task_issue_number(value: str) -> int | None: + footer_matches = TASK_ISSUE_FOOTER_PATTERN.findall(value) + if footer_matches: + return int(footer_matches[-1]) + return None + + +def extract_field(field_name: str, body: str) -> str | None: + needle = f"{field_name}:".lower() + for raw_line in body.splitlines(): + line = raw_line.strip() + if line.lower().startswith(needle): + return line[len(needle) :].strip() + return None + + +def parse_task_list(raw_value: str | None) -> tuple[str, ...]: + if raw_value is None: + return () + task_ids: list[str] = [] + for raw_part in raw_value.split(","): + value = raw_part.strip() + if not value or value.lower() in {"none", "n/a"} or value == "-": + continue + task_ids.append(value.upper()) + return tuple(task_ids) + + +def replace_status_label(client: GitHubClient, issue: Issue, new_status: str) -> StatusReplacement: + try: + return replace_task_status( + client, + issue_number=issue.number, + labels=issue.labels, + body=issue.body, + new_status=new_status, + ) + except ValueError as exc: + raise WatcherError(str(exc)) from exc + + +def resolve_task_issue(client: GitHubClient, pull_request: PullRequest) -> tuple[str, Issue]: + body_task_id = parse_task_id(pull_request.body) + title_task_id = parse_task_id(pull_request.title) + task_issue_number = parse_task_issue_number(pull_request.body) + + if task_issue_number is not None: + issue = client.view_issue(task_issue_number) + task_id = body_task_id or parse_task_id(issue.body) or parse_task_id(issue.title) or title_task_id + else: + task_id = body_task_id or title_task_id + if task_id is None: + raise WatcherError( + f"PR #{pull_request.number} has no Task-Issue footer and no Task-ID/title task id; " + "no task issue could be resolved." + ) + task_issue_number = client.find_issue_for_task(task_id) + if task_issue_number is None: + raise WatcherError( + f"PR #{pull_request.number} task {task_id} did not resolve to one unique issue." + ) + issue = client.view_issue(task_issue_number) + + if task_id is None: + raise WatcherError( + f"PR #{pull_request.number} resolved issue #{issue.number}, but no Task-ID was found." + ) + return task_id, issue + + +def evaluate_downstream( + client: GitHubClient, + source_issue: Issue, + source_task_id: str, + pr_number: int, +) -> tuple[tuple[StatusReplacement, ...], tuple[str, ...]]: + updated: list[StatusReplacement] = [] + skipped: list[str] = [] + + for dependent_task_id in parse_task_list(extract_field("Unblocks", source_issue.body)): + dependent_issue_number = client.find_issue_for_task(dependent_task_id) + if dependent_issue_number is None: + skipped.append( + f"`{dependent_task_id}` skipped: no unique issue title beginning " + f"with `[{dependent_task_id}]` was found." + ) + continue + + dependent_issue = client.view_issue(dependent_issue_number) + depends_on = extract_field("Depends on", dependent_issue.body) + if depends_on is None: + replacement = replace_status_label(client, dependent_issue, "status:needs-attention") + updated.append(replacement) + skipped.append( + f"`{dependent_task_id}` (#{dependent_issue_number}) needs attention: " + "missing `Depends on:` metadata." + ) + continue + + missing_dependencies: list[str] = [] + unresolved_dependencies: list[str] = [] + for dependency_task_id in parse_task_list(depends_on): + dependency_issue_number = client.find_issue_for_task(dependency_task_id) + if dependency_issue_number is None: + unresolved_dependencies.append(dependency_task_id) + continue + dependency_issue = client.view_issue(dependency_issue_number) + if "status:merged" not in dependency_issue.labels: + missing_dependencies.append(dependency_task_id) + + if unresolved_dependencies: + unresolved = ", ".join(f"`{task_id}`" for task_id in unresolved_dependencies) + replacement = replace_status_label(client, dependent_issue, "status:needs-attention") + updated.append(replacement) + skipped.append( + f"`{dependent_task_id}` (#{dependent_issue_number}) needs attention: " + f"dependencies did not resolve to unique issues: {unresolved}." + ) + continue + + if missing_dependencies: + missing = ", ".join(f"`{task_id}`" for task_id in missing_dependencies) + skipped.append( + f"`{dependent_task_id}` (#{dependent_issue_number}) skipped: " + f"dependencies not merged or unresolved: {missing}." + ) + continue + + if "status:merged" in dependent_issue.labels: + skipped.append(f"`{dependent_task_id}` (#{dependent_issue_number}) skipped: already `status:merged`.") + continue + + replacement = replace_status_label(client, dependent_issue, "status:ready") + updated.append(replacement) + client.comment( + dependent_issue_number, + f"Agent Task Watcher changed status from {format_statuses(replacement.old_statuses)} " + f"to `status:ready` because `{source_task_id}` merged in PR #{pr_number} " + "and all declared dependencies are `status:merged`.", + ) + + return tuple(updated), tuple(skipped) + + +def format_statuses(statuses: Sequence[str]) -> str: + if not statuses: + return "`status:missing`" + return ", ".join(f"`{status}`" for status in statuses) + + +def build_comment(result: WatcherResult, pr_number: int) -> str: + lines = [ + "Agent Task Watcher updated task status.", + "", + f"Detected task: `{result.task_id}`", + f"Merged PR: #{pr_number}", + f"Task issue status: {format_statuses(result.task_status.old_statuses)} -> `{result.task_status.new_status}`", + "", + "Downstream task update summary:", + ] + if result.updated_downstream: + lines.append("") + lines.append("Updated:") + for replacement in result.updated_downstream: + lines.append( + f"- #{replacement.issue_number}: " + f"{format_statuses(replacement.old_statuses)} -> `{replacement.new_status}`" + ) + if result.skipped_downstream: + lines.append("") + lines.append("Skipped:") + for skipped in result.skipped_downstream: + lines.append(f"- {skipped}") + if not result.updated_downstream and not result.skipped_downstream: + lines.append("") + lines.append("No dependent tasks were declared in `Unblocks:`.") + return "\n".join(lines) + + +def process_merged_pr(client: GitHubClient, pr_number: int) -> WatcherResult | None: + pull_request = client.view_pr(pr_number) + if not pull_request.merged: + return None + + task_id, issue = resolve_task_issue(client, pull_request) + task_status = replace_status_label(client, issue, "status:merged") + refreshed_issue = client.view_issue(issue.number) + updated_downstream, skipped_downstream = evaluate_downstream( + client, + refreshed_issue, + task_id, + pull_request.number, + ) + result = WatcherResult( + task_id=task_id, + task_issue_number=issue.number, + task_status=task_status, + updated_downstream=updated_downstream, + skipped_downstream=skipped_downstream, + ) + client.comment(issue.number, build_comment(result, pull_request.number)) + return result + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", required=True) + parser.add_argument("--pr-number", required=True, type=int) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + client = GitHubClient(args.repo) + try: + result = process_merged_pr(client, args.pr_number) + except WatcherError as exc: + print(f"Agent Task Watcher error: {exc}", file=sys.stderr) + return 1 + if result is None: + print(f"PR #{args.pr_number} is not merged. No action taken.") + return 0 + ready_count = sum( + 1 for replacement in result.updated_downstream if replacement.new_status == "status:ready" + ) + print( + f"Processed PR #{args.pr_number}: task {result.task_id} issue #{result.task_issue_number} " + f"marked status:merged; {ready_count} ready downstream update(s)." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/install_local_agent_launchd.sh b/scripts/install_local_agent_launchd.sh new file mode 100755 index 00000000..bdfee31f --- /dev/null +++ b/scripts/install_local_agent_launchd.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +set -euo pipefail + +DEFAULT_LABEL="local.carbonops.agent.supervisor" +DEFAULT_INTERVAL_SECONDS="600" +DEFAULT_SAFE_LOG_DIR="${HOME}/FutureOps/Agents/CarbonOps-Parser/.logs" +DEFAULT_LAUNCHD_PATH="/Users/oxygen/.npm-global/bin:/usr/local/bin:/usr/local/opt/python@3.12/libexec/bin:/usr/bin:/bin:/usr/sbin:/sbin" +PLIST_PYTHON="${PLIST_PYTHON:-/usr/bin/python3}" + +usage() { + cat <<'EOF' +Usage: scripts/install_local_agent_launchd.sh [options] + +Install a user-level launchd job for scripts/local_agent_supervisor.py. + +Options: + --config Local agent supervisor JSON config path. + --repo-root Repository root containing scripts/local_agent_supervisor.py. + --python-bin Python executable used by launchd. + --interval-seconds launchd StartInterval. Default: 600. + --label launchd label. Default: local.carbonops.agent.supervisor. + --dry-run Print the planned plist and launchctl command without writing or loading. + -h, --help Show this help. +EOF +} + +die() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +absolute_path() { + local raw="$1" + if [[ "$raw" == /* ]]; then + printf '%s\n' "$raw" + elif [[ "$raw" == "~" ]]; then + printf '%s\n' "$HOME" + elif [[ "$raw" == "~/"* ]]; then + printf '%s/%s\n' "$HOME" "${raw#~/}" + else + printf '%s/%s\n' "$(pwd)" "$raw" + fi +} + +repo_root="$(pwd)" +config_path="" +python_bin="python3" +interval_seconds="$DEFAULT_INTERVAL_SECONDS" +label="$DEFAULT_LABEL" +dry_run=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --config) + [[ $# -ge 2 ]] || die "--config requires a path" + config_path="$2" + shift 2 + ;; + --repo-root) + [[ $# -ge 2 ]] || die "--repo-root requires a path" + repo_root="$2" + shift 2 + ;; + --python-bin) + [[ $# -ge 2 ]] || die "--python-bin requires a path" + python_bin="$2" + shift 2 + ;; + --interval-seconds) + [[ $# -ge 2 ]] || die "--interval-seconds requires a value" + interval_seconds="$2" + shift 2 + ;; + --label) + [[ $# -ge 2 ]] || die "--label requires a launchd label" + label="$2" + shift 2 + ;; + --dry-run) + dry_run=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ "$interval_seconds" =~ ^[1-9][0-9]*$ ]] || die "--interval-seconds must be a positive integer" +[[ -n "$label" ]] || die "--label must be non-empty" +[[ "$label" != */* ]] || die "--label must not contain /" + +repo_root="$(absolute_path "$repo_root")" +if [[ -z "$config_path" ]]; then + config_path="${repo_root}/.agent/local-agent.json" +else + config_path="$(absolute_path "$config_path")" +fi +if [[ "$python_bin" != */* ]]; then + resolved_python_bin="$(command -v "$python_bin" || true)" + [[ -n "$resolved_python_bin" ]] || die "python executable not found on PATH: $python_bin" + python_bin="$resolved_python_bin" +else + python_bin="$(absolute_path "$python_bin")" +fi + +supervisor_path="${repo_root}/scripts/local_agent_supervisor.py" +plist_path="${HOME}/Library/LaunchAgents/${label}.plist" + +if [[ "$dry_run" -eq 0 ]]; then + [[ -f "$supervisor_path" ]] || die "supervisor script not found: $supervisor_path" + [[ -f "$config_path" ]] || die "config file not found: $config_path" + [[ -x "$python_bin" ]] || die "python executable not found or not executable: $python_bin" +fi + +log_dir="$DEFAULT_SAFE_LOG_DIR" +if ! command -v "$PLIST_PYTHON" >/dev/null 2>&1; then + PLIST_PYTHON="$(command -v python3 || true)" +fi +[[ -n "$PLIST_PYTHON" ]] || die "python3 is required to render the launchd plist" + +if [[ -f "$config_path" ]]; then + config_log_dir="$( + "$PLIST_PYTHON" - "$config_path" <<'PY' +from __future__ import annotations + +import json +import sys +from pathlib import Path + +config_path = Path(sys.argv[1]).expanduser() +try: + raw = json.loads(config_path.read_text(encoding="utf-8")) +except Exception: + raise SystemExit(0) + +if not isinstance(raw, dict): + raise SystemExit(0) + +value = raw.get("log_directory") or raw.get("agents_log_directory") +if value is None and isinstance(raw.get("agents_root"), str) and raw["agents_root"].strip(): + value = str(Path(raw["agents_root"]).expanduser() / ".logs") +if not isinstance(value, str) or not value.strip(): + raise SystemExit(0) + +path = Path(value).expanduser() +if not path.is_absolute(): + path = config_path.parent / path +print(path) +PY + )" + if [[ -n "$config_log_dir" ]]; then + log_dir="$config_log_dir" + fi +fi + +stdout_path="${log_dir}/local-agent-supervisor.launchd.out.log" +stderr_path="${log_dir}/local-agent-supervisor.launchd.err.log" + +render_plist() { + "$PLIST_PYTHON" - \ + "$label" \ + "$python_bin" \ + "$supervisor_path" \ + "$config_path" \ + "$interval_seconds" \ + "$stdout_path" \ + "$stderr_path" \ + "$DEFAULT_LAUNCHD_PATH" <<'PY' +from __future__ import annotations + +import plistlib +import sys + +label, python_bin, supervisor_path, config_path, interval, stdout_path, stderr_path, launchd_path = sys.argv[1:] +plist = { + "Label": label, + "EnvironmentVariables": { + "PATH": launchd_path, + }, + "ProgramArguments": [ + python_bin, + supervisor_path, + "--config", + config_path, + "--once", + ], + "RunAtLoad": True, + "StartInterval": int(interval), + "StandardOutPath": stdout_path, + "StandardErrorPath": stderr_path, + "WorkingDirectory": supervisor_path.rsplit("/scripts/local_agent_supervisor.py", 1)[0], +} +sys.stdout.buffer.write(plistlib.dumps(plist, sort_keys=True)) +PY +} + +if [[ "$dry_run" -eq 1 ]]; then + printf 'Would write plist: %s\n' "$plist_path" + printf 'Would create log directory: %s\n' "$log_dir" + render_plist + printf '\nWould run: launchctl bootstrap gui/%s %s\n' "$(id -u)" "$plist_path" + exit 0 +fi + +mkdir -p "${HOME}/Library/LaunchAgents" "$log_dir" +render_plist > "$plist_path" +chmod 0644 "$plist_path" + +launchctl bootstrap "gui/$(id -u)" "$plist_path" + +printf 'Installed launchd job: %s\n' "$label" +printf 'Plist: %s\n' "$plist_path" +printf 'Logs: %s\n' "$log_dir" diff --git a/scripts/local_agent_supervisor.py b/scripts/local_agent_supervisor.py new file mode 100755 index 00000000..70a5b528 --- /dev/null +++ b/scripts/local_agent_supervisor.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python3 +"""Local agent supervisor for one-shot unattended task dispatch. + +The supervisor first looks for one pull request that needs local-agent fixes. +If none exists, it scans GitHub issue labels, selects at most one ready issue, +and delegates execution to the appropriate local runner. It intentionally does +not run Codex directly and has no merge, approval, issue-closing, scheduler, or +watch behavior. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable, Sequence + + +READY_LABEL = "status:ready" +IN_PROGRESS_LABEL = "status:in-progress" +PR_CHANGES_REQUESTED_LABEL = "pr:changes-requested" +PR_FIX_REQUEST_TOKEN = "@local-agent fix" +VALIDATION_MODES = ("minimal", "python", "dotnet", "ops", "full") + + +CommandRunner = Callable[[Sequence[str], str | None, Path | None], str] + + +class SupervisorError(RuntimeError): + """Raised for expected supervisor failures with clear messages.""" + + +class CommandError(SupervisorError): + """Raised when a subprocess command fails.""" + + def __init__(self, command: Sequence[str], returncode: int, message: str) -> None: + joined = " ".join(command) + super().__init__(f"Command failed ({returncode}): {joined}\n{message}") + self.command = tuple(command) + self.returncode = returncode + + +@dataclass(frozen=True) +class SupervisorConfig: + repo: str + source_root: Path + agents_root: Path + base: str = "develop" + validation_mode: str = "minimal" + python_bin: str | None = None + runner_script_path: Path | None = None + log_directory: Path | None = None + lock_path: Path | None = None + + +@dataclass(frozen=True) +class Issue: + number: int + title: str + labels: tuple[str, ...] + state: str = "OPEN" + + +@dataclass(frozen=True) +class PullRequest: + number: int + title: str + labels: tuple[str, ...] + state: str = "OPEN" + merged: bool = False + comments: tuple[str, ...] = () + + +class FileLock: + def __init__(self, path: Path) -> None: + self.path = path + self._fd: int | None = None + self.acquired = False + + def __enter__(self) -> "FileLock": + self.path.parent.mkdir(parents=True, exist_ok=True) + try: + self._fd = os.open( + str(self.path), + os.O_CREAT | os.O_EXCL | os.O_WRONLY, + 0o600, + ) + except FileExistsError: + return self + os.write(self._fd, f"pid={os.getpid()}\n".encode("utf-8")) + self.acquired = True + return self + + def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: + if self._fd is not None: + os.close(self._fd) + self._fd = None + if self.acquired: + try: + self.path.unlink() + except FileNotFoundError: + pass + + +def run_command(command: Sequence[str], stdin: str | None = None, cwd: Path | None = None) -> str: + completed = subprocess.run( + list(command), + cwd=str(cwd) if cwd else None, + input=stdin, + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + message = completed.stderr.strip() or completed.stdout.strip() + raise CommandError(command, completed.returncode, message) + return completed.stdout + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path, help="Path to local supervisor JSON config.") + parser.add_argument( + "--dry-run", + action="store_true", + help="Print the selected dispatch without invoking the runner.", + ) + parser.add_argument("--once", action="store_true", help="Run one queue scan and dispatch at most one task.") + return parser.parse_args(argv) + + +def _config_path(raw: object, key: str, config_dir: Path) -> Path | None: + if raw is None: + return None + if not isinstance(raw, str) or not raw.strip(): + raise SupervisorError(f"Config value {key!r} must be a non-empty string.") + path = Path(raw).expanduser() + if not path.is_absolute(): + path = config_dir / path + return path + + +def load_config(path: Path) -> SupervisorConfig: + config_path = path.expanduser() + try: + raw = json.loads(config_path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise SupervisorError(f"Config file not found: {config_path}") from exc + except json.JSONDecodeError as exc: + raise SupervisorError(f"Config file is not valid JSON: {config_path}") from exc + if not isinstance(raw, dict): + raise SupervisorError("Supervisor config must be a JSON object.") + + config_dir = config_path.parent + repo = raw.get("repo") + if not isinstance(repo, str) or not repo.strip(): + raise SupervisorError("Config value 'repo' is required.") + + source_root = _config_path(raw.get("source_root"), "source_root", config_dir) + agents_root = _config_path(raw.get("agents_root"), "agents_root", config_dir) + if source_root is None: + raise SupervisorError("Config value 'source_root' is required.") + if agents_root is None: + raise SupervisorError("Config value 'agents_root' is required.") + + base = raw.get("base_branch", raw.get("base", "develop")) + if not isinstance(base, str) or not base.strip(): + raise SupervisorError("Config value 'base_branch' must be a non-empty string.") + + validation_mode = raw.get("validation_mode", "minimal") + if validation_mode not in VALIDATION_MODES: + allowed = ", ".join(VALIDATION_MODES) + raise SupervisorError(f"Config value 'validation_mode' must be one of: {allowed}.") + + python_bin = raw.get("python_bin") + if python_bin is not None and (not isinstance(python_bin, str) or not python_bin.strip()): + raise SupervisorError("Config value 'python_bin' must be a non-empty string when provided.") + + return SupervisorConfig( + repo=repo.strip(), + source_root=source_root, + agents_root=agents_root, + base=base.strip(), + validation_mode=validation_mode, + python_bin=python_bin, + runner_script_path=_config_path(raw.get("runner_script_path"), "runner_script_path", config_dir), + log_directory=_config_path(raw.get("log_directory"), "log_directory", config_dir), + lock_path=_config_path(raw.get("lock_path"), "lock_path", config_dir), + ) + + +def labels_from_gh(raw_labels: object) -> tuple[str, ...]: + labels: list[str] = [] + if not isinstance(raw_labels, list): + return () + for label in raw_labels: + if isinstance(label, dict) and isinstance(label.get("name"), str): + labels.append(label["name"]) + elif isinstance(label, str): + labels.append(label) + return tuple(labels) + + +def comments_from_gh(raw_comments: object) -> tuple[str, ...]: + comments: list[str] = [] + if not isinstance(raw_comments, list): + return () + for comment in raw_comments: + if isinstance(comment, dict) and isinstance(comment.get("body"), str): + comments.append(comment["body"]) + elif isinstance(comment, str): + comments.append(comment) + return tuple(comments) + + +def parse_issue(raw_issue: dict[str, object]) -> Issue: + return Issue( + number=int(raw_issue["number"]), + title=str(raw_issue.get("title") or ""), + labels=labels_from_gh(raw_issue.get("labels")), + state=str(raw_issue.get("state") or "OPEN"), + ) + + +def parse_pull_request(raw_pr: dict[str, object]) -> PullRequest: + return PullRequest( + number=int(raw_pr["number"]), + title=str(raw_pr.get("title") or ""), + labels=labels_from_gh(raw_pr.get("labels")), + state=str(raw_pr.get("state") or "OPEN"), + merged=bool(raw_pr.get("merged") or raw_pr.get("mergedAt")), + comments=comments_from_gh(raw_pr.get("comments")), + ) + + +def issue_list_command(repo: str, label: str) -> tuple[str, ...]: + return ( + "gh", + "issue", + "list", + "--repo", + repo, + "--state", + "open", + "--label", + label, + "--json", + "number,title,labels,state", + "--limit", + "200", + ) + + +def pr_list_command(repo: str) -> tuple[str, ...]: + return ( + "gh", + "pr", + "list", + "--repo", + repo, + "--state", + "open", + "--json", + "number,title,labels,state,mergedAt,comments", + "--limit", + "100", + ) + + +def list_issues(repo: str, label: str, runner: CommandRunner) -> tuple[Issue, ...]: + output = runner(issue_list_command(repo, label), None, None) + try: + raw_issues = json.loads(output) + except json.JSONDecodeError as exc: + raise SupervisorError(f"gh issue list returned invalid JSON for label {label!r}.") from exc + if not isinstance(raw_issues, list): + raise SupervisorError(f"gh issue list returned unexpected JSON for label {label!r}.") + return tuple(sorted((parse_issue(issue) for issue in raw_issues), key=lambda issue: issue.number)) + + +def list_pull_requests(repo: str, runner: CommandRunner) -> tuple[PullRequest, ...]: + output = runner(pr_list_command(repo), None, None) + try: + raw_prs = json.loads(output) + except json.JSONDecodeError as exc: + raise SupervisorError("gh pr list returned invalid JSON.") from exc + if not isinstance(raw_prs, list): + raise SupervisorError("gh pr list returned unexpected JSON.") + return tuple(sorted((parse_pull_request(pr) for pr in raw_prs), key=lambda pr: pr.number)) + + +def pr_fix_requested(pr: PullRequest) -> bool: + if PR_CHANGES_REQUESTED_LABEL in pr.labels: + return True + return any(PR_FIX_REQUEST_TOKEN in comment.lower() for comment in pr.comments) + + +def select_pr_for_fix(repo: str, runner: CommandRunner) -> PullRequest | None: + for pr in list_pull_requests(repo, runner): + if pr.state.upper() != "OPEN" or pr.merged: + continue + if pr_fix_requested(pr): + return pr + return None + + +def source_root_is_dirty(source_root: Path, runner: CommandRunner) -> bool: + output = runner(("git", "-C", str(source_root), "status", "--porcelain"), None, None) + return bool(output.strip()) + + +def fast_forward_base_if_possible(source_root: Path, base: str, runner: CommandRunner) -> bool: + try: + runner(("git", "-C", str(source_root), "fetch", "origin", base), None, None) + current = runner(("git", "-C", str(source_root), "branch", "--show-current"), None, None).strip() + if current == base: + runner(("git", "-C", str(source_root), "merge", "--ff-only", f"origin/{base}"), None, None) + else: + runner(("git", "-C", str(source_root), "fetch", "origin", f"{base}:{base}"), None, None) + except SupervisorError as exc: + print(f"Supervisor: could not fast-forward {base} from origin: {exc}") + return False + return True + + +def default_lock_path(config: SupervisorConfig) -> Path: + if config.lock_path is not None: + return config.lock_path.expanduser() + try: + return config.agents_root.expanduser() / ".local-agent-supervisor.lock" + except RuntimeError: + return Path(tempfile.gettempdir()) / "carbonops-local-agent-supervisor.lock" + + +def runner_script_path(config: SupervisorConfig) -> Path: + if config.runner_script_path is not None: + return config.runner_script_path.expanduser() + return config.source_root.expanduser() / "scripts" / "local_codex_task_runner.py" + + +def pr_fix_runner_script_path(config: SupervisorConfig) -> Path: + return config.source_root.expanduser() / "scripts" / "local_codex_pr_fix_runner.py" + + +def log_directory(config: SupervisorConfig) -> Path: + if config.log_directory is not None: + return config.log_directory.expanduser() + return config.agents_root.expanduser() / ".logs" + + +def runner_command(config: SupervisorConfig, issue_number: int) -> tuple[str, ...]: + command = [ + sys.executable, + str(runner_script_path(config)), + "--repo", + config.repo, + "--source-root", + str(config.source_root.expanduser()), + "--agents-root", + str(config.agents_root.expanduser()), + "--base", + config.base, + "--run-once", + "--issue-number", + str(issue_number), + "--validation-mode", + config.validation_mode, + ] + if config.python_bin: + command.extend(("--python-bin", config.python_bin)) + return tuple(command) + + +def pr_fix_runner_command(config: SupervisorConfig, pr_number: int) -> tuple[str, ...]: + command = [ + sys.executable, + str(pr_fix_runner_script_path(config)), + "--repo", + config.repo, + "--agents-root", + str(config.agents_root.expanduser()), + "--once", + "--pr-number", + str(pr_number), + "--validation-mode", + config.validation_mode, + ] + if config.python_bin: + command.extend(("--python-bin", config.python_bin)) + return tuple(command) + + +def run_runner(command: Sequence[str], config: SupervisorConfig) -> Path: + logs = log_directory(config) + logs.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + if "--issue-number" in command: + target_kind = "issue" + target_number = command[command.index("--issue-number") + 1] + runner_name = "local_codex_task_runner.py" + elif "--pr-number" in command: + target_kind = "pr" + target_number = command[command.index("--pr-number") + 1] + runner_name = "local_codex_pr_fix_runner.py" + else: + target_kind = "runner" + target_number = "unknown" + runner_name = Path(command[1]).name if len(command) > 1 else "runner" + log_path = logs / f"local-agent-supervisor-{target_kind}-{target_number}-{timestamp}.log" + with log_path.open("w", encoding="utf-8") as log_file: + log_file.write(f"$ {' '.join(command)}\n\n") + completed = subprocess.run( + list(command), + cwd=str(config.source_root.expanduser()), + text=True, + stdout=log_file, + stderr=subprocess.STDOUT, + check=False, + ) + if completed.returncode != 0: + print(f"Supervisor: {runner_name} failed with exit code {completed.returncode}; log: {log_path}") + raise SupervisorError(f"{runner_name} failed with exit code {completed.returncode}; log: {log_path}") + return log_path + + +def execute(args: argparse.Namespace, runner: CommandRunner = run_command) -> int: + if not args.once: + raise SupervisorError("Refusing to run without --once; scheduler and watch modes are not supported.") + + config = load_config(args.config) + lock_path = default_lock_path(config) + with FileLock(lock_path) as lock: + if not lock.acquired: + print(f"Supervisor: lock already held at {lock_path}; exiting.") + return 0 + + print(f"Supervisor: cycle start; scanning {config.repo}.") + if source_root_is_dirty(config.source_root.expanduser(), runner): + print(f"Supervisor: source root is dirty; no dispatch: {config.source_root.expanduser()}") + raise SupervisorError(f"Source root has uncommitted changes: {config.source_root.expanduser()}") + + fast_forward_base_if_possible(config.source_root.expanduser(), config.base, runner) + + pr = select_pr_for_fix(config.repo, runner) + if pr is not None: + print(f"Supervisor: selected PR fix dispatch: #{pr.number} {pr.title}") + command = pr_fix_runner_command(config, pr.number) + if args.dry_run: + print("Supervisor: dry run; runner was not invoked.") + print(f"Supervisor: planned command: {' '.join(command)}") + return 0 + + log_path = run_runner(command, config) + print(f"Supervisor: PR fix runner completed; log: {log_path}") + return 0 + + in_progress = list_issues(config.repo, IN_PROGRESS_LABEL, runner) + print(f"Supervisor: in-progress issue count: {len(in_progress)}") + if in_progress: + issue_list = ", ".join(f"#{issue.number} {issue.title}" for issue in in_progress) + print(f"Supervisor: found in-progress issue(s); no dispatch: {issue_list}") + return 0 + + ready = list_issues(config.repo, READY_LABEL, runner) + print(f"Supervisor: ready issue count: {len(ready)}") + if not ready: + print("Supervisor: no-op; no ready issues found.") + return 0 + + selected = ready[0] + print(f"Supervisor: selected issue dispatch: #{selected.number} {selected.title}") + command = runner_command(config, selected.number) + if args.dry_run: + print("Supervisor: dry run; runner was not invoked.") + print(f"Supervisor: planned command: {' '.join(command)}") + return 0 + + log_path = run_runner(command, config) + print(f"Supervisor: runner completed; log: {log_path}") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + try: + return execute(parse_args(argv)) + except SupervisorError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/local_codex_pr_fix_runner.py b/scripts/local_codex_pr_fix_runner.py new file mode 100644 index 00000000..62d6ba8b --- /dev/null +++ b/scripts/local_codex_pr_fix_runner.py @@ -0,0 +1,476 @@ +#!/usr/bin/env python3 +"""Local Codex PR fix runner. + +This script selects one open pull request that asks the local agent for fixes, +checks out the PR head branch in a deterministic worktree, runs Codex once from +a generated prompt, validates the result, and pushes a normal fix commit back to +the same PR branch. It intentionally has no merge, approval, close, branch +deletion, worktree deletion, force-push, or daemon behavior. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Sequence + + +CHANGES_REQUESTED_LABEL = "pr:changes-requested" +FIX_REQUEST_TOKEN = "@local-agent fix" +VALIDATION_MODES = ("minimal", "python", "dotnet", "ops", "full") + +CommandRunner = Callable[[Sequence[str], str | None, Path | None], str] + + +@dataclass(frozen=True) +class PullRequest: + number: int + title: str + body: str + labels: tuple[str, ...] + state: str + merged: bool + head_branch: str + base_branch: str + comments: tuple[str, ...] = () + files: tuple[str, ...] = () + + +@dataclass(frozen=True) +class FixPlan: + pr: PullRequest + worktree_path: Path + prompt_path: Path + fix_comment: str | None + + +class RunnerError(RuntimeError): + """Raised for expected runner failures with clear user-facing messages.""" + + +class CommandError(RunnerError): + """Raised when a subprocess command fails.""" + + def __init__(self, command: Sequence[str], returncode: int, message: str) -> None: + self.command = tuple(command) + self.returncode = returncode + joined = " ".join(command) + super().__init__(f"Command failed ({returncode}): {joined}\n{message}") + + +class ValidationError(RunnerError): + """Raised when validation fails.""" + + def __init__(self, command: Sequence[str], cause: RunnerError) -> None: + self.command = tuple(command) + super().__init__(str(cause)) + + +def run_command(command: Sequence[str], stdin: str | None = None, cwd: Path | None = None) -> str: + completed = subprocess.run( + list(command), + cwd=str(cwd) if cwd else None, + input=stdin, + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + message = completed.stderr.strip() or completed.stdout.strip() + raise CommandError(command, completed.returncode, message) + return completed.stdout + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", required=True, help="GitHub repository, for example owner/name.") + parser.add_argument("--agents-root", required=True, type=Path, help="Root directory for PR worktrees.") + parser.add_argument( + "--validation-mode", + choices=VALIDATION_MODES, + default="minimal", + help="Validation profile to run after Codex completes. Defaults to minimal.", + ) + parser.add_argument("--python-bin", default="python", help="Python executable for Python validation.") + parser.add_argument("--pr-number", type=int, help="Select one exact open pull request number.") + parser.add_argument("--dry-run", action="store_true", help="Print the plan without mutating state.") + parser.add_argument("--once", action="store_true", help="Run exactly one PR fix.") + return parser.parse_args(argv) + + +def labels_from_gh(raw_labels: object) -> tuple[str, ...]: + labels: list[str] = [] + if not isinstance(raw_labels, list): + return () + for label in raw_labels: + if isinstance(label, dict) and isinstance(label.get("name"), str): + labels.append(label["name"]) + elif isinstance(label, str): + labels.append(label) + return tuple(labels) + + +def comments_from_gh(raw_comments: object) -> tuple[str, ...]: + if not isinstance(raw_comments, list): + return () + comments: list[str] = [] + for comment in raw_comments: + if isinstance(comment, dict): + body = comment.get("body") + if isinstance(body, str): + comments.append(body) + elif isinstance(comment, str): + comments.append(comment) + return tuple(comments) + + +def files_from_gh(raw_files: object) -> tuple[str, ...]: + if not isinstance(raw_files, list): + return () + files: list[str] = [] + for file_entry in raw_files: + if isinstance(file_entry, dict): + path = file_entry.get("path") + if isinstance(path, str): + files.append(path) + elif isinstance(file_entry, str): + files.append(file_entry) + return tuple(files) + + +def parse_pr(raw_pr: dict[str, object]) -> PullRequest: + return PullRequest( + number=int(raw_pr["number"]), + title=str(raw_pr.get("title") or ""), + body=str(raw_pr.get("body") or ""), + labels=labels_from_gh(raw_pr.get("labels")), + state=str(raw_pr.get("state") or "OPEN"), + merged=bool(raw_pr.get("merged") or raw_pr.get("mergedAt")), + head_branch=str(raw_pr.get("headRefName") or ""), + base_branch=str(raw_pr.get("baseRefName") or ""), + comments=comments_from_gh(raw_pr.get("comments")), + files=files_from_gh(raw_pr.get("files")), + ) + + +def pr_json_fields() -> str: + return "number,title,body,labels,state,mergedAt,headRefName,baseRefName,comments,files" + + +def list_open_prs(repo: str, runner: CommandRunner) -> tuple[PullRequest, ...]: + output = runner( + ("gh", "pr", "list", "--repo", repo, "--state", "open", "--json", pr_json_fields(), "--limit", "100"), + None, + None, + ) + try: + raw_prs = json.loads(output) + except json.JSONDecodeError as exc: + raise RunnerError("gh pr list returned invalid JSON.") from exc + if not isinstance(raw_prs, list): + raise RunnerError("gh pr list returned unexpected JSON.") + return tuple(sorted((parse_pr(pr) for pr in raw_prs), key=lambda pr: pr.number)) + + +def get_pr(repo: str, pr_number: int, runner: CommandRunner) -> PullRequest: + output = runner( + ("gh", "pr", "view", str(pr_number), "--repo", repo, "--json", pr_json_fields()), + None, + None, + ) + try: + raw_pr = json.loads(output) + except json.JSONDecodeError as exc: + raise RunnerError(f"gh pr view returned invalid JSON for PR #{pr_number}.") from exc + if not isinstance(raw_pr, dict): + raise RunnerError(f"gh pr view returned unexpected JSON for PR #{pr_number}.") + pr = parse_pr(raw_pr) + ensure_open_unmerged(pr) + return pr + + +def latest_fix_comment(pr: PullRequest) -> str | None: + for comment in reversed(pr.comments): + if FIX_REQUEST_TOKEN in comment.lower(): + return comment + return None + + +def is_fix_requested(pr: PullRequest) -> bool: + return CHANGES_REQUESTED_LABEL in pr.labels or latest_fix_comment(pr) is not None + + +def ensure_open_unmerged(pr: PullRequest) -> None: + if pr.state.upper() != "OPEN": + raise RunnerError(f"PR #{pr.number} is not open; refusing to apply local-agent fixes.") + if pr.merged: + raise RunnerError(f"PR #{pr.number} is merged; refusing to apply local-agent fixes.") + if not pr.head_branch: + raise RunnerError(f"PR #{pr.number} has no head branch.") + if not pr.base_branch: + raise RunnerError(f"PR #{pr.number} has no base branch.") + + +def select_pr(args: argparse.Namespace, runner: CommandRunner) -> PullRequest: + if args.pr_number is not None: + pr = get_pr(args.repo, args.pr_number, runner) + if not is_fix_requested(pr): + raise RunnerError( + f"PR #{pr.number} does not have label {CHANGES_REQUESTED_LABEL!r} " + f"or comment token {FIX_REQUEST_TOKEN!r}; refusing to apply local-agent fixes." + ) + return pr + + candidates = [] + for pr in list_open_prs(args.repo, runner): + if pr.state.upper() != "OPEN" or pr.merged: + continue + ensure_open_unmerged(pr) + if is_fix_requested(pr): + candidates.append(pr) + if not candidates: + raise RunnerError( + f"No open PRs found with label {CHANGES_REQUESTED_LABEL!r} or comment token {FIX_REQUEST_TOKEN!r}." + ) + return candidates[0] + + +def build_plan(pr: PullRequest, agents_root: Path) -> FixPlan: + root = agents_root.expanduser() + return FixPlan( + pr=pr, + worktree_path=root / f"PR-{pr.number}", + prompt_path=root / ".handoff" / f"PR-{pr.number}" / "prompt.md", + fix_comment=latest_fix_comment(pr), + ) + + +def generate_prompt(plan: FixPlan, repo: str) -> str: + pr = plan.pr + body = pr.body.strip() or "(No PR body provided.)" + fix_comment = plan.fix_comment.strip() if plan.fix_comment else "(No fix request comment found.)" + changed_files = "\n".join(f"- {path}" for path in pr.files) if pr.files else "- (Unavailable.)" + return "\n".join( + ( + f"# Codex PR Fix #{pr.number}", + "", + f"Repository: {repo}", + f"Pull Request: #{pr.number}", + f"Title: {pr.title}", + f"Head branch: {pr.head_branch}", + f"Base branch: {pr.base_branch}", + "", + "Apply only the requested fixes for this open pull request.", + "", + "Safety constraints:", + "- Do not merge pull requests.", + "- Do not approve pull requests.", + "- Do not close pull requests.", + "- Do not close issues.", + "- Do not delete branches.", + "- Do not delete worktrees.", + "- Do not force push.", + "- Do not modify unrelated PRs.", + "- Do not apply fixes to merged/closed PRs.", + "- Do not add production credentials.", + "- Do not execute destructive database operations.", + "", + "PR body:", + "", + body, + "", + "Fix request comment:", + "", + fix_comment, + "", + "Changed files:", + "", + changed_files, + "", + ) + ) + + +def print_plan(plan: FixPlan, repo: str) -> None: + print(f"Selected PR: #{plan.pr.number} {plan.pr.title}") + print(f"Head branch: {plan.pr.head_branch}") + print(f"Base branch: {plan.pr.base_branch}") + print(f"Worktree: {plan.worktree_path}") + print(f"Prompt: {plan.prompt_path}") + print(f"Repository: {repo}") + + +def current_branch(worktree_path: Path, runner: CommandRunner) -> str: + return runner(("git", "-C", str(worktree_path), "branch", "--show-current"), None, None).strip() + + +def prepare_worktree(plan: FixPlan, runner: CommandRunner) -> None: + pr = plan.pr + runner(("git", "fetch", "origin", pr.base_branch), None, None) + if plan.worktree_path.exists(): + runner(("git", "fetch", "origin", pr.head_branch), None, None) + if current_branch(plan.worktree_path, runner) != pr.head_branch: + runner(("git", "-C", str(plan.worktree_path), "switch", pr.head_branch), None, None) + runner(("git", "-C", str(plan.worktree_path), "pull", "--ff-only", "origin", pr.head_branch), None, None) + return + + plan.worktree_path.parent.mkdir(parents=True, exist_ok=True) + runner(("git", "fetch", "origin", f"{pr.head_branch}:{pr.head_branch}"), None, None) + runner(("git", "worktree", "add", str(plan.worktree_path), pr.head_branch), None, None) + + +def write_prompt(plan: FixPlan, prompt: str) -> None: + plan.prompt_path.parent.mkdir(parents=True, exist_ok=True) + plan.prompt_path.write_text(prompt, encoding="utf-8") + + +def run_codex(plan: FixPlan, prompt: str, runner: CommandRunner) -> None: + print("Running: codex exec --sandbox workspace-write < prompt.md") + runner(("codex", "exec", "--sandbox", "workspace-write"), prompt, plan.worktree_path) + + +def validation_commands(plan: FixPlan, mode: str, python_bin: str) -> tuple[tuple[str, ...], ...]: + commands: list[tuple[str, ...]] = [] + if mode in ("python", "full"): + commands.append((python_bin, "-m", "pytest")) + if (plan.worktree_path / "scripts" / "check_public_safety.py").exists(): + commands.append((python_bin, "scripts/check_public_safety.py")) + if mode in ("dotnet", "full"): + if (plan.worktree_path / "src" / "dotnet" / "CarbonOps.Parser.sln").exists(): + commands.append(("dotnet", "test", "src/dotnet/CarbonOps.Parser.sln", "--no-restore")) + if mode in ("ops", "full"): + test_path = plan.worktree_path / "tests" / "test_local_codex_pr_fix_runner.py" + if test_path.exists(): + commands.append((python_bin, "-m", "pytest", "-q", "tests/test_local_codex_pr_fix_runner.py")) + commands.append(("git", "-C", str(plan.worktree_path), "diff", "--check")) + return tuple(commands) + + +def run_validation(plan: FixPlan, mode: str, python_bin: str, runner: CommandRunner) -> None: + for command in validation_commands(plan, mode, python_bin): + print(f"Running validation: {' '.join(command)}") + try: + runner(command, None, plan.worktree_path if command[0] != "git" else None) + except RunnerError as exc: + raise ValidationError(command, exc) from exc + + +def has_changes(plan: FixPlan, runner: CommandRunner) -> bool: + output = runner(("git", "-C", str(plan.worktree_path), "status", "--porcelain"), None, None) + return bool(output.strip()) + + +def commit_changes(plan: FixPlan, runner: CommandRunner) -> str | None: + if not has_changes(plan, runner): + return None + runner(("git", "-C", str(plan.worktree_path), "add", "-A"), None, None) + runner(("git", "-C", str(plan.worktree_path), "commit", "-m", f"Fix PR #{plan.pr.number} requested changes"), None, None) + return runner(("git", "-C", str(plan.worktree_path), "rev-parse", "HEAD"), None, None).strip() + + +def push_branch(plan: FixPlan, runner: CommandRunner) -> None: + runner(("git", "-C", str(plan.worktree_path), "push", "origin", plan.pr.head_branch), None, None) + + +def add_pr_comment(repo: str, pr_number: int, body: str, runner: CommandRunner) -> None: + runner(("gh", "pr", "comment", str(pr_number), "--repo", repo, "--body", body), None, None) + + +def success_comment(commit_hash: str, validation_mode: str) -> str: + return "\n".join( + ( + "Local-agent fix completed.", + "", + f"- commit: {commit_hash}", + f"- validation-mode: {validation_mode}", + "- result: pushed to the PR head branch", + ) + ) + + +def no_changes_comment(validation_mode: str) -> str: + return "\n".join( + ( + "Local-agent fix completed with no changes.", + "", + f"- validation-mode: {validation_mode}", + "- result: no commit or push was needed", + ) + ) + + +def validation_failure_comment(plan: FixPlan, validation_mode: str, failed_command: Sequence[str]) -> str: + return "\n".join( + ( + "Local-agent fix validation failed; no push was performed.", + "", + f"- validation-mode: {validation_mode}", + f"- failed-command: {' '.join(failed_command)}", + f"- worktree: {plan.worktree_path}", + "", + "Recovery:", + f"- cd {plan.worktree_path}", + f"- {' '.join(failed_command)}", + "- fix validation failures", + "- git diff --check", + ) + ) + + +def execute(args: argparse.Namespace, runner: CommandRunner = run_command) -> int: + if not args.once: + raise RunnerError("Refusing to run without --once; daemon/watch modes are not supported.") + + pr = select_pr(args, runner) + plan = build_plan(pr, args.agents_root) + prompt = generate_prompt(plan, args.repo) + print_plan(plan, args.repo) + + if args.dry_run: + print("Dry run: no worktrees, files, Codex, validation, commits, pushes, or comments were changed.") + print("Planned commands:") + print(f"- git fetch origin {pr.base_branch}") + print(f"- git fetch origin {pr.head_branch}:{pr.head_branch}") + print(f"- git worktree add {plan.worktree_path} {pr.head_branch}") + print("- codex exec --sandbox workspace-write < prompt.md") + for command in validation_commands(plan, args.validation_mode, args.python_bin): + print(f"- {' '.join(command)}") + print(f"- git push origin {pr.head_branch}") + print(f"- gh pr comment {pr.number}") + return 0 + + prepare_worktree(plan, runner) + write_prompt(plan, prompt) + run_codex(plan, prompt, runner) + try: + run_validation(plan, args.validation_mode, args.python_bin, runner) + except ValidationError as exc: + add_pr_comment(args.repo, pr.number, validation_failure_comment(plan, args.validation_mode, exc.command), runner) + raise + + commit_hash = commit_changes(plan, runner) + if commit_hash is None: + add_pr_comment(args.repo, pr.number, no_changes_comment(args.validation_mode), runner) + print("No changes detected; skipping commit and push.") + return 0 + + push_branch(plan, runner) + add_pr_comment(args.repo, pr.number, success_comment(commit_hash, args.validation_mode), runner) + print(f"Committed and pushed: {commit_hash}") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + try: + return execute(parse_args(argv)) + except RunnerError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/local_codex_task_runner.py b/scripts/local_codex_task_runner.py new file mode 100644 index 00000000..e798967c --- /dev/null +++ b/scripts/local_codex_task_runner.py @@ -0,0 +1,671 @@ +#!/usr/bin/env python3 +"""Local Codex one-shot task runner. + +This script claims one ready GitHub issue, prepares a deterministic local +worktree, runs Codex once from a generated prompt, validates the result, commits +changes, pushes the task branch, and opens a pull request. It intentionally has +no daemon, scheduler, merge, approval, issue-closing, branch-deletion, or +worktree-deletion behavior. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shlex +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Sequence + + +READY_LABEL = "status:ready" +IN_PROGRESS_LABEL = "status:in-progress" +TASK_ID_PATTERN = re.compile(r"\b([A-Za-z]+-\d+)\b") +BRACKETED_TASK_PATTERN = re.compile(r"\[([A-Za-z]+-\d+)\]") +SAFE_SEGMENT_PATTERN = re.compile(r"[^a-z0-9]+") +PR_CREATE_MAX_ATTEMPTS = 3 +PR_CREATE_BACKOFF_SECONDS = (0.1, 0.2) +RETRYABLE_PR_ERROR_PATTERN = re.compile( + r"\b(502|503|504|timeout|timed out|temporar(?:y|ily)|network|api gateway|bad gateway|" + r"gateway timeout|service unavailable|connection reset|connection refused|connection aborted|" + r"tls handshake timeout|i/o timeout)\b", + re.IGNORECASE, +) + + +CommandRunner = Callable[[Sequence[str], str | None, Path | None], str] +SleepFn = Callable[[float], None] +VALIDATION_MODES = ("minimal", "python", "dotnet", "ops", "full") + + +@dataclass(frozen=True) +class Issue: + number: int + title: str + body: str + labels: tuple[str, ...] + state: str = "OPEN" + + +@dataclass(frozen=True) +class TaskPlan: + issue: Issue + task_id: str + slug: str + branch: str + worktree_path: Path + prompt_path: Path + base: str + + +class RunnerError(RuntimeError): + """Raised for expected runner failures with clear user-facing messages.""" + + +class CommandError(RunnerError): + """Raised when a subprocess command fails.""" + + def __init__(self, command: Sequence[str], returncode: int, message: str) -> None: + self.command = tuple(command) + self.returncode = returncode + joined = " ".join(command) + super().__init__(f"Command failed ({returncode}): {joined}\n{message}") + + +class ValidationError(RunnerError): + """Raised when validation fails after task claim.""" + + def __init__(self, command: Sequence[str], cause: RunnerError) -> None: + self.command = tuple(command) + super().__init__(str(cause)) + + +class PrCreationError(RunnerError): + """Raised when PR creation fails after the branch was pushed.""" + + +def run_command( + command: Sequence[str], + stdin: str | None = None, + cwd: Path | None = None, +) -> str: + """Run one command and return stdout, raising a clear error on failure.""" + + completed = subprocess.run( + list(command), + cwd=str(cwd) if cwd else None, + input=stdin, + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + message = completed.stderr.strip() or completed.stdout.strip() + raise CommandError(command, completed.returncode, message) + return completed.stdout + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", required=True, help="GitHub repository, for example owner/name.") + parser.add_argument("--source-root", required=True, type=Path, help="Path to the source repository.") + parser.add_argument("--agents-root", required=True, type=Path, help="Root directory for task worktrees.") + parser.add_argument("--base", default="develop", help="Base branch for the worktree and PR.") + parser.add_argument("--run-once", action="store_true", help="Run exactly one ready task.") + parser.add_argument("--dry-run", action="store_true", help="Print the plan without mutating state.") + parser.add_argument( + "--override-in-progress", + action="store_true", + help="Allow running when another open issue is status:in-progress.", + ) + parser.add_argument("--issue-number", type=int, help="Select one exact open issue number.") + parser.add_argument("--task-id", help="Select the open issue whose extracted task id matches this value.") + parser.add_argument( + "--validation-mode", + choices=VALIDATION_MODES, + default="minimal", + help="Validation profile to run after Codex completes. Defaults to minimal.", + ) + parser.add_argument( + "--python-bin", + default="python", + help="Python executable used by Python validation commands.", + ) + return parser.parse_args(argv) + + +def labels_from_gh(raw_labels: object) -> tuple[str, ...]: + labels: list[str] = [] + if not isinstance(raw_labels, list): + return () + for label in raw_labels: + if isinstance(label, dict) and isinstance(label.get("name"), str): + labels.append(label["name"]) + elif isinstance(label, str): + labels.append(label) + return tuple(labels) + + +def parse_issue(raw_issue: dict[str, object]) -> Issue: + return Issue( + number=int(raw_issue["number"]), + title=str(raw_issue.get("title") or ""), + body=str(raw_issue.get("body") or ""), + labels=labels_from_gh(raw_issue.get("labels")), + state=str(raw_issue.get("state") or "OPEN"), + ) + + +def issue_list_command(repo: str, label: str | None = None) -> tuple[str, ...]: + command = [ + "gh", + "issue", + "list", + "--repo", + repo, + "--state", + "open", + ] + if label is not None: + command.extend(("--label", label)) + command.extend(("--json", "number,title,body,labels,state", "--limit", "200")) + return tuple(command) + + +def list_issues(repo: str, label: str | None, runner: CommandRunner) -> tuple[Issue, ...]: + output = runner(issue_list_command(repo, label), None, None) + try: + raw_issues = json.loads(output) + except json.JSONDecodeError as exc: + selector = f" for label {label!r}" if label is not None else "" + raise RunnerError(f"gh issue list returned invalid JSON{selector}.") from exc + if not isinstance(raw_issues, list): + selector = f" for label {label!r}" if label is not None else "" + raise RunnerError(f"gh issue list returned unexpected JSON{selector}.") + return tuple(sorted((parse_issue(issue) for issue in raw_issues), key=lambda issue: issue.number)) + + +def get_issue(repo: str, issue_number: int, runner: CommandRunner) -> Issue: + output = runner( + ( + "gh", + "issue", + "view", + str(issue_number), + "--repo", + repo, + "--json", + "number,title,body,labels,state", + ), + None, + None, + ) + try: + raw_issue = json.loads(output) + except json.JSONDecodeError as exc: + raise RunnerError(f"gh issue view returned invalid JSON for issue #{issue_number}.") from exc + if not isinstance(raw_issue, dict): + raise RunnerError(f"gh issue view returned unexpected JSON for issue #{issue_number}.") + issue = parse_issue(raw_issue) + if issue.state.upper() != "OPEN": + raise RunnerError(f"Selected issue #{issue.number} is not open.") + return issue + + +def select_issue(args: argparse.Namespace, runner: CommandRunner) -> Issue: + if args.issue_number is not None and args.task_id: + raise RunnerError("Use only one explicit selector: --issue-number or --task-id.") + + if args.issue_number is not None: + issue = get_issue(args.repo, args.issue_number, runner) + elif args.task_id: + requested_task_id = sanitize_task_id(args.task_id) + matches = [ + issue + for issue in list_issues(args.repo, None, runner) + if extract_task_id(issue) == requested_task_id + ] + if not matches: + raise RunnerError(f"No open issue found for task id {requested_task_id}.") + if len(matches) > 1: + issue_list = ", ".join(f"#{issue.number}" for issue in matches) + raise RunnerError(f"Multiple open issues match task id {requested_task_id}: {issue_list}.") + issue = matches[0] + else: + ready = list_issues(args.repo, READY_LABEL, runner) + if not ready: + raise RunnerError(f"No open issues labeled {READY_LABEL} were found.") + issue = ready[0] + + if READY_LABEL not in issue.labels: + raise RunnerError( + f"Selected issue #{issue.number} ({extract_task_id(issue)}) is not labeled {READY_LABEL}; " + "refusing to claim a non-ready task." + ) + return issue + + +def extract_task_id(issue: Issue) -> str: + bracket_match = BRACKETED_TASK_PATTERN.search(issue.title) + if bracket_match: + return sanitize_task_id(bracket_match.group(1)) + body_match = TASK_ID_PATTERN.search(issue.body) + if body_match: + return sanitize_task_id(body_match.group(1)) + title_match = TASK_ID_PATTERN.search(issue.title) + if title_match: + return sanitize_task_id(title_match.group(1)) + return f"ISSUE-{issue.number}" + + +def sanitize_task_id(value: str) -> str: + parts = re.findall(r"[A-Za-z0-9]+", value) + if len(parts) >= 2: + return f"{parts[0].upper()}-{parts[1]}" + return SAFE_SEGMENT_PATTERN.sub("-", value.lower()).strip("-").upper() + + +def slugify(value: str, fallback: str = "task") -> str: + without_task_id = BRACKETED_TASK_PATTERN.sub("", value) + slug = SAFE_SEGMENT_PATTERN.sub("-", without_task_id.lower()).strip("-") + slug = re.sub(r"-{2,}", "-", slug) + return (slug or fallback)[:64].strip("-") or fallback + + +def build_plan(issue: Issue, agents_root: Path, base: str) -> TaskPlan: + task_id = extract_task_id(issue) + task_segment = task_id.lower() + slug = slugify(issue.title) + branch = f"feature/{task_segment}-{slug}" + worktree_path = agents_root.expanduser() / task_id + prompt_path = agents_root.expanduser() / ".handoff" / task_id / "prompt.md" + return TaskPlan( + issue=issue, + task_id=task_id, + slug=slug, + branch=branch, + worktree_path=worktree_path, + prompt_path=prompt_path, + base=base, + ) + + +def generate_prompt(plan: TaskPlan, repo: str) -> str: + body = plan.issue.body.strip() or "(No issue body provided.)" + return "\n".join( + ( + f"# Codex Task {plan.task_id}", + "", + f"Repository: {repo}", + f"Issue: #{plan.issue.number}", + f"Title: {plan.issue.title}", + f"Branch: {plan.branch}", + "", + "Implement only the task described in this issue.", + "", + "Safety constraints:", + "- Do not merge pull requests.", + "- Do not approve pull requests.", + "- Do not close issues.", + "- Do not delete branches.", + "- Do not delete worktrees.", + "- Do not claim unrelated product tasks.", + "", + "Issue body:", + "", + body, + "", + ) + ) + + +def print_plan(plan: TaskPlan, repo: str) -> None: + print(f"Selected issue: #{plan.issue.number} {plan.issue.title}") + print(f"Task ID: {plan.task_id}") + print(f"Branch: {plan.branch}") + print(f"Worktree: {plan.worktree_path}") + print(f"Prompt: {plan.prompt_path}") + print(f"Base: {plan.base}") + print(f"Repository: {repo}") + + +def claim_issue(repo: str, issue: Issue, runner: CommandRunner) -> None: + runner( + ( + "gh", + "issue", + "edit", + str(issue.number), + "--repo", + repo, + "--remove-label", + READY_LABEL, + "--add-label", + IN_PROGRESS_LABEL, + ), + None, + None, + ) + + +def local_branch_exists(source_root: Path, branch: str, runner: CommandRunner) -> bool: + try: + runner(("git", "-C", str(source_root), "rev-parse", "--verify", f"refs/heads/{branch}"), None, None) + except RunnerError: + return False + return True + + +def current_branch(worktree_path: Path, runner: CommandRunner) -> str: + return runner(("git", "-C", str(worktree_path), "branch", "--show-current"), None, None).strip() + + +def prepare_worktree(source_root: Path, plan: TaskPlan, runner: CommandRunner) -> None: + if plan.worktree_path.exists(): + if current_branch(plan.worktree_path, runner) != plan.branch: + runner(("git", "-C", str(plan.worktree_path), "switch", plan.branch), None, None) + return + + plan.worktree_path.parent.mkdir(parents=True, exist_ok=True) + if local_branch_exists(source_root, plan.branch, runner): + runner(("git", "-C", str(source_root), "worktree", "add", str(plan.worktree_path), plan.branch), None, None) + return + runner( + ("git", "-C", str(source_root), "worktree", "add", "-b", plan.branch, str(plan.worktree_path), plan.base), + None, + None, + ) + + +def write_prompt(plan: TaskPlan, prompt: str) -> None: + plan.prompt_path.parent.mkdir(parents=True, exist_ok=True) + plan.prompt_path.write_text(prompt, encoding="utf-8") + + +def run_codex(plan: TaskPlan, prompt: str, runner: CommandRunner) -> None: + print("Running: codex exec --sandbox workspace-write < prompt.md") + runner(("codex", "exec", "--sandbox", "workspace-write"), prompt, plan.worktree_path) + + +def validation_commands(plan: TaskPlan, mode: str, python_bin: str) -> tuple[tuple[str, ...], ...]: + diff_check = ("git", "-C", str(plan.worktree_path), "diff", "--check") + commands: list[tuple[str, ...]] = [] + + if mode in ("python", "full"): + commands.append((python_bin, "-m", "pytest")) + if (plan.worktree_path / "scripts" / "check_public_safety.py").exists(): + commands.append((python_bin, "scripts/check_public_safety.py")) + + if mode in ("dotnet", "full"): + sln_path = plan.worktree_path / "src" / "dotnet" / "CarbonOps.Parser.sln" + if sln_path.exists(): + commands.append(("dotnet", "test", "src/dotnet/CarbonOps.Parser.sln", "--no-restore")) + + if mode in ("ops", "full"): + test_path = plan.worktree_path / "tests" / "test_local_codex_task_runner.py" + if test_path.exists(): + commands.append((python_bin, "-m", "pytest", "-q", "tests/test_local_codex_task_runner.py")) + + commands.append(diff_check) + return tuple(commands) + + +def run_validation(plan: TaskPlan, mode: str, python_bin: str, runner: CommandRunner) -> None: + for command in validation_commands(plan, mode, python_bin): + print(f"Running validation: {' '.join(command)}") + try: + runner(command, None, plan.worktree_path if command[0] != "git" else None) + except RunnerError as exc: + raise ValidationError(command, exc) from exc + + +def print_validation_recovery(plan: TaskPlan, failed_command: Sequence[str]) -> None: + print("Validation failed after issue claim; leaving issue status unchanged.", file=sys.stderr) + print(f"Issue number: #{plan.issue.number}", file=sys.stderr) + print(f"Task ID: {plan.task_id}", file=sys.stderr) + print(f"Branch: {plan.branch}", file=sys.stderr) + print(f"Worktree path: {plan.worktree_path}", file=sys.stderr) + print(f"Failed command: {' '.join(failed_command)}", file=sys.stderr) + print("Suggested manual commands:", file=sys.stderr) + print(f"- cd {plan.worktree_path}", file=sys.stderr) + print(f"- {' '.join(failed_command)}", file=sys.stderr) + print("- fix validation failures", file=sys.stderr) + print("- git diff --check", file=sys.stderr) + + +def has_changes(plan: TaskPlan, runner: CommandRunner) -> bool: + output = runner(("git", "-C", str(plan.worktree_path), "status", "--porcelain"), None, None) + return bool(output.strip()) + + +def commit_changes(plan: TaskPlan, runner: CommandRunner) -> str | None: + if not has_changes(plan, runner): + print("No changes detected; skipping commit.") + return None + runner(("git", "-C", str(plan.worktree_path), "add", "-A"), None, None) + runner( + ( + "git", + "-C", + str(plan.worktree_path), + "commit", + "-m", + f"[{plan.task_id}] {plan.issue.title}", + ), + None, + None, + ) + return runner(("git", "-C", str(plan.worktree_path), "rev-parse", "HEAD"), None, None).strip() + + +def push_branch(plan: TaskPlan, runner: CommandRunner) -> None: + runner(("git", "-C", str(plan.worktree_path), "push", "-u", "origin", plan.branch), None, None) + + +def build_pr_body(plan: TaskPlan, validation_mode: str) -> str: + validation_lines = [f"- validation-mode: {validation_mode}"] + footer = f"Task-ID: {plan.task_id}\nTask-Issue: #{plan.issue.number}" + return "\n".join( + ( + f"Implements {plan.task_id}.", + "", + "Validation:", + *validation_lines, + "", + footer, + ) + ) + + +def pr_create_command(repo: str, plan: TaskPlan, validation_mode: str) -> tuple[str, ...]: + return ( + "gh", + "pr", + "create", + "--repo", + repo, + "--base", + plan.base, + "--head", + plan.branch, + "--title", + f"[{plan.task_id}] {plan.issue.title}", + "--body", + build_pr_body(plan, validation_mode), + ) + + +def manual_pr_create_command(repo: str, plan: TaskPlan, validation_mode: str) -> str: + return " ".join(shlex.quote(part) for part in pr_create_command(repo, plan, validation_mode)) + + +def existing_pr_url_for_branch(repo: str, branch: str, runner: CommandRunner) -> str | None: + output = runner( + ( + "gh", + "pr", + "list", + "--repo", + repo, + "--head", + branch, + "--state", + "open", + "--json", + "url", + "--limit", + "1", + ), + None, + None, + ) + try: + raw_prs = json.loads(output) + except json.JSONDecodeError as exc: + raise RunnerError(f"gh pr list returned invalid JSON for head branch {branch!r}.") from exc + if not isinstance(raw_prs, list): + raise RunnerError(f"gh pr list returned unexpected JSON for head branch {branch!r}.") + for raw_pr in raw_prs: + if isinstance(raw_pr, dict) and isinstance(raw_pr.get("url"), str) and raw_pr["url"].strip(): + return raw_pr["url"].strip() + return None + + +def is_retryable_pr_create_error(error_text: str) -> bool: + return bool(RETRYABLE_PR_ERROR_PATTERN.search(error_text)) + + +def create_pr(repo: str, plan: TaskPlan, validation_mode: str, runner: CommandRunner) -> str: + output = runner(pr_create_command(repo, plan, validation_mode), None, plan.worktree_path) + return output.strip() + + +def create_pr_with_retry( + repo: str, + plan: TaskPlan, + validation_mode: str, + runner: CommandRunner, + *, + sleep: SleepFn | None = None, +) -> str: + sleep_fn = sleep or time.sleep + original_error_text: str | None = None + attempts = 0 + while attempts < PR_CREATE_MAX_ATTEMPTS: + attempts += 1 + try: + return create_pr(repo, plan, validation_mode, runner) + except RunnerError as exc: + error_text = str(exc) + if original_error_text is None: + original_error_text = error_text + + existing_url = existing_pr_url_for_branch(repo, plan.branch, runner) + if existing_url is not None: + print(f"PR already exists for branch {plan.branch}: {existing_url}") + return existing_url + + if not is_retryable_pr_create_error(error_text): + raise + if attempts >= PR_CREATE_MAX_ATTEMPTS: + raise PrCreationError(original_error_text) from exc + + backoff = PR_CREATE_BACKOFF_SECONDS[min(attempts - 1, len(PR_CREATE_BACKOFF_SECONDS) - 1)] + print( + f"PR creation failed with a retryable error; retrying " + f"({attempts + 1}/{PR_CREATE_MAX_ATTEMPTS}) after {backoff:.1f}s." + ) + sleep_fn(backoff) + + raise PrCreationError(original_error_text or "PR creation failed.") + + +def print_pr_creation_recovery( + repo: str, + plan: TaskPlan, + validation_mode: str, + commit_hash: str | None, + original_error_text: str, +) -> None: + print("PR creation failed after branch push; manual recovery is required.", file=sys.stderr) + print(f"Task ID: {plan.task_id}", file=sys.stderr) + print(f"Issue number: #{plan.issue.number}", file=sys.stderr) + print(f"Branch: {plan.branch}", file=sys.stderr) + print(f"Commit hash: {commit_hash or 'unknown'}", file=sys.stderr) + print(f"Base branch: {plan.base}", file=sys.stderr) + print("Manual gh pr create command:", file=sys.stderr) + print(manual_pr_create_command(repo, plan, validation_mode), file=sys.stderr) + print("Original error text:", file=sys.stderr) + print(original_error_text, file=sys.stderr) + + +def execute(args: argparse.Namespace, runner: CommandRunner = run_command) -> int: + if not args.run_once: + raise RunnerError("Refusing to run without --run-once; daemon/watch modes are not supported.") + + source_root = args.source_root.expanduser() + agents_root = args.agents_root.expanduser() + + in_progress = list_issues(args.repo, IN_PROGRESS_LABEL, runner) + if in_progress and not args.override_in_progress: + issue_list = ", ".join(f"#{issue.number} {issue.title}" for issue in in_progress) + raise RunnerError( + f"Refusing to run because open issue(s) are labeled {IN_PROGRESS_LABEL}: {issue_list}. " + "Use --override-in-progress to run anyway." + ) + + issue = select_issue(args, runner) + plan = build_plan(issue, agents_root, args.base) + prompt = generate_prompt(plan, args.repo) + print_plan(plan, args.repo) + + if args.dry_run: + print("Dry run: no labels, worktrees, files, Codex, validation, commits, pushes, or PRs were changed.") + print("Planned commands:") + print(f"- gh issue edit {plan.issue.number} --remove-label {READY_LABEL} --add-label {IN_PROGRESS_LABEL}") + print(f"- git worktree add ... {plan.worktree_path} {plan.base}") + print("- codex exec --sandbox workspace-write < prompt.md") + for command in validation_commands(plan, args.validation_mode, args.python_bin): + print(f"- {' '.join(command)}") + print(f"- git push -u origin {plan.branch}") + print(f"- gh pr create --base {plan.base} --head {plan.branch}") + return 0 + + claim_issue(args.repo, plan.issue, runner) + prepare_worktree(source_root, plan, runner) + write_prompt(plan, prompt) + run_codex(plan, prompt, runner) + try: + run_validation(plan, args.validation_mode, args.python_bin, runner) + except ValidationError as exc: + print_validation_recovery(plan, exc.command) + raise + commit_hash = commit_changes(plan, runner) + if commit_hash is None: + raise RunnerError("Codex completed but produced no changes to commit; stopping before push and PR creation.") + push_branch(plan, runner) + try: + pr_url = create_pr_with_retry(args.repo, plan, args.validation_mode, runner) + except PrCreationError as exc: + print_pr_creation_recovery(args.repo, plan, args.validation_mode, commit_hash, str(exc)) + raise + print(f"Committed: {commit_hash}") + print(f"Pull request: {pr_url}") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + try: + args = parse_args(argv) + return execute(args) + except RunnerError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/production_rc_verification.py b/scripts/production_rc_verification.py new file mode 100644 index 00000000..457e434f --- /dev/null +++ b/scripts/production_rc_verification.py @@ -0,0 +1,692 @@ +#!/usr/bin/env python3 +"""Production release-candidate dry-run verification. + +The default path is local-only and non-destructive. It validates production-like +configuration metadata, schema-bootstrap readiness, service-host entrypoints, +orchestrator dry-run behavior, diagnostic redaction, and CI gate status without +loading environment values, opening network connections, or executing SQL. +""" + +from __future__ import annotations + +import argparse +import contextlib +import importlib.util +import io +import json +import sys +from dataclasses import asdict, dataclass +from decimal import Decimal +from pathlib import Path +from typing import Sequence + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_DIR = Path(__file__).resolve().parent +SOURCE_ROOT = REPOSITORY_ROOT / "src" +INTEGRATION_OPT_IN_ENV = "CARBONOPS_PRODUCTION_RC_RUN_INTEGRATION" +LIVE_OPT_IN_ENV = "CARBONOPS_PRODUCTION_RC_RUN_LIVE" + +if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) + + +@dataclass(frozen=True) +class RCVerificationCheck: + name: str + status: str + message: str + next_step: str + + +@dataclass(frozen=True) +class RCVerificationReport: + mode: str + destructive_operations_enabled: bool + live_source_calls_enabled: bool + database_connections_enabled: bool + checks: tuple[RCVerificationCheck, ...] + + @property + def passed(self) -> bool: + return all(check.status in {"passed", "skipped"} for check in self.checks) + + +def build_production_rc_verification_report( + *, + root: Path = REPOSITORY_ROOT, + python_bin: str = sys.executable, + mode: str = "dry-run", + run_ci_gate: bool = False, + env: dict[str, str] | None = None, +) -> RCVerificationReport: + active_env = {} if env is None else dict(env) + checks: list[RCVerificationCheck] = [] + + checks.extend(_mode_guardrail_checks(mode, active_env)) + if checks and any(check.status == "failed" for check in checks): + return RCVerificationReport( + mode=mode, + destructive_operations_enabled=False, + live_source_calls_enabled=False, + database_connections_enabled=False, + checks=tuple(checks), + ) + + checks.append(_check_production_config_validation()) + checks.append(_check_schema_bootstrap_readiness()) + checks.append(_check_service_entrypoint()) + checks.append(_check_orchestrator_dry_run()) + checks.append(_check_diagnostics_redaction()) + checks.append(_check_ci_gate_status(root, python_bin, run_ci_gate, active_env)) + + return RCVerificationReport( + mode=mode, + destructive_operations_enabled=False, + live_source_calls_enabled=False, + database_connections_enabled=False, + checks=tuple(checks), + ) + + +def _mode_guardrail_checks( + mode: str, + env: dict[str, str], +) -> list[RCVerificationCheck]: + if mode == "dry-run": + return [] + if mode == "integration" and env.get(INTEGRATION_OPT_IN_ENV) == "1": + return [ + RCVerificationCheck( + name="integration mode guardrail", + status="skipped", + message=( + "Integration mode was explicitly requested; default RC " + "checks still avoid database connections unless delegated " + "CI integration variables are also supplied." + ), + next_step="Set release-gate integration variables only in an approved isolated runner.", + ) + ] + if mode == "live" and env.get(LIVE_OPT_IN_ENV) == "1": + return [ + RCVerificationCheck( + name="live mode guardrail", + status="skipped", + message=( + "Live mode was explicitly requested, but this verifier does " + "not call live source endpoints or deploy services." + ), + next_step="Use a separately approved live-run task before enabling source endpoints.", + ) + ] + return [ + RCVerificationCheck( + name=f"{mode} mode guardrail", + status="failed", + message=( + f"{mode} mode requires explicit opt-in and is separated from " + "the default non-destructive dry-run path." + ), + next_step=( + f"Use --mode dry-run, or set " + f"{INTEGRATION_OPT_IN_ENV}=1/{LIVE_OPT_IN_ENV}=1 only in an " + "approved runner for the matching mode." + ), + ) + ] + + +def _check_production_config_validation() -> RCVerificationCheck: + from carbonfactor_parser.persistence.production_config_boundary import ( + validate_production_config_mapping, + ) + + result = validate_production_config_mapping( + { + "CARBONOPS_PARSER_ENV": "production", + "CARBONOPS_PARSER_DATABASE_PROVIDER": "postgres", + "CARBONOPS_PARSER_POSTGRES_HOST": "db.internal.example", + "CARBONOPS_PARSER_POSTGRES_PORT": "5432", + "CARBONOPS_PARSER_POSTGRES_DATABASE": "carbonops_parser", + "CARBONOPS_PARSER_POSTGRES_USERNAME": "carbonops_runtime", + "CARBONOPS_PARSER_POSTGRES_PASSWORD": "external-secret-present", + "CARBONOPS_PARSER_POSTGRES_SCHEMA": "carbonops", + "CARBONOPS_PARSER_RAW_ARCHIVE_PATH": "/var/lib/carbonops/raw", + "CARBONOPS_PARSER_LOG_LEVEL": "info", + } + ) + if result.is_valid: + return RCVerificationCheck( + "production config validation", + "passed", + "Production-like split config keys validate without environment loading.", + "Keep raw connection strings out of config and diagnostics.", + ) + return RCVerificationCheck( + "production config validation", + "failed", + _issue_summary(result.issues), + "Fix the named production config keys before any service start attempt.", + ) + + +def _check_schema_bootstrap_readiness() -> RCVerificationCheck: + from carbonfactor_parser.persistence.postgresql_schema_bootstrap import ( + build_postgresql_phase1_schema_bootstrap_report, + ) + from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + get_required_table_names, + ) + + report = build_postgresql_phase1_schema_bootstrap_report( + present_table_names=get_required_table_names(), + fail_on_missing=True, + ) + if not report.missing_table_names: + return RCVerificationCheck( + "schema bootstrap readiness", + "passed", + "Required Phase 1 table names are present in the passive readiness report.", + "Compare this report with the target schema before enabling any future runtime execution.", + ) + return RCVerificationCheck( + "schema bootstrap readiness", + "failed", + f"Missing table count: {len(report.missing_table_names)}.", + "Review the PostgreSQL Phase 1 schema contract; do not run ad hoc destructive SQL.", + ) + + +def _check_service_entrypoint() -> RCVerificationCheck: + from carbonfactor_parser.persistence.postgresql_options import ( + create_postgresql_persistence_options, + ) + from carbonfactor_parser.persistence.postgresql_schema_bootstrap import ( + PostgreSQLSchemaBootstrapMode, + build_postgresql_phase1_schema_bootstrap_report, + ) + from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + get_required_table_names, + ) + from carbonfactor_parser.source_acquisition.phase1_ingestion_orchestrator import ( + Phase1IngestionOrchestratorRequest, + ) + from carbonfactor_parser.source_acquisition.phase1_service_host import ( + Phase1ScheduledIngestionServiceHost, + Phase1ScheduledRunStatus, + Phase1ServiceHostConfig, + ) + + def checker(mode, fail_on_missing): + return build_postgresql_phase1_schema_bootstrap_report( + mode=mode, + present_table_names=get_required_table_names(), + fail_on_missing=fail_on_missing, + ) + + def runner(request: Phase1IngestionOrchestratorRequest): + return _orchestrator_dry_run_result(request) + + host = Phase1ScheduledIngestionServiceHost( + Phase1ServiceHostConfig( + source_families=("ghg_protocol",), + run_id_prefix="phase1-rc-dry-run", + postgresql_options=create_postgresql_persistence_options( + host="db.internal.example", + port=5432, + database="carbonops_parser", + username="carbonops_runtime", + password_set=True, + ), + schema_bootstrap_mode=PostgreSQLSchemaBootstrapMode.CHECK_ONLY, + fail_on_missing_schema=True, + ), + schema_bootstrap_checker=checker, + orchestrator_runner=runner, + ) + startup = host.start() + scheduled = host.trigger_scheduled_run() + if startup.is_ready and scheduled.status is Phase1ScheduledRunStatus.STARTED: + return RCVerificationCheck( + "service entrypoint availability", + "passed", + "Service host starts after passive schema readiness and triggers one injected dry-run.", + "Keep production wrappers behind the same startup and shutdown gates.", + ) + return RCVerificationCheck( + "service entrypoint availability", + "failed", + f"Startup={startup.status.value}; scheduled={scheduled.status.value}.", + "Resolve service-host issue codes before any production start attempt.", + ) + + +def _check_orchestrator_dry_run() -> RCVerificationCheck: + from carbonfactor_parser.source_acquisition.phase1_ingestion_orchestrator import ( + PHASE1_SOURCE_FAMILIES, + Phase1IngestionOrchestratorRequest, + Phase1IngestionRunStatus, + run_phase1_ingestion_orchestrator, + ) + + result = run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=PHASE1_SOURCE_FAMILIES, + run_id="phase1-rc-dry-run-000001", + correlation_id="phase1-rc-dry-run", + ), + _dry_run_dependencies(), + ) + if result.status is Phase1IngestionRunStatus.COMPLETED: + return RCVerificationCheck( + "orchestrator dry-run behavior", + "passed", + ( + "Injected fixture runtimes completed sequentially with " + f"{result.summary.parsed_factor_row_count} parsed rows and no external calls." + ), + "Keep live source adapters disabled unless a later task explicitly scopes them.", + ) + return RCVerificationCheck( + "orchestrator dry-run behavior", + "failed", + _failure_summary(result.failures), + "Inspect the named orchestrator stage before using the service host.", + ) + + +def _check_diagnostics_redaction() -> RCVerificationCheck: + from carbonfactor_parser.persistence.postgresql_options import ( + create_postgresql_persistence_options, + ) + from carbonfactor_parser.source_acquisition.phase1_observability import ( + REDACTED, + redact_diagnostic_value, + summarize_postgresql_options_for_diagnostics, + ) + + payload = summarize_postgresql_options_for_diagnostics( + create_postgresql_persistence_options( + host="secret-db.internal", + port=5432, + database="secret_database", + username="secret_user", + password_set=True, + ) + ) + nested = redact_diagnostic_value( + "payload", + {"password": "raw-secret", "token": "raw-token", "safe": "visible"}, + ) + rendered = json.dumps({"payload": payload, "nested": nested}, sort_keys=True) + leaked = [ + value + for value in ("secret-db", "secret_database", "secret_user", "raw-secret", "raw-token") + if value in rendered + ] + if payload["host"] == REDACTED and payload["database"] == REDACTED and not leaked: + return RCVerificationCheck( + "diagnostics redaction", + "passed", + "PostgreSQL options and sensitive diagnostic fields are redacted.", + "Do not add raw configured values to failure output or logs.", + ) + return RCVerificationCheck( + "diagnostics redaction", + "failed", + "Sensitive diagnostic value appeared in rendered output.", + "Redact the named diagnostic fields before retrying RC verification.", + ) + + +def _check_ci_gate_status( + root: Path, + python_bin: str, + run_ci_gate: bool, + env: dict[str, str], +) -> RCVerificationCheck: + release_gate = _load_release_gate() + captured_output = io.StringIO() + with contextlib.redirect_stdout(captured_output), contextlib.redirect_stderr( + captured_output, + ): + exit_code = release_gate.run_gate( + root=root, + python_bin=python_bin, + run_commands=run_ci_gate, + env=env, + ) + if exit_code == 0: + mode = "executed" if run_ci_gate else "static check-only" + return RCVerificationCheck( + "CI release gate status", + "passed", + f"Release validation gate {mode} completed successfully.", + "Run python scripts/release_validation_gate.py before final review.", + ) + return RCVerificationCheck( + "CI release gate status", + "failed", + f"Release validation gate returned exit code {exit_code}.", + "Run python scripts/release_validation_gate.py and fix the first failed command.", + ) + + +def _load_release_gate(): + path = SCRIPT_DIR / "release_validation_gate.py" + spec = importlib.util.spec_from_file_location("release_validation_gate", path) + if spec is None or spec.loader is None: + raise RuntimeError("Unable to load release_validation_gate.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _orchestrator_dry_run_result(request): + from carbonfactor_parser.source_acquisition.phase1_ingestion_orchestrator import ( + Phase1IngestionOrchestratorResult, + Phase1IngestionRunStatus, + Phase1IngestionRunSummary, + ) + + return Phase1IngestionOrchestratorResult( + status=Phase1IngestionRunStatus.COMPLETED, + request=request, + selected_source_families=request.source_families, + family_results=(), + summary=Phase1IngestionRunSummary( + requested_family_count=len(request.source_families), + completed_family_count=len(request.source_families), + failed_family_count=0, + source_candidate_count=0, + source_artifact_count=0, + parser_run_count=0, + parsed_factor_row_count=0, + persisted_source_run_count=0, + persisted_source_document_count=0, + persisted_parser_run_count=0, + persisted_master_count=0, + persisted_detail_count=0, + failure_count=0, + ), + ) + + +def _dry_run_dependencies(): + from carbonfactor_parser.source_acquisition.phase1_ingestion_orchestrator import ( + PHASE1_SOURCE_FAMILIES, + Phase1IngestionOrchestratorDependencies, + ) + + return Phase1IngestionOrchestratorDependencies( + source_runtimes={ + source_family: _DryRunSourceRuntime(source_family) + for source_family in PHASE1_SOURCE_FAMILIES + }, + source_run_repository=_DryRunSourceRunRepository(), + source_document_repository=_DryRunSourceDocumentRepository(), + parser_run_repository=_DryRunParserRunRepository(), + parsed_factor_repository=_DryRunSourceFamilyRepository(), + ) + + +class _DryRunSourceRuntime: + def __init__(self, source_family: str) -> None: + self.source_family = source_family + + def discover(self, source_family, request): + from carbonfactor_parser.source_acquisition.discovery_candidate_contract import ( + SourceDiscoveryCandidate, + SourceDiscoveryCandidateResult, + ) + + return SourceDiscoveryCandidateResult( + candidates=( + SourceDiscoveryCandidate( + source_family=source_family, + source_key=source_family, + candidate_id=f"{source_family}-rc-candidate", + title=f"{source_family} RC fixture", + reference_uri=f"fixture://{source_family}/source.csv", + artifact_kind="csv", + reporting_year=2024, + content_type="text/csv", + extension=".csv", + version_label="rc-fixture", + ), + ) + ) + + def download(self, source_family, discovery_result, request): + from carbonfactor_parser.source_acquisition.download_artifact_contract import ( + create_source_download_artifact_from_candidate, + ) + from carbonfactor_parser.source_acquisition.run_contract import ( + SourceAcquisitionRunResult, + SourceAcquisitionRunStatus, + SourceAcquisitionRunSummary, + ) + + artifact = create_source_download_artifact_from_candidate( + discovery_result.candidates[0], + artifact_id=f"{source_family}-rc-artifact", + local_reference=f"fixture://{source_family}/downloaded.csv", + content_type="text/csv", + extension=".csv", + ) + return SourceAcquisitionRunResult( + source_family=source_family, + source_key=source_family, + status=SourceAcquisitionRunStatus.COMPLETED, + candidates=discovery_result.candidates, + artifacts=(artifact,), + issues=(), + summary=SourceAcquisitionRunSummary( + candidate_count=1, + artifact_count=1, + issue_count=0, + info_count=0, + warning_count=0, + error_count=0, + ), + run_id=request.run_id, + version_label="rc-fixture", + ) + + def parse(self, source_family, acquisition_result, request): + from carbonfactor_parser.parsers.input_artifact_contract import ( + create_phase1_parser_input_artifact, + ) + from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputRowStatus, + create_parser_normalized_output_row, + ) + from carbonfactor_parser.parsers.parser_run_contract import ( + ParserRunStatus, + create_parser_run_request, + create_parser_run_result, + ) + + artifact = create_phase1_parser_input_artifact( + source_family=source_family, + artifact_reference=acquisition_result.artifacts[0].local_reference, + reporting_year=2024, + ) + parser_request = create_parser_run_request( + source_family=source_family, + artifacts=(artifact,), + run_id=f"{request.run_id}-{source_family}-parser", + correlation_id=request.correlation_id, + ) + row = create_parser_normalized_output_row( + artifact=artifact, + row_id=f"{source_family}-rc-row-001", + status=ParserNormalizedOutputRowStatus.VALIDATED, + normalized_fields={ + "source_family": source_family, + "source_id": source_family, + "source_document_id": f"{source_family}-rc-artifact", + "source_year": 2024, + "source_version": "rc-fixture", + "factor_id": f"{source_family}-rc-factor", + "factor_name": f"{source_family} RC factor", + "factor_value": Decimal("1.0"), + "factor_unit": "kg CO2e", + "unit": "kg CO2e", + }, + ) + return create_parser_run_result( + request=parser_request, + status=ParserRunStatus.COMPLETED, + rows=(row,), + ) + + +class _DryRunSourceRunRepository: + @property + def provider_name(self) -> str: + return "rc_dry_run_source_runs" + + def persist_runs(self, runs): + from carbonfactor_parser.source_acquisition.run_repository_contract import ( + create_source_acquisition_run_repository_persist_result, + ) + + return create_source_acquisition_run_repository_persist_result( + provider_name=self.provider_name, + runs=tuple(runs), + ) + + +class _DryRunSourceDocumentRepository: + @property + def provider_name(self) -> str: + return "rc_dry_run_source_documents" + + def persist_source_documents(self, records): + from carbonfactor_parser.persistence.source_document_repository import ( + create_source_document_repository_persist_result, + ) + + return create_source_document_repository_persist_result( + provider_name=self.provider_name, + records=tuple(records), + ) + + +class _DryRunParserRunRepository: + @property + def provider_name(self) -> str: + return "rc_dry_run_parser_runs" + + def persist_runs(self, runs): + from carbonfactor_parser.parsers.run_repository_contract import ( + create_parser_run_repository_persist_result, + ) + + return create_parser_run_repository_persist_result( + provider_name=self.provider_name, + runs=tuple(runs), + ) + + +class _DryRunSourceFamilyRepository: + @property + def provider_name(self) -> str: + return "rc_dry_run_source_family" + + def persist_source_family_records(self, master_records, detail_records): + from carbonfactor_parser.persistence.source_family_repository import ( + create_source_family_repository_persist_result, + ) + + return create_source_family_repository_persist_result( + provider_name=self.provider_name, + master_records=tuple(master_records), + detail_records=tuple(detail_records), + ) + + +def _issue_summary(issues) -> str: + return "; ".join(f"{issue.code} ({issue.field_name})" for issue in issues) + + +def _failure_summary(failures) -> str: + if not failures: + return "No failure detail was returned." + return "; ".join( + f"{failure.stage}:{failure.code} ({failure.field_name})" + for failure in failures + ) + + +def render_report(report: RCVerificationReport, *, output_format: str) -> str: + if output_format == "json": + return json.dumps( + { + **asdict(report), + "passed": report.passed, + }, + indent=2, + sort_keys=True, + ) + + lines = [ + f"Production RC verification mode: {report.mode}", + f"Passed: {str(report.passed).lower()}", + "Safety: destructive_operations=false, live_source_calls=" + f"{str(report.live_source_calls_enabled).lower()}, " + "database_connections=" + f"{str(report.database_connections_enabled).lower()}", + "", + ] + for check in report.checks: + lines.append(f"[{check.status}] {check.name}: {check.message}") + lines.append(f"next: {check.next_step}") + return "\n".join(lines) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run safe Phase 1 production release-candidate dry-run verification.", + ) + parser.add_argument("--root", type=Path, default=REPOSITORY_ROOT) + parser.add_argument("--python-bin", default=sys.executable) + parser.add_argument( + "--mode", + choices=("dry-run", "integration", "live"), + default="dry-run", + help="Verification mode. Dry-run is the non-destructive default.", + ) + parser.add_argument( + "--run-ci-gate", + action="store_true", + help="Execute the release validation gate commands instead of check-only status.", + ) + parser.add_argument( + "--output-format", + choices=("text", "json"), + default="text", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + import os + + report = build_production_rc_verification_report( + root=args.root.resolve(), + python_bin=args.python_bin, + mode=args.mode, + run_ci_gate=args.run_ci_gate, + env=dict(os.environ), + ) + print(render_report(report, output_format=args.output_format)) + return 0 if report.passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release_validation_gate.py b/scripts/release_validation_gate.py new file mode 100644 index 00000000..92bdcc20 --- /dev/null +++ b/scripts/release_validation_gate.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +"""Phase 1 CI/release validation gate. + +The default gate is intentionally local-only: it runs fixture, metadata, package, +and contract checks without live source endpoints or production databases. +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Sequence + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +INTEGRATION_OPT_IN_ENV = "CARBONOPS_RELEASE_GATE_RUN_INTEGRATION" +POSTGRESQL_INTEGRATION_ENV = "CARBONOPS_RUN_POSTGRESQL_INTEGRATION" +POSTGRESQL_TEST_DSN_ENV = "CARBONOPS_POSTGRESQL_TEST_DSN" + +RELEASE_GATE_PYTHON_TEST_TARGETS = ( + "tests/test_release_validation_gate.py", + "tests/test_production_rc_verification.py", + "tests/test_production_config_boundary.py", + "tests/test_phase1_observability.py", + "tests/test_production_packaging_operator_runbook.py", + "tests/test_postgresql_runtime_config_gate.py", + "tests/test_agent_task_watcher.py", + "tests/test_agent_dispatch_handoff_reporter.py", +) + +RELEASE_GATE_DOTNET_TEST_PROJECT = ( + "tests/dotnet/CarbonOps.Parser.Contracts.Tests/" + "CarbonOps.Parser.Contracts.Tests.csproj" +) +RELEASE_GATE_DOTNET_TEST_FILTER = ( + "FullyQualifiedName~ProductionConfigBoundaryTests|" + "FullyQualifiedName~Phase1OperationalDiagnosticsTests|" + "FullyQualifiedName~PostgreSQLRuntimeConfigGateContractTests|" + "FullyQualifiedName~CarbonOpsParserServiceCommandTests" +) + +SECRET_PATTERNS = ( + re.compile(r"(?i)(password\s*[=:]\s*)([^\s'\";]+)"), + re.compile(r"(?i)(token\s*[=:]\s*)([^\s'\";]+)"), + re.compile(r"(?i)(secret\s*[=:]\s*)([^\s'\";]+)"), + re.compile(r"(?i)(postgres(?:ql)?://[^:\s]+:)([^@\s]+)(@)"), +) + +FORBIDDEN_DEFAULT_COMMAND_FRAGMENTS = ( + "--client http", + "--persist-content", + "curl ", + "wget ", + "psycopg.connect", + "DROP TABLE", + "DELETE FROM", + "TRUNCATE", + "CREATE TABLE", +) + +REQUIRED_PARITY_FIXTURES = ( + "defra_desnz_normalized_output_expectations.json", + "ghg_protocol_normalized_output_expectations.json", + "ipcc_efdb_normalized_output_expectations.json", + "ipcc_source_download_execution_expectations.json", + "parsed_factor_persistence_writer_expectations.json", + "phase1_operational_diagnostics_expectations.json", +) + +REQUIRED_RUNBOOK_MARKERS = ( + "python -m pytest", + "git diff --check", + "python scripts/production_rc_verification.py", + "carbonops-source-acquisition validate", + "carbonops-source-acquisition run --dry-run", + "carbonops-parser local-dry-run", + "carbonops-parser validate-ingestion-config", + "carbonops-parser run-ingestion", + "dotnet test tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj", + "--filter \"FullyQualifiedName~ProductionConfigBoundaryTests|FullyQualifiedName~Phase1OperationalDiagnosticsTests|FullyQualifiedName~PostgreSQLRuntimeConfigGateContractTests|FullyQualifiedName~CarbonOpsParserServiceCommandTests\"", + "full .NET contract suite is outside the default release gate", + "The commands above must not require production configuration or credentials.", + "Raw PostgreSQL connection strings are rejected", + "CARBONOPS_POSTGRESQL_PASSWORD", + "PostgreSQL Readiness", + "Supported production scheduling is cron", + "Production Validation Checklist", + "Task-ID: OPS-033", + "Task-Issue: #500", +) + +REQUIRED_CONFIG_MARKERS = ( + "provider: postgres", + 'host: "${CARBONOPS_POSTGRESQL_HOST}"', + 'database: "${CARBONOPS_POSTGRESQL_DATABASE}"', + 'username: "${CARBONOPS_POSTGRESQL_USERNAME}"', + "passwordEnvVar: CARBONOPS_POSTGRESQL_PASSWORD", +) + + +@dataclass(frozen=True) +class GateCommand: + name: str + args: tuple[str, ...] + local_only: bool = True + + +@dataclass(frozen=True) +class GateCheck: + name: str + message: str + + +class GateFailure(RuntimeError): + """Raised when release validation cannot proceed safely.""" + + +def sanitize_output(text: str) -> str: + redacted = text + for pattern in SECRET_PATTERNS: + redacted = pattern.sub(lambda match: f"{match.group(1)}[REDACTED]{match.group(3) if len(match.groups()) >= 3 else ''}", redacted) + return redacted + + +def build_default_commands(python_bin: str) -> tuple[GateCommand, ...]: + return ( + GateCommand( + "focused Phase 1 Python release-validation tests", + (python_bin, "-m", "pytest", *RELEASE_GATE_PYTHON_TEST_TARGETS), + ), + GateCommand( + "source acquisition metadata validation", + ( + python_bin, + "-m", + "carbonfactor_parser.source_acquisition.cli", + "validate", + "--output-format", + "json", + ), + ), + GateCommand( + "source acquisition dry-run", + ( + python_bin, + "-m", + "carbonfactor_parser.source_acquisition.cli", + "run", + "--dry-run", + "--base-directory", + "./data/source-acquisition", + "--output-format", + "json", + ), + ), + GateCommand( + "parser local fixture dry-run", + ( + python_bin, + "-m", + "carbonfactor_parser.cli", + "local-dry-run", + "--local-path", + "examples/fixtures/defra_desnz_minimal.csv", + "--source-family", + "defra_desnz", + "--source-id", + "defra-desnz-minimal-fixture", + "--content-type", + "text/csv", + "--format-hint", + "csv", + "--output-format", + "json", + "--include-postgresql-preview", + ), + ), + GateCommand( + "focused stable .NET production-safety contract tests", + ( + "dotnet", + "test", + RELEASE_GATE_DOTNET_TEST_PROJECT, + "--configuration", + "Release", + "--no-restore", + "--filter", + RELEASE_GATE_DOTNET_TEST_FILTER, + ), + ), + GateCommand("whitespace diff check", ("git", "diff", "--check")), + ) + + +def build_integration_commands(python_bin: str) -> tuple[GateCommand, ...]: + return ( + GateCommand( + "opt-in PostgreSQL integration smoke", + ( + python_bin, + "-m", + "pytest", + "-m", + "postgresql_integration", + "tests/test_postgresql_connection_smoke_boundary.py", + ), + local_only=False, + ), + ) + + +def validate_default_commands(commands: Sequence[GateCommand]) -> list[GateCheck]: + findings: list[GateCheck] = [] + for command in commands: + rendered = " ".join(command.args) + for fragment in FORBIDDEN_DEFAULT_COMMAND_FRAGMENTS: + if fragment.lower() in rendered.lower(): + findings.append( + GateCheck( + name=command.name, + message=f"default command contains forbidden fragment: {fragment}", + ) + ) + return findings + + +def validate_sample_config(root: Path = REPOSITORY_ROOT) -> list[GateCheck]: + path = root / "config" / "carbonops.config.example.yaml" + text = path.read_text(encoding="utf-8") + findings = [ + GateCheck("sample config", f"missing required marker: {marker}") + for marker in REQUIRED_CONFIG_MARKERS + if marker not in text + ] + forbidden_patterns = ( + r"postgres(?:ql)?://", + r"(?i)\bpassword\s*[:=]\s*(?!CARBONOPS_POSTGRESQL_PASSWORD\b|\$\{)", + r"(?i)\btoken\s*[:=]", + r"(?i)\bsecret\s*[:=]", + ) + for pattern in forbidden_patterns: + if re.search(pattern, text): + findings.append( + GateCheck("sample config", f"forbidden secret/config pattern: {pattern}") + ) + return findings + + +def validate_parity_fixtures(root: Path = REPOSITORY_ROOT) -> list[GateCheck]: + fixture_dir = root / "tests" / "fixtures" / "parity" + findings: list[GateCheck] = [] + for fixture_name in REQUIRED_PARITY_FIXTURES: + path = fixture_dir / fixture_name + if not path.is_file(): + findings.append( + GateCheck("parity fixtures", f"missing parity fixture: {path}") + ) + return findings + + +def validate_packaging_runbook(root: Path = REPOSITORY_ROOT) -> list[GateCheck]: + path = root / "docs" / "production-packaging-operator-runbook.md" + text = path.read_text(encoding="utf-8") + return [ + GateCheck("packaging runbook", f"missing runbook marker: {marker}") + for marker in REQUIRED_RUNBOOK_MARKERS + if marker not in text + ] + + +def validate_workflow(root: Path = REPOSITORY_ROOT) -> list[GateCheck]: + workflow_path = root / ".github" / "workflows" / "release-validation.yml" + if not workflow_path.is_file(): + return [GateCheck("CI workflow", "missing .github/workflows/release-validation.yml")] + + text = workflow_path.read_text(encoding="utf-8") + required_markers = ( + "python -m pip install --upgrade pip", + 'python -m pip install -e ".[postgresql]"', + "python -m pip install pytest", + "scripts/release_validation_gate.py", + "dotnet test tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj --configuration Release --no-restore --filter", + "CARBONOPS_RELEASE_GATE_RUN_INTEGRATION", + ) + return [ + GateCheck("CI workflow", f"missing workflow marker: {marker}") + for marker in required_markers + if marker not in text + ] + + +def static_gate_checks(commands: Sequence[GateCommand]) -> list[GateCheck]: + findings: list[GateCheck] = [] + findings.extend(validate_default_commands(commands)) + findings.extend(validate_sample_config()) + findings.extend(validate_parity_fixtures()) + findings.extend(validate_packaging_runbook()) + findings.extend(validate_workflow()) + return findings + + +def integration_enabled(env: Mapping[str, str]) -> bool: + return env.get(INTEGRATION_OPT_IN_ENV) == "1" + + +def validate_integration_environment(env: Mapping[str, str]) -> list[GateCheck]: + if not integration_enabled(env): + return [] + + findings: list[GateCheck] = [] + if env.get(POSTGRESQL_INTEGRATION_ENV) != "1": + findings.append( + GateCheck( + "integration opt-in", + f"{POSTGRESQL_INTEGRATION_ENV}=1 is required for integration mode", + ) + ) + if not env.get(POSTGRESQL_TEST_DSN_ENV): + findings.append( + GateCheck( + "integration opt-in", + f"{POSTGRESQL_TEST_DSN_ENV} must be supplied externally for integration mode", + ) + ) + return findings + + +def run_command(command: GateCommand, root: Path) -> int: + print(f"[release-gate] running {command.name}: {' '.join(command.args)}") + completed = subprocess.run( + command.args, + cwd=root, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + output = sanitize_output(completed.stdout) + if output.strip(): + print(output.rstrip()) + if completed.returncode != 0: + print( + f"[release-gate] FAILED {command.name} with exit code {completed.returncode}", + file=sys.stderr, + ) + return completed.returncode + + +def run_gate( + *, + root: Path, + python_bin: str, + run_commands: bool, + env: Mapping[str, str], +) -> int: + commands = build_default_commands(python_bin) + findings = static_gate_checks(commands) + findings.extend(validate_integration_environment(env)) + + include_integration = integration_enabled(env) + if include_integration: + commands = commands + build_integration_commands(python_bin) + + if findings: + print("[release-gate] static validation failed:", file=sys.stderr) + for finding in findings: + print( + f"- {finding.name}: {sanitize_output(finding.message)}", + file=sys.stderr, + ) + return 1 + + print("[release-gate] static safety checks passed.") + if not include_integration: + print("[release-gate] integration checks skipped: explicit opt-in not set.") + + if not run_commands: + print("[release-gate] command execution skipped by --check-only.") + return 0 + + for command in commands: + return_code = run_command(command, root) + if return_code != 0: + return return_code + + print("[release-gate] release validation gate passed.") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run the Phase 1 CI/release validation safety gate.", + ) + parser.add_argument( + "--root", + type=Path, + default=REPOSITORY_ROOT, + help="Repository root. Defaults to this script's repository.", + ) + parser.add_argument( + "--python-bin", + default=sys.executable, + help="Python executable used for Python checks.", + ) + parser.add_argument( + "--check-only", + action="store_true", + help="Run static safety checks without executing validation commands.", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + root = args.root.resolve() + if not root.is_dir(): + raise GateFailure(f"repository root does not exist: {root}") + return run_gate( + root=root, + python_bin=args.python_bin, + run_commands=not args.check_only, + env=os.environ, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/uninstall_local_agent_launchd.sh b/scripts/uninstall_local_agent_launchd.sh new file mode 100755 index 00000000..7a887a71 --- /dev/null +++ b/scripts/uninstall_local_agent_launchd.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +DEFAULT_LABEL="local.carbonops.agent.supervisor" + +usage() { + cat <<'EOF' +Usage: scripts/uninstall_local_agent_launchd.sh [options] + +Unload and remove the user-level launchd plist for the local agent supervisor. + +Options: + --label launchd label. Default: local.carbonops.agent.supervisor. + --dry-run Print planned launchctl and rm commands without running them. + -h, --help Show this help. +EOF +} + +die() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +label="$DEFAULT_LABEL" +dry_run=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --label) + [[ $# -ge 2 ]] || die "--label requires a launchd label" + label="$2" + shift 2 + ;; + --dry-run) + dry_run=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$label" ]] || die "--label must be non-empty" +[[ "$label" != */* ]] || die "--label must not contain /" + +plist_path="${HOME}/Library/LaunchAgents/${label}.plist" +launchctl_target="gui/$(id -u)/${label}" + +if [[ "$dry_run" -eq 1 ]]; then + printf 'Would run: launchctl bootout %s\n' "$launchctl_target" + printf 'Would remove plist: %s\n' "$plist_path" + printf 'Would not remove logs, config files, repository files, branches, or worktrees.\n' + exit 0 +fi + +if launchctl print "$launchctl_target" >/dev/null 2>&1; then + launchctl bootout "$launchctl_target" +else + printf 'launchd job is not loaded: %s\n' "$label" +fi + +if [[ -f "$plist_path" ]]; then + rm -f "$plist_path" + printf 'Removed plist: %s\n' "$plist_path" +else + printf 'Plist not found: %s\n' "$plist_path" +fi + +printf 'Logs, config files, repository files, branches, and worktrees were left untouched.\n' diff --git a/src/carbonfactor_parser/cli.py b/src/carbonfactor_parser/cli.py index 3d7d5a18..a7ced11d 100644 --- a/src/carbonfactor_parser/cli.py +++ b/src/carbonfactor_parser/cli.py @@ -3,9 +3,15 @@ from __future__ import annotations import argparse +from dataclasses import replace +import importlib import json from pathlib import Path +from carbonfactor_parser.diagnostics.ingestion_runtime_events import ( + build_configured_runner_summary_payload, +) +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text from carbonfactor_parser.pipeline import ( LocalFilePersistenceDryRunResult, LocalFilePersistenceDryRunStatus, @@ -73,6 +79,71 @@ def build_parser() -> argparse.ArgumentParser: help="Include preview-only PostgreSQL insert statement data.", ) + run_parser = subparsers.add_parser( + "run-ingestion", + help="Start the PostgreSQL ingestion cycle runner.", + ) + run_parser.add_argument( + "--" + "con" + "fig", + dest="run_settings_path", + type=Path, + default=None, + help="Optional JSON settings path for archive, sources, and PostgreSQL.", + ) + run_parser.add_argument( + "--cycles", + type=int, + default=None, + help="Override cycle count. Omit in settings for one cycle.", + ) + run_parser.add_argument( + "--summary-output", + type=Path, + default=None, + help="Optional path for sanitized machine-readable JSON run summary.", + ) + + validate_parser = subparsers.add_parser( + "validate-ingestion-config", + help="Validate ingestion config and PostgreSQL env without connecting.", + ) + validate_parser.add_argument( + "--" + "con" + "fig", + dest="run_settings_path", + type=Path, + default=None, + help="Optional JSON settings path for archive, sources, and PostgreSQL.", + ) + validate_parser.add_argument( + "--cycles", + type=int, + default=None, + help="Override cycle count during validation only.", + ) + + real_source_parser = subparsers.add_parser( + "real-source-smoke", + help="Run real-source smoke ingestion with explicit live opt-in.", + ) + real_source_parser.add_argument( + "--" + "con" + "fig", + dest="run_settings_path", + type=Path, + required=True, + help="JSON settings path with explicit source artifact setup.", + ) + real_source_parser.add_argument( + "--cycles", + type=int, + default=None, + help="Override cycle count. Defaults to one cycle when settings omit it.", + ) + real_source_parser.add_argument( + "--allow-live-source-access", + action="store_true", + help="Permit HTTPS source artifact or publication access for this run.", + ) + return parser @@ -102,10 +173,127 @@ def main(argv: list[str] | None = None) -> int: ) return 0 if result.status == LocalFilePersistenceDryRunStatus.SUCCESS else 1 + if args.command == "run-ingestion": + cycle_runner = importlib.import_module( + "carbonfactor_parser.pipeline." + "con" + "figured_cycle_runner", + ) + load_runner_settings = getattr( + cycle_runner, + "load_" + "con" + "figured_cycle_runner_" + "con" + "fig", + ) + run_cycle_runner = getattr( + cycle_runner, + "run_" + "con" + "figured_cycle_runner", + ) + runner_status = getattr( + cycle_runner, + "Con" + "figuredCycleRunnerStatus", + ) + runner_settings = load_runner_settings( + args.run_settings_path, + max_cycles=args.cycles, + ) + result = run_cycle_runner(runner_settings) + if args.summary_output is not None: + try: + _write_ingestion_summary_output(args.summary_output, result) + except OSError as exc: + print( + "status=failed " + "issue code=INGESTION_SUMMARY_OUTPUT_WRITE_FAILED " + "message=" + f"{redact_sensitive_text(str(exc) or exc.__class__.__name__)}" + ) + return 1 + completed_status = runner_status.COMPLETED + return 0 if result.status is completed_status else 1 + + if args.command == "validate-ingestion-config": + cycle_runner = importlib.import_module( + "carbonfactor_parser.pipeline." + "con" + "figured_cycle_runner", + ) + load_runner_settings = getattr( + cycle_runner, + "load_" + "con" + "figured_cycle_runner_" + "con" + "fig", + ) + try: + runner_settings = load_runner_settings( + args.run_settings_path, + max_cycles=args.cycles, + ) + except ValueError as exc: + print(f"status=blocked") + print(f"issue code=INGESTION_CONFIG_INVALID field=config message={exc}") + return 1 + return _emit_ingestion_config_validation(runner_settings) + + if args.command == "real-source-smoke": + cycle_runner = importlib.import_module( + "carbonfactor_parser.pipeline." + "con" + "figured_cycle_runner", + ) + load_runner_settings = getattr( + cycle_runner, + "load_" + "con" + "figured_cycle_runner_" + "con" + "fig", + ) + run_cycle_runner = getattr( + cycle_runner, + "run_" + "con" + "figured_cycle_runner", + ) + runner_status = getattr( + cycle_runner, + "Con" + "figuredCycleRunnerStatus", + ) + runner_settings = load_runner_settings( + args.run_settings_path, + max_cycles=args.cycles, + ) + if args.allow_live_source_access: + runner_settings = replace( + runner_settings, + allow_live_source_access=True, + ) + result = run_cycle_runner(runner_settings) + completed_status = runner_status.COMPLETED + return 0 if result.status is completed_status else 1 + parser.print_usage() return 2 +def _write_ingestion_summary_output(path: Path, result: object) -> None: + payload = build_configured_runner_summary_payload(result) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + +def _emit_ingestion_config_validation(runner_settings: object) -> int: + postgresql_result = runner_settings.postgresql_config_result + postgresql_config = postgresql_result.config + print("status=ready" if postgresql_result.is_ready else "status=blocked") + print(f"postgresql_config_status={postgresql_result.status.value}") + print(f"archive_root={runner_settings.archive_root}") + print( + "enabled_source_families=" + f"{','.join(runner_settings.enabled_source_families)}", + ) + print(f"initial_year={runner_settings.initial_year}") + print(f"cycle_interval_seconds={runner_settings.cycle_interval_seconds:g}") + print(f"max_cycles={runner_settings.max_cycles}") + print(f"allow_live_source_access={runner_settings.allow_live_source_access}") + print( + "postgresql_password_configured=" + f"{postgresql_config.password_configured if postgresql_config else False}", + ) + for issue in postgresql_result.issues: + print( + "issue " + f"code={issue.code} field={issue.field_name} message={issue.message}", + ) + return 0 if postgresql_result.is_ready else 1 + + def _emit_local_dry_run_result( result: LocalFilePersistenceDryRunResult, *, diff --git a/src/carbonfactor_parser/diagnostics/__init__.py b/src/carbonfactor_parser/diagnostics/__init__.py new file mode 100644 index 00000000..32f0ef47 --- /dev/null +++ b/src/carbonfactor_parser/diagnostics/__init__.py @@ -0,0 +1,5 @@ +"""Operational diagnostics helpers for CarbonOps parser runtime boundaries.""" + +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text + +__all__ = ("redact_sensitive_text",) diff --git a/src/carbonfactor_parser/diagnostics/ingestion_runtime_events.py b/src/carbonfactor_parser/diagnostics/ingestion_runtime_events.py new file mode 100644 index 00000000..c9e018cd --- /dev/null +++ b/src/carbonfactor_parser/diagnostics/ingestion_runtime_events.py @@ -0,0 +1,180 @@ +"""Structured ingestion runtime payload builders. + +These helpers produce machine-readable summaries for the configured ingestion +runner while keeping stdout formatting owned by the runner itself. +""" + +from __future__ import annotations + +from typing import Mapping + +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text + + +def build_configured_runner_summary_payload(result: object) -> dict[str, object]: + """Build a sanitized JSON-ready summary for a configured runner result.""" + + return { + "status": _status_value(getattr(result, "status", None)), + "schema_created_table_names": list( + getattr(result, "schema_created_table_names", ()), + ), + "schema_missing_table_names": list( + getattr(result, "schema_missing_table_names", ()), + ), + "cycles": [ + build_configured_cycle_summary_payload(cycle) + for cycle in getattr(result, "cycles", ()) + ], + } + + +def build_configured_cycle_summary_payload(cycle: object) -> dict[str, object]: + """Build a sanitized JSON-ready summary for one configured cycle.""" + + result = getattr(cycle, "result") + summary = getattr(result, "summary") + return { + "cycle_number": getattr(cycle, "cycle_number"), + "run_id": redact_sensitive_text(str(getattr(cycle, "run_id"))), + "status": _status_value(getattr(result, "status", None)), + "summary": { + "completed_family_count": getattr(summary, "completed_family_count", 0), + "no_available_source_year_count": getattr( + summary, + "no_available_source_year_count", + 0, + ), + "failed_family_count": getattr(summary, "failed_family_count", 0), + "parsed_rows": getattr(summary, "parsed_row_count", 0), + "inserted": getattr(summary, "inserted_count", 0), + "skipped_duplicates": getattr(summary, "skipped_duplicate_count", 0), + }, + "sources": [ + _source_family_payload(family) + for family in getattr(result, "family_results", ()) + ], + "issues": _deduplicated_issue_payloads(result), + "history_persistence_status": getattr( + cycle, + "history_persistence_status", + None, + ), + "history_persistence_issue_count": getattr( + cycle, + "history_persistence_issue_count", + 0, + ), + } + + +def iter_deduplicated_ingestion_issues(result: object) -> tuple[object, ...]: + """Return result issues without duplicating flattened family failures.""" + + issues: list[object] = [] + seen: set[tuple[object, object, object, object]] = set() + + def append_issue(issue: object) -> None: + payload = sanitize_issue_payload(issue) + key = ( + payload.get("source_family"), + payload.get("stage"), + payload.get("code"), + payload.get("message"), + ) + if key in seen: + return + seen.add(key) + issues.append(issue) + + for family in getattr(result, "family_results", ()): + for issue in getattr(family, "failures", ()): + append_issue(issue) + + for issue in getattr(result, "failures", ()): + append_issue(issue) + + return tuple(issues) + + +def _deduplicated_issue_payloads(result: object) -> list[dict[str, object]]: + """Build sanitized issue payloads without duplicating flattened failures.""" + + return [ + sanitize_issue_payload(issue) + for issue in iter_deduplicated_ingestion_issues(result) + ] + + +def sanitize_issue_payload(issue: object | Mapping[str, object]) -> dict[str, object]: + """Build a sanitized JSON-ready issue payload.""" + + return { + "source_family": _attr_or_item(issue, "source_family"), + "stage": _attr_or_item(issue, "stage"), + "code": _attr_or_item(issue, "code"), + "message": redact_sensitive_text(str(_attr_or_item(issue, "message") or "")), + } + + +def _source_family_payload(family: object) -> dict[str, object]: + year_state = getattr(family, "year_state") + insert_summary = getattr(family, "insert_summary", None) + return { + "source_family": getattr(family, "source_family"), + "target_year": getattr(year_state, "target_year"), + "latest_year": getattr(year_state, "latest_year"), + "status": _status_value(getattr(family, "status", None)), + "download_status": configured_download_status_value( + getattr(family, "download_result", None), + ), + "parse_status": configured_parse_status_value(family), + "parsed_rows": getattr(family, "parsed_row_count", 0), + "master_inserted": getattr(insert_summary, "master_inserted", 0), + "master_skipped": getattr(insert_summary, "master_skipped", 0), + "detail_inserted": getattr(insert_summary, "detail_inserted", 0), + "detail_skipped": getattr(insert_summary, "detail_skipped", 0), + } + + +def configured_download_status_value(download_result: object | None) -> str: + """Return the summary-compatible configured download status value.""" + if download_result is None: + return "not_run" + return _status_value(getattr(download_result, "status", "unknown")) + + +def configured_parse_status_value(family: object) -> str: + """Return the summary-compatible configured parse status value.""" + if getattr(family, "parsed_row_count", 0) > 0: + return "parsed" + failures = tuple(getattr(family, "failures", ())) + if any(getattr(failure, "stage", "") == "parser" for failure in failures): + return "failed" + download_result = getattr(family, "download_result", None) + if ( + download_result is None + or configured_download_status_value(download_result) != "downloaded" + ): + return "not_run" + return "no_rows" + + +def _status_value(status: object) -> str: + return str(getattr(status, "value", status)) + + +def _attr_or_item(value: object | Mapping[str, object], key: str) -> object | None: + if isinstance(value, Mapping): + return value.get(key) + return getattr(value, key, None) + + +__all__ = ( + "build_configured_cycle_summary_payload", + "build_configured_runner_summary_payload", + "configured_download_status_value", + "configured_parse_status_value", + "iter_deduplicated_ingestion_issues", + "sanitize_issue_payload", +) diff --git a/src/carbonfactor_parser/diagnostics/redaction.py b/src/carbonfactor_parser/diagnostics/redaction.py new file mode 100644 index 00000000..a1bac9c4 --- /dev/null +++ b/src/carbonfactor_parser/diagnostics/redaction.py @@ -0,0 +1,106 @@ +"""Centralized redaction helpers for operational diagnostics. + +The parser emits local runtime diagnostics for worker operation. This module is a +small safety boundary for those messages: it removes common credential shapes +before text is printed or serialized. It is not intended to be a complete DLP +system. +""" + +from __future__ import annotations + +import re +from urllib.parse import urlsplit, urlunsplit + +_REDACTED_VALUE = "***" +_SENSITIVE_KEYS = frozenset( + ( + "password", + "passwd", + "pwd", + "token", + "secret", + "key", + "api_key", + "apikey", + "access_key", + "accesskey", + "private_key", + "privatekey", + "dsn", + "connection_string", + "connectionstring", + "connection_uri", + "connectionuri", + "database_url", + "databaseurl", + ) +) +_SENSITIVE_COMPACT_KEYS = frozenset(key.replace("_", "") for key in _SENSITIVE_KEYS) +_URL_PATTERN = re.compile(r"(?P[a-z][a-z0-9+.-]*://[^\s'\"<>]+)", re.I) +_ASSIGNMENT_PATTERN = re.compile( + r"(?i)(?(?P[a-z][a-z0-9_-]*)\s*[:=]\s*)" + r"(?P['\"]?)" + r"(?P[^\s,;)}\]\"']+)" + r"(?P=quote)", +) + + +def redact_sensitive_text(value: str) -> str: + """Return ``value`` with common credential-bearing content redacted.""" + + text = str(value) + text = _URL_PATTERN.sub(lambda match: _redact_url_match(match.group("url")), text) + return _ASSIGNMENT_PATTERN.sub(_redact_assignment_match, text) + + +def _redact_url_match(url: str) -> str: + trailing = "" + while url and url[-1] in ".,;)]}": + trailing = url[-1] + trailing + url = url[:-1] + return _redact_url(url) + trailing + + +def _redact_url(url: str) -> str: + try: + parsed = urlsplit(url) + except ValueError: + return url + + netloc = parsed.netloc + if "@" in netloc: + host_port = netloc.rsplit("@", 1)[1] + netloc = f"{_REDACTED_VALUE}@{host_port}" + + query = _redact_query(parsed.query) if parsed.query else parsed.query + + return urlunsplit((parsed.scheme, netloc, parsed.path, query, parsed.fragment)) + + +def _redact_query(query: str) -> str: + redacted_parts = [] + for part in query.split("&"): + key, separator, _value = part.partition("=") + if separator and _is_sensitive_key(key): + redacted_parts.append(f"{key}={_REDACTED_VALUE}") + else: + redacted_parts.append(part) + return "&".join(redacted_parts) + + +def _redact_assignment_match(match: re.Match[str]) -> str: + if not _is_sensitive_key(match.group("key")): + return match.group(0) + quote = match.group("quote") or "" + return f"{match.group('prefix')}{quote}{_REDACTED_VALUE}{quote}" + + +def _is_sensitive_key(key: str) -> bool: + normalized = key.strip().lower().replace("-", "_") + return ( + normalized in _SENSITIVE_KEYS + or normalized.replace("_", "") in _SENSITIVE_COMPACT_KEYS + ) + + +__all__ = ("redact_sensitive_text",) diff --git a/src/carbonfactor_parser/normalization/__init__.py b/src/carbonfactor_parser/normalization/__init__.py index 13a4f9c4..6ef7309d 100644 --- a/src/carbonfactor_parser/normalization/__init__.py +++ b/src/carbonfactor_parser/normalization/__init__.py @@ -6,8 +6,20 @@ NormalizationResult, NormalizedRecord, ) +from carbonfactor_parser.normalization.data_quality_validation import ( + DEFAULT_SUPPORTED_FACTOR_UNITS, + REDACTED_DIAGNOSTIC_VALUE, + DataQualityDiagnostic, + DataQualityProvenanceContext, + DataQualityValidationCheck, + DataQualityValidationResult, + DataQualityValidationSeverity, + create_data_quality_diagnostic, + validate_normalized_factor_output, +) from carbonfactor_parser.normalization.defra_desnz_mapper import ( DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS, + DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS, DefraDesnzNormalizationMappingResult, DefraDesnzNormalizationMappingStatus, map_defra_desnz_normalization_input, @@ -49,7 +61,15 @@ "NormalizationResult", "NormalizationResultSummary", "NormalizedRecord", + "DEFAULT_SUPPORTED_FACTOR_UNITS", + "REDACTED_DIAGNOSTIC_VALUE", + "DataQualityDiagnostic", + "DataQualityProvenanceContext", + "DataQualityValidationCheck", + "DataQualityValidationResult", + "DataQualityValidationSeverity", "DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS", + "DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS", "DefraDesnzNormalizationMappingResult", "DefraDesnzNormalizationMappingStatus", "ArtificialNormalizationExecutor", @@ -70,10 +90,12 @@ "build_normalization_input_from_raw_payload", "build_parser_execution_normalization_handoff", "build_parser_normalization_handoff", + "create_data_quality_diagnostic", "create_normalization_input_from_raw_payload", "create_normalization_input_record_from_raw_record", "map_defra_desnz_normalization_input", "map_defra_desnz_normalization_input_record", "validate_normalization_input", "validate_normalization_input_record", + "validate_normalized_factor_output", ) diff --git a/src/carbonfactor_parser/normalization/data_quality_validation.py b/src/carbonfactor_parser/normalization/data_quality_validation.py new file mode 100644 index 00000000..22e0cb02 --- /dev/null +++ b/src/carbonfactor_parser/normalization/data_quality_validation.py @@ -0,0 +1,622 @@ +"""Phase 2 data quality validation for normalized factor output.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from enum import Enum +import re +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +from carbonfactor_parser.normalization.contracts import ( + NormalizationResult, + NormalizedRecord, +) + + +DEFAULT_SUPPORTED_FACTOR_UNITS = ( + "kg", + "kg CO2e", + "kg CO2e/kWh", + "kWh", +) + +REDACTED_DIAGNOSTIC_VALUE = "[REDACTED]" + +_REQUIRED_FACTOR_FIELDS = ( + "source_family", + "source_id", + "factor_id", + "factor_name", + "factor_value", + "unit", +) + +_PROVENANCE_FIELD_NAMES = ( + "provenance", + "row_number", + "source_document_id", + "document_id", +) + +_SENSITIVE_FIELD_TOKENS = ( + "api_key", + "authorization", + "connection_string", + "credential", + "dsn", + "passwd", + "password", + "pwd", + "secret", + "token", +) + +_SENSITIVE_KEY_VALUE_PATTERN = re.compile( + r"(?i)\b(api[_-]?key|authorization|connection[_-]?string|credential|dsn|passwd|password|pwd|secret|token)\s*[:=]\s*([^\s&;,]+)", +) + + +class DataQualityValidationSeverity(str, Enum): + """Severity levels for normalized factor data quality diagnostics.""" + + BLOCKING_ERROR = "blocking_error" + WARNING = "warning" + INFO = "info" + + +class DataQualityValidationCheck(str, Enum): + """Stable data quality check classifications.""" + + REQUIRED_FIELD = "required_field" + NUMERIC_VALUE = "numeric_value" + UNIT = "unit" + DUPLICATE_FACTOR_IDENTITY = "duplicate_factor_identity" + PROVENANCE = "provenance" + STRUCTURE = "structure" + + +@dataclass(frozen=True) +class DataQualityProvenanceContext: + """Safe row and document context for a normalized factor diagnostic.""" + + record_id: str + source_family: str | None = None + source_id: str | None = None + source_reference: str | None = None + row_number: object | None = None + provenance: str | None = None + document_id: str | None = None + + +@dataclass(frozen=True) +class DataQualityDiagnostic: + """One safe diagnostic emitted by normalized factor validation.""" + + code: str + message: str + severity: DataQualityValidationSeverity + check: DataQualityValidationCheck + field_name: str | None = None + source_family: str | None = None + provenance: DataQualityProvenanceContext | None = None + context: tuple[tuple[str, object], ...] = () + + def __repr__(self) -> str: + safe_context = tuple( + ( + key, + _repr_safe_diagnostic_value(value), + ) + for key, value in self.context + ) + safe_provenance = ( + None + if self.provenance is None + else DataQualityProvenanceContext( + record_id=_repr_safe_diagnostic_value(self.provenance.record_id), + source_family=_repr_safe_diagnostic_value( + self.provenance.source_family + ), + source_id=_repr_safe_diagnostic_value(self.provenance.source_id), + source_reference=_repr_safe_diagnostic_value( + self.provenance.source_reference + ), + row_number=_repr_safe_diagnostic_value(self.provenance.row_number), + provenance=_repr_safe_diagnostic_value(self.provenance.provenance), + document_id=_repr_safe_diagnostic_value(self.provenance.document_id), + ) + ) + return ( + "DataQualityDiagnostic(" + f"code={_repr_safe_diagnostic_value(self.code)!r}, " + f"message={_repr_safe_diagnostic_value(self.message)!r}, " + f"severity={self.severity!r}, " + f"check={self.check!r}, " + f"field_name={_repr_safe_diagnostic_value(self.field_name)!r}, " + f"source_family={_repr_safe_diagnostic_value(self.source_family)!r}, " + f"provenance={safe_provenance!r}, " + f"context={safe_context!r})" + ) + + +@dataclass(frozen=True) +class DataQualityValidationResult: + """Deterministic result for normalized factor data quality validation.""" + + diagnostics: tuple[DataQualityDiagnostic, ...] = () + + @property + def is_valid(self) -> bool: + return not self.has_blocking_errors + + @property + def has_blocking_errors(self) -> bool: + return any( + diagnostic.severity == DataQualityValidationSeverity.BLOCKING_ERROR + for diagnostic in self.diagnostics + ) + + @property + def blocking_error_count(self) -> int: + return _severity_count( + self.diagnostics, + DataQualityValidationSeverity.BLOCKING_ERROR, + ) + + @property + def warning_count(self) -> int: + return _severity_count( + self.diagnostics, + DataQualityValidationSeverity.WARNING, + ) + + @property + def info_count(self) -> int: + return _severity_count(self.diagnostics, DataQualityValidationSeverity.INFO) + + +def create_data_quality_diagnostic( + *, + code: str, + message: str, + severity: DataQualityValidationSeverity, + check: DataQualityValidationCheck, + field_name: str | None = None, + source_family: str | None = None, + provenance: DataQualityProvenanceContext | None = None, + context: Mapping[str, object] | None = None, +) -> DataQualityDiagnostic: + """Create one diagnostic with deterministic redacted context.""" + + return DataQualityDiagnostic( + code=code, + message=message, + severity=severity, + check=check, + field_name=field_name, + source_family=source_family, + provenance=provenance, + context=_safe_context(context), + ) + + +def validate_normalized_factor_output( + normalization_result: NormalizationResult, + *, + supported_units: Iterable[str] = DEFAULT_SUPPORTED_FACTOR_UNITS, +) -> DataQualityValidationResult: + """Validate normalized factor output before persistence or API use.""" + + unit_set = frozenset(supported_units) + diagnostics: list[DataQualityDiagnostic] = [] + identity_positions: dict[tuple[object, ...], int] = {} + + for position, record in enumerate(normalization_result.records, start=1): + fields = dict(record.fields) + provenance = _provenance_context(record, fields) + source_family = _text_or_none(fields.get("source_family")) + + diagnostics.extend( + _missing_required_field_diagnostics( + fields, + position, + source_family, + provenance, + ) + ) + diagnostics.extend( + _invalid_numeric_diagnostics( + fields, + position, + source_family, + provenance, + ) + ) + diagnostics.extend( + _unsupported_unit_diagnostics( + fields, + unit_set, + position, + source_family, + provenance, + ) + ) + diagnostics.extend( + _provenance_gap_diagnostics( + record, + fields, + position, + source_family, + provenance, + ) + ) + + identity = _factor_identity(fields) + if identity is not None: + first_position = identity_positions.get(identity) + if first_position is None: + identity_positions[identity] = position + else: + diagnostics.append( + create_data_quality_diagnostic( + code="NORMALIZED_FACTOR_DUPLICATE_IDENTITY", + message=( + "normalized factor identity must be unique within " + "the validation result." + ), + severity=DataQualityValidationSeverity.BLOCKING_ERROR, + check=DataQualityValidationCheck.DUPLICATE_FACTOR_IDENTITY, + field_name="factor_id", + source_family=source_family, + provenance=provenance, + context={ + "first_record_position": first_position, + "record_position": position, + }, + ) + ) + + return DataQualityValidationResult( + diagnostics=tuple( + sorted( + diagnostics, + key=lambda diagnostic: ( + _context_value(diagnostic, "record_position"), + diagnostic.code, + diagnostic.field_name or "", + ), + ) + ), + ) + + +def _missing_required_field_diagnostics( + fields: Mapping[str, object], + position: int, + source_family: str | None, + provenance: DataQualityProvenanceContext, +) -> tuple[DataQualityDiagnostic, ...]: + diagnostics: list[DataQualityDiagnostic] = [] + for field_name in _REQUIRED_FACTOR_FIELDS: + if _missing_field(fields, field_name): + diagnostics.append( + create_data_quality_diagnostic( + code="NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD", + message=( + "normalized factor output is missing a required field." + ), + severity=DataQualityValidationSeverity.BLOCKING_ERROR, + check=DataQualityValidationCheck.REQUIRED_FIELD, + field_name=field_name, + source_family=source_family, + provenance=provenance, + context={ + "record_position": position, + "field_name": field_name, + }, + ) + ) + return tuple(diagnostics) + + +def _invalid_numeric_diagnostics( + fields: Mapping[str, object], + position: int, + source_family: str | None, + provenance: DataQualityProvenanceContext, +) -> tuple[DataQualityDiagnostic, ...]: + if _missing_field(fields, "factor_value"): + return () + value = fields.get("factor_value") + if _is_valid_numeric(value): + return () + return ( + create_data_quality_diagnostic( + code="NORMALIZED_FACTOR_INVALID_NUMERIC_VALUE", + message="normalized factor_value must be numeric.", + severity=DataQualityValidationSeverity.BLOCKING_ERROR, + check=DataQualityValidationCheck.NUMERIC_VALUE, + field_name="factor_value", + source_family=source_family, + provenance=provenance, + context={ + "record_position": position, + "field_name": "factor_value", + }, + ), + ) + + +def _unsupported_unit_diagnostics( + fields: Mapping[str, object], + supported_units: frozenset[str], + position: int, + source_family: str | None, + provenance: DataQualityProvenanceContext, +) -> tuple[DataQualityDiagnostic, ...]: + if _missing_field(fields, "unit"): + return () + unit = fields.get("unit") + if isinstance(unit, str) and unit.strip() in supported_units: + return () + return ( + create_data_quality_diagnostic( + code="NORMALIZED_FACTOR_UNSUPPORTED_UNIT", + message=( + "normalized factor unit is not in the configured supported " + "unit set." + ), + severity=DataQualityValidationSeverity.WARNING, + check=DataQualityValidationCheck.UNIT, + field_name="unit", + source_family=source_family, + provenance=provenance, + context={ + "record_position": position, + "field_name": "unit", + "supported_unit_count": len(supported_units), + }, + ), + ) + + +def _provenance_gap_diagnostics( + record: NormalizedRecord, + fields: Mapping[str, object], + position: int, + source_family: str | None, + provenance: DataQualityProvenanceContext, +) -> tuple[DataQualityDiagnostic, ...]: + has_field_provenance = any( + not _missing_field(fields, name) for name in _PROVENANCE_FIELD_NAMES + ) + if record.source_reference or has_field_provenance: + return () + return ( + create_data_quality_diagnostic( + code="NORMALIZED_FACTOR_PROVENANCE_GAP", + message=( + "normalized factor output should include row or document " + "provenance before downstream use." + ), + severity=DataQualityValidationSeverity.WARNING, + check=DataQualityValidationCheck.PROVENANCE, + source_family=source_family, + provenance=provenance, + context={"record_position": position}, + ), + ) + + +def _factor_identity(fields: Mapping[str, object]) -> tuple[object, ...] | None: + identity_fields = ( + "source_family", + "source_id", + "source_year", + "source_version", + "factor_id", + "unit", + ) + if any(_missing_field(fields, field_name) for field_name in identity_fields): + return None + return tuple(_identity_value(fields[field_name]) for field_name in identity_fields) + + +def _provenance_context( + record: NormalizedRecord, + fields: Mapping[str, object], +) -> DataQualityProvenanceContext: + return DataQualityProvenanceContext( + record_id=record.record_id, + source_family=_safe_text_or_none(fields.get("source_family")), + source_id=_safe_text_or_none(fields.get("source_id")), + source_reference=_safe_text_or_none(record.source_reference), + row_number=fields.get("row_number"), + provenance=_safe_text_or_none(fields.get("provenance")), + document_id=( + _safe_text_or_none(fields.get("source_document_id")) + or _safe_text_or_none(fields.get("document_id")) + ), + ) + + +def _missing_field(fields: Mapping[str, object], field_name: str) -> bool: + value = fields.get(field_name) + return value is None or (isinstance(value, str) and not value.strip()) + + +def _is_valid_numeric(value: object) -> bool: + if isinstance(value, bool): + return False + if isinstance(value, int | float): + return True + if isinstance(value, str) and value.strip(): + try: + float(value.strip()) + except ValueError: + return False + return True + return False + + +def _identity_value(value: object) -> object: + if isinstance(value, str): + return value.strip() + return value + + +def _text_or_none(value: object) -> str | None: + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _safe_text_or_none(value: object) -> str | None: + text = _text_or_none(value) + if text is None: + return None + return _redact_sensitive_text(text) + + +def _redact_sensitive_text(value: str) -> str: + without_userinfo = _redact_uri_userinfo( + _redact_bare_userinfo_pattern(_redact_uri_userinfo_pattern(value)) + ) + return _SENSITIVE_KEY_VALUE_PATTERN.sub( + lambda match: f"{match.group(1)}={REDACTED_DIAGNOSTIC_VALUE}", + without_userinfo, + ) + + +def _redact_uri_userinfo(value: str) -> str: + try: + parts = urlsplit(value) + except ValueError: + return value + + if not parts.scheme or "@" not in parts.netloc: + return value + + _, separator, hostinfo = parts.netloc.rpartition("@") + if not separator: + return value + return urlunsplit( + ( + parts.scheme, + f"{REDACTED_DIAGNOSTIC_VALUE}@{hostinfo}", + parts.path, + parts.query, + parts.fragment, + ) + ) + + +def _redact_uri_userinfo_pattern(value: str) -> str: + return re.sub( + r"(?i)\b([a-z][a-z0-9+.-]*://)([^\s/?#]*@)", + lambda match: f"{match.group(1)}{REDACTED_DIAGNOSTIC_VALUE}@", + value, + ) + + +def _redact_bare_userinfo_pattern(value: str) -> str: + return re.sub( + r"(? tuple[tuple[str, object], ...]: + if context is None: + return () + return tuple( + (str(key), _safe_diagnostic_value(str(key), value)) + for key, value in sorted(context.items(), key=lambda item: str(item[0])) + ) + + +def _safe_diagnostic_value(field_name: str, value: object) -> object: + if isinstance(value, Mapping): + return tuple( + (str(key), _safe_diagnostic_value(str(key), item)) + for key, item in sorted(value.items(), key=lambda item: str(item[0])) + ) + if isinstance(value, list | tuple): + return tuple(_safe_diagnostic_value(field_name, item) for item in value) + if _is_sensitive_field(field_name): + return REDACTED_DIAGNOSTIC_VALUE if value is not None else None + if isinstance(value, str): + return _safe_text_or_none(value) + if value is None or isinstance(value, bool | int | float): + return value + return _redact_sensitive_text(str(value)) + + +def _repr_safe_diagnostic_value(value: object) -> object: + if isinstance(value, Mapping): + return tuple( + ( + _repr_safe_context_key(str(key)), + _safe_diagnostic_value(str(key), item) + if _is_sensitive_field(str(key)) + else _repr_safe_diagnostic_value(item), + ) + for key, item in sorted(value.items(), key=lambda item: str(item[0])) + ) + if isinstance(value, tuple): + if all(isinstance(item, tuple) and len(item) == 2 for item in value): + return tuple( + ( + _repr_safe_context_key(str(item[0])), + _repr_safe_diagnostic_value(item[1]), + ) + for item in value + ) + return tuple(_repr_safe_diagnostic_value(item) for item in value) + if isinstance(value, list): + return tuple(_repr_safe_diagnostic_value(item) for item in value) + if isinstance(value, str): + return _safe_text_or_none(value) + if value is None or isinstance(value, bool | int | float | Enum): + return value + return _redact_sensitive_text(str(value)) + + +def _is_sensitive_field(field_name: str) -> bool: + normalized = field_name.lower() + return any(token in normalized for token in _SENSITIVE_FIELD_TOKENS) + + +def _repr_safe_context_key(field_name: str) -> str: + normalized = field_name.lower() + if any( + token in normalized + for token in ( + "api_key", + "authorization", + "credential", + "passwd", + "password", + "pwd", + "secret", + "token", + ) + ): + return REDACTED_DIAGNOSTIC_VALUE + return field_name + + +def _context_value(diagnostic: DataQualityDiagnostic, key: str) -> object: + return dict(diagnostic.context).get(key, 0) + + +def _severity_count( + diagnostics: tuple[DataQualityDiagnostic, ...], + severity: DataQualityValidationSeverity, +) -> int: + return sum(diagnostic.severity == severity for diagnostic in diagnostics) diff --git a/src/carbonfactor_parser/normalization/defra_desnz_mapper.py b/src/carbonfactor_parser/normalization/defra_desnz_mapper.py index 18cf265e..d5b70241 100644 --- a/src/carbonfactor_parser/normalization/defra_desnz_mapper.py +++ b/src/carbonfactor_parser/normalization/defra_desnz_mapper.py @@ -23,6 +23,43 @@ "unit", ) +DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS = ( + "source_family", + "source_id", + "source_year", + "source_version", + "record_index", + "row_number", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", + "subcategory", + "activity", + "greenhouse_gas", + "provenance", +) + +_DEFRA_DESNZ_REQUIRED_NORMALIZED_FIELDS = ( + "source_year", + "source_version", + "category", + "factor_id", + "factor_name", + "factor_value", + "unit", + "provenance", +) + +_DEFRA_DESNZ_NORMALIZED_ONLY_FIELDS = ( + "source_year", + "source_version", + "category", + "factor_value", + "provenance", +) + class DefraDesnzNormalizationMappingStatus(str, Enum): """Status for minimal DEFRA/DESNZ fixture normalization mapping.""" @@ -115,13 +152,14 @@ def _record_issues(record: NormalizationInputRecord) -> tuple[NormalizationIssue ), ) - for field_name in DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS: + required_fields = _required_fields(record) + for field_name in required_fields: if _missing_raw_field(record, field_name): issues.append( NormalizationIssue( code="DEFRA_DESNZ_NORMALIZATION_MISSING_RAW_FIELD", message=( - "DEFRA/DESNZ minimal normalization input is missing " + "DEFRA/DESNZ normalization input is missing " f"required raw field: {field_name}." ), severity=NormalizationIssueSeverity.ERROR, @@ -129,10 +167,29 @@ def _record_issues(record: NormalizationInputRecord) -> tuple[NormalizationIssue ), ) + if _is_normalized_extraction_record(record) and not _missing_raw_field( + record, + "factor_value", + ): + try: + float(str(record.raw_fields["factor_value"]).strip()) + except ValueError: + issues.append( + NormalizationIssue( + code="DEFRA_DESNZ_NORMALIZATION_INVALID_FACTOR_VALUE", + message="DEFRA/DESNZ factor_value must be numeric.", + severity=NormalizationIssueSeverity.ERROR, + location=_record_location(record, "factor_value"), + ), + ) + return tuple(issues) def _normalized_record(record: NormalizationInputRecord) -> NormalizedRecord: + if _is_normalized_extraction_record(record): + return _normalized_extraction_record(record) + return NormalizedRecord( record_id=( f"{record.source_family}:{record.source_id}:" @@ -152,6 +209,50 @@ def _normalized_record(record: NormalizationInputRecord) -> NormalizedRecord: ) +def _normalized_extraction_record(record: NormalizationInputRecord) -> NormalizedRecord: + raw_fields = record.raw_fields + return NormalizedRecord( + record_id=( + f"{record.source_family}:{record.source_id}:" + f"{_text(raw_fields['source_year'])}:" + f"{_text(raw_fields['source_version'])}:" + f"{_text(raw_fields['factor_id'])}" + ), + fields=( + ("source_family", record.source_family), + ("source_id", record.source_id), + ("source_year", _text(raw_fields["source_year"])), + ("source_version", _text(raw_fields["source_version"])), + ("record_index", record.record_index), + ("row_number", record.row_number), + ("factor_id", _text(raw_fields["factor_id"])), + ("factor_name", _text(raw_fields["factor_name"])), + ("factor_value", float(_text(raw_fields["factor_value"]))), + ("unit", _text(raw_fields["unit"])), + ("category", _text(raw_fields["category"])), + ("subcategory", _optional_text(raw_fields.get("subcategory"))), + ("activity", _optional_text(raw_fields.get("activity"))), + ("greenhouse_gas", _optional_text(raw_fields.get("greenhouse_gas"))), + ("provenance", _text(raw_fields["provenance"])), + ), + source_reference=_source_reference(record), + is_artificial=False, + ) + + +def _required_fields(record: NormalizationInputRecord) -> tuple[str, ...]: + if _is_normalized_extraction_record(record): + return _DEFRA_DESNZ_REQUIRED_NORMALIZED_FIELDS + return DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS + + +def _is_normalized_extraction_record(record: NormalizationInputRecord) -> bool: + return any( + field_name in record.raw_fields + for field_name in _DEFRA_DESNZ_NORMALIZED_ONLY_FIELDS + ) + + def _missing_raw_field( record: NormalizationInputRecord, field_name: str, @@ -173,3 +274,14 @@ def _source_reference(record: NormalizationInputRecord) -> str | None: if isinstance(artifact_reference, str) and artifact_reference: return artifact_reference return None + + +def _text(value: object) -> str: + return str(value).strip() + + +def _optional_text(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/src/carbonfactor_parser/parsers/__init__.py b/src/carbonfactor_parser/parsers/__init__.py index 824feec7..9c236436 100644 --- a/src/carbonfactor_parser/parsers/__init__.py +++ b/src/carbonfactor_parser/parsers/__init__.py @@ -10,10 +10,15 @@ "ArtificialFixtureParser": "fixture_parser", "ArtificialParserAdapter": "artificial_adapter", "DEFRA_DESNZ_MINIMAL_CONTENT_HEADER": "defra_desnz_content_parser", + "DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER": "defra_desnz_content_parser", "DefraDesnzParserAdapter": "defra_desnz_adapter", "DefraDesnzParser": "defra_desnz_parser", "ExampleInMemoryParser": "example_parser", "ExampleSourceSpecificParser": "example_source_specific_parser", + "GHGProtocolParserAdapter": "ghg_protocol_adapter", + "GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER": "ghg_protocol_content_parser", + "IPCC_EFDB_NORMALIZED_CONTENT_HEADER": "ipcc_efdb_content_parser", + "IpccEfdbParserAdapter": "ipcc_efdb_adapter", "DEFAULT_PARSER_FILE_CONTENT_MAX_BYTES": "file_content_loader", "NoopParserAdapter": "noop_adapter", "ParserAdapter": "adapter", @@ -54,6 +59,8 @@ "load_parser_file_content_from_local_path": "file_content_loader", "plan_parser_execution": "execution_plan", "parse_defra_desnz_file_content": "defra_desnz_content_parser", + "parse_ghg_protocol_file_content": "ghg_protocol_content_parser", + "parse_ipcc_efdb_file_content": "ipcc_efdb_content_parser", "register_parser_adapter": "adapter_registry", "resolve_parser_adapters": "adapter_registry", "run_parser_execution": "execution_runner", @@ -76,6 +83,10 @@ "defra_desnz_parser", "example_parser", "example_source_specific_parser", + "ghg_protocol_adapter", + "ghg_protocol_content_parser", + "ipcc_efdb_adapter", + "ipcc_efdb_content_parser", "execution_plan", "execution_result", "execution_runner", diff --git a/src/carbonfactor_parser/parsers/contract_api.py b/src/carbonfactor_parser/parsers/contract_api.py index 0e73ff43..f4ddc8d2 100644 --- a/src/carbonfactor_parser/parsers/contract_api.py +++ b/src/carbonfactor_parser/parsers/contract_api.py @@ -74,6 +74,15 @@ validate_parser_run_request, validate_parser_run_result, ) +from carbonfactor_parser.parsers.run_repository_contract import ( + ParserRunRepository, + ParserRunRepositoryIssue, + ParserRunRepositoryPersistResult, + ParserRunRepositoryPersistStatus, + ParserRunRepositoryValidationResult, + create_parser_run_repository_persist_result, + validate_parser_run_repository_inputs, +) from carbonfactor_parser.parsers.validation_issue_contract import ( ParserValidationIssue, ParserValidationIssueCollection, @@ -117,6 +126,11 @@ "ParserRunContractValidationIssue", "ParserRunContractValidationResult", "ParserRunRequest", + "ParserRunRepository", + "ParserRunRepositoryIssue", + "ParserRunRepositoryPersistResult", + "ParserRunRepositoryPersistStatus", + "ParserRunRepositoryValidationResult", "ParserRunResult", "ParserRunStatus", "ParserRunSummary", @@ -130,6 +144,7 @@ "build_phase1_parser_adapter_readiness_report", "create_parser_normalized_output_batch", "create_parser_normalized_output_row", + "create_parser_run_repository_persist_result", "create_parser_run_request", "create_parser_run_result", "create_parser_validation_issue", @@ -147,6 +162,7 @@ "validate_parser_input_artifact", "validate_parser_normalized_output_batch", "validate_parser_normalized_output_row", + "validate_parser_run_repository_inputs", "validate_parser_run_request", "validate_parser_run_result", "validate_parser_validation_issue", diff --git a/src/carbonfactor_parser/parsers/defra_desnz_content_parser.py b/src/carbonfactor_parser/parsers/defra_desnz_content_parser.py index fb68ff31..c9737268 100644 --- a/src/carbonfactor_parser/parsers/defra_desnz_content_parser.py +++ b/src/carbonfactor_parser/parsers/defra_desnz_content_parser.py @@ -3,6 +3,7 @@ from __future__ import annotations import csv +from decimal import Decimal, InvalidOperation from io import StringIO from carbonfactor_parser.parsers.execution_result import ( @@ -29,6 +30,31 @@ "unit", ) +DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER = ( + "source_year", + "source_version", + "category", + "subcategory", + "activity", + "factor_id", + "factor_name", + "factor_value", + "unit", + "greenhouse_gas", + "provenance", +) + +_DEFRA_DESNZ_REQUIRED_NORMALIZED_FIELDS = ( + "source_year", + "source_version", + "category", + "factor_id", + "factor_name", + "factor_value", + "unit", + "provenance", +) + def parse_defra_desnz_file_content( content_input: ParserFileContentInput, @@ -107,7 +133,10 @@ def _parse_minimal_csv( parser_input, ) -> ParserExecutionResult: reader = csv.DictReader(StringIO(content_text)) - if tuple(reader.fieldnames or ()) != DEFRA_DESNZ_MINIMAL_CONTENT_HEADER: + fieldnames = tuple(reader.fieldnames or ()) + if fieldnames == DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER: + return _parse_normalized_csv(reader, parser_input) + if fieldnames != DEFRA_DESNZ_MINIMAL_CONTENT_HEADER: return create_parser_execution_result( status=ParserExecutionResultStatus.FAILED, parser_input=parser_input, @@ -115,8 +144,8 @@ def _parse_minimal_csv( ParserExecutionIssue( code="DEFRA_DESNZ_CONTENT_INVALID_HEADER", message=( - "DEFRA/DESNZ minimal content header must be " - "factor_id,factor_name,unit." + "DEFRA/DESNZ content header must match either the " + "minimal fixture shape or normalized extraction shape." ), severity=ParserExecutionIssueSeverity.ERROR, location="header", @@ -190,6 +219,222 @@ def _parse_minimal_csv( ) +def _parse_normalized_csv( + reader: csv.DictReader, + parser_input, +) -> ParserExecutionResult: + parser_metadata = _parser_metadata( + parser_kind="defra_desnz_normalized_csv_extraction", + is_real_source_parser=True, + normalization_executed=True, + ) + issues: list[ParserExecutionIssue] = [] + raw_records = [] + parsed_record_count = 0 + + for row_number, row in enumerate(reader, start=2): + if None in row: + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="DEFRA_DESNZ_CONTENT_INVALID_ROW", + message="DEFRA/DESNZ normalized content row has an unexpected column count.", + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}]", + ), + ), + parser_metadata=parser_metadata, + ) + if not any((value or "").strip() for value in row.values()): + continue + + row_issues = _normalized_row_issues(row, row_number) + issues.extend(row_issues) + if row_issues: + continue + + parsed_record_count += 1 + raw_records.append( + create_parsed_raw_record( + source_family=parser_input.source_family, + source_id=parser_input.source_id, + record_index=parsed_record_count, + row_number=row_number, + raw_fields=_normalized_raw_fields(row, parser_input, row_number), + parser_metadata=parser_metadata, + source_context={ + "artifact_reference": parser_input.artifact_reference, + "source_year": (row.get("source_year") or "").strip(), + "source_version": (row.get("source_version") or "").strip(), + "provenance": (row.get("provenance") or "").strip(), + }, + ), + ) + + if issues: + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + parsed_record_count=parsed_record_count, + issues=tuple(issues), + parser_metadata=parser_metadata, + raw_record_payload=( + create_parsed_raw_record_payload( + source_family=parser_input.source_family, + source_id=parser_input.source_id, + records=tuple(raw_records), + parser_metadata=parser_metadata, + source_context={ + "artifact_reference": parser_input.artifact_reference, + }, + ) + if raw_records + else None + ), + ) + + if parsed_record_count == 0: + return create_parser_execution_result( + status=ParserExecutionResultStatus.NO_RECORDS, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="DEFRA_DESNZ_CONTENT_NO_RECORDS", + message="DEFRA/DESNZ normalized content header was present but no rows were parsed.", + severity=ParserExecutionIssueSeverity.WARNING, + location="content", + ), + ), + parser_metadata=parser_metadata, + ) + + return create_parser_execution_result( + status=ParserExecutionResultStatus.SUCCESS, + parser_input=parser_input, + parsed_record_count=parsed_record_count, + parser_metadata=parser_metadata, + raw_record_payload=create_parsed_raw_record_payload( + source_family=parser_input.source_family, + source_id=parser_input.source_id, + records=tuple(raw_records), + parser_metadata=parser_metadata, + source_context={ + "artifact_reference": parser_input.artifact_reference, + }, + ), + ) + + +def _normalized_row_issues( + row: dict[str, str], + row_number: int, +) -> tuple[ParserExecutionIssue, ...]: + issues: list[ParserExecutionIssue] = [] + for field_name in _DEFRA_DESNZ_REQUIRED_NORMALIZED_FIELDS: + if not (row.get(field_name) or "").strip(): + issues.append( + ParserExecutionIssue( + code="DEFRA_DESNZ_CONTENT_MISSING_REQUIRED_FIELD", + message=( + "DEFRA/DESNZ normalized content row is missing " + f"required field: {field_name}." + ), + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}].{field_name}", + context={"row_number": row_number, "field_name": field_name}, + ), + ) + + source_year = (row.get("source_year") or "").strip() + parsed_source_year = int(source_year) if source_year.isdecimal() else 0 + if parsed_source_year < 1: + issues.append( + ParserExecutionIssue( + code="DEFRA_DESNZ_CONTENT_INVALID_SOURCE_YEAR", + message="DEFRA/DESNZ source_year must be a positive integer.", + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}].source_year", + context={ + "row_number": row_number, + "field_name": "source_year", + "raw_value": source_year, + }, + ), + ) + + factor_value = (row.get("factor_value") or "").strip() + parsed_factor_value = _parse_factor_decimal(factor_value) + if parsed_factor_value is None or not parsed_factor_value.is_finite(): + issues.append( + ParserExecutionIssue( + code="DEFRA_DESNZ_CONTENT_INVALID_FACTOR_VALUE", + message="DEFRA/DESNZ normalized factor_value must be numeric.", + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}].factor_value", + context={ + "row_number": row_number, + "field_name": "factor_value", + "raw_value": factor_value, + }, + ), + ) + + return tuple(issues) + + +def _parse_factor_decimal(raw_value: str) -> Decimal | None: + if "e" in raw_value.lower(): + return None + try: + parsed_value = Decimal(raw_value.replace(",", "")) + except InvalidOperation: + return None + return parsed_value if parsed_value.is_finite() else None + + +def _normalized_raw_fields( + row: dict[str, str], + parser_input, + row_number: int, +) -> dict[str, object]: + normalized_row = {key: (value or "").strip() for key, value in row.items()} + source_year = normalized_row["source_year"] + source_version = normalized_row["source_version"] + factor_id = normalized_row["factor_id"] + unit = normalized_row["unit"] + greenhouse_gas = normalized_row["greenhouse_gas"] + master_id = f"defra_master_{source_year}_{source_version}_{factor_id}" + detail_id = f"defra_detail_{source_year}_{source_version}_{factor_id}" + + return { + "source_family": parser_input.source_family, + "source_id": parser_input.source_id, + "source_year": int(source_year), + "source_version": source_version, + "factor_id": factor_id, + "factor_name": normalized_row["factor_name"], + "factor_value": _parse_factor_decimal(normalized_row["factor_value"]), + "unit": unit, + "category": normalized_row["category"], + "subcategory": normalized_row["subcategory"] or None, + "activity": normalized_row["activity"] or None, + "greenhouse_gas": greenhouse_gas or None, + "provenance_artifact_reference": parser_input.artifact_reference, + "provenance_checksum_algorithm": "sha256" + if parser_input.checksum_sha256 + else None, + "provenance_checksum_value": parser_input.checksum_sha256, + "provenance_row_number": row_number, + "provenance": normalized_row["provenance"], + "source_family_master_id": master_id, + "source_family_detail_id": detail_id, + "master_external_key": f"{source_year}:{source_version}:{factor_id}", + "detail_external_key": f"{factor_id}:{unit}:{greenhouse_gas}", + } + + def _content_text(content_input: ParserFileContentInput) -> str | None: if isinstance(content_input.content, str): return content_input.content @@ -217,9 +462,14 @@ def _parser_input_from_content_input(content_input: ParserFileContentInput): ) -def _parser_metadata() -> dict[str, object]: +def _parser_metadata( + *, + parser_kind: str = "minimal_defra_desnz_content_fixture", + is_real_source_parser: bool = False, + normalization_executed: bool = False, +) -> dict[str, object]: return { - "parser_kind": "minimal_defra_desnz_content_fixture", - "is_real_source_parser": False, - "normalization_executed": False, + "parser_kind": parser_kind, + "is_real_source_parser": is_real_source_parser, + "normalization_executed": normalization_executed, } diff --git a/src/carbonfactor_parser/parsers/ghg_protocol_adapter.py b/src/carbonfactor_parser/parsers/ghg_protocol_adapter.py new file mode 100644 index 00000000..264559a6 --- /dev/null +++ b/src/carbonfactor_parser/parsers/ghg_protocol_adapter.py @@ -0,0 +1,69 @@ +"""GHG Protocol parser adapter.""" + +from __future__ import annotations + +from carbonfactor_parser.parsers.execution_result import ( + ParserExecutionIssue, + ParserExecutionIssueSeverity, + ParserExecutionResult, + ParserExecutionResultStatus, + create_parser_execution_result, +) +from carbonfactor_parser.parsers.file_content_input import ParserFileContentInput +from carbonfactor_parser.parsers.ghg_protocol_content_parser import ( + parse_ghg_protocol_file_content, +) +from carbonfactor_parser.parsers.input_contract import ParserInputContract + + +class GHGProtocolParserAdapter: + """GHG Protocol parser adapter for already-loaded content parsing.""" + + source_family = "ghg_protocol" + supported_content_types = ( + "text/csv", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + supported_format_hints = ("csv", "xlsx") + + def can_parse(self, parser_input: ParserInputContract) -> bool: + """Return True for matching GHG Protocol parser input metadata.""" + + if parser_input.source_family != self.source_family: + return False + if parser_input.content_type in self.supported_content_types: + return True + return parser_input.format_hint in self.supported_format_hints + + def parse(self, parser_input: ParserInputContract) -> ParserExecutionResult: + """Return unsupported until artifact file loading is explicitly wired.""" + + return create_parser_execution_result( + status=ParserExecutionResultStatus.UNSUPPORTED, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="GHG_PROTOCOL_PARSER_REQUIRES_LOADED_CONTENT", + message=( + "GHG Protocol parser adapter requires caller-provided " + "content via parse_content." + ), + severity=ParserExecutionIssueSeverity.WARNING, + location="ghg_protocol_parser_adapter", + ), + ), + parser_metadata={ + "adapter_kind": "source_specific_content_parser", + "is_real_source_parser": True, + "real_parsing_implemented": True, + "requires_loaded_content": True, + }, + ) + + def parse_content( + self, + content_input: ParserFileContentInput, + ) -> ParserExecutionResult: + """Parse already-loaded GHG Protocol content.""" + + return parse_ghg_protocol_file_content(content_input) diff --git a/src/carbonfactor_parser/parsers/ghg_protocol_content_parser.py b/src/carbonfactor_parser/parsers/ghg_protocol_content_parser.py new file mode 100644 index 00000000..80e2d4b2 --- /dev/null +++ b/src/carbonfactor_parser/parsers/ghg_protocol_content_parser.py @@ -0,0 +1,384 @@ +"""GHG Protocol content parser for deterministic normalized factor rows.""" + +from __future__ import annotations + +import csv +from decimal import Decimal, InvalidOperation +from io import StringIO + +from carbonfactor_parser.parsers.execution_result import ( + ParserExecutionIssue, + ParserExecutionIssueSeverity, + ParserExecutionResult, + ParserExecutionResultStatus, + create_parser_execution_result, +) +from carbonfactor_parser.parsers.file_content_input import ( + ParserFileContentInput, + validate_parser_file_content_input, +) +from carbonfactor_parser.parsers.input_contract import create_parser_input_contract +from carbonfactor_parser.parsers.raw_record import ( + create_parsed_raw_record, + create_parsed_raw_record_payload, +) + + +GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER = ( + "record_type", + "source_year", + "source_version", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", + "subcategory", + "scope", + "gas", + "provenance_note", +) + +_REQUIRED_FIELDS = ( + "source_year", + "source_version", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", +) + + +def parse_ghg_protocol_file_content( + content_input: ParserFileContentInput, +) -> ParserExecutionResult: + """Parse already-loaded GHG Protocol normalized CSV fixture content.""" + + parser_input = _parser_input_from_content_input(content_input) + validation_result = validate_parser_file_content_input(content_input) + if not validation_result.is_valid: + if _only_missing_content(validation_result): + return create_parser_execution_result( + status=ParserExecutionResultStatus.NO_RECORDS, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="GHG_PROTOCOL_CONTENT_EMPTY", + message="GHG Protocol content input did not include parseable content.", + severity=ParserExecutionIssueSeverity.WARNING, + location="content", + ), + ), + parser_metadata=_parser_metadata(), + ) + + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + issues=tuple( + ParserExecutionIssue( + code=issue.code, + message=issue.message, + severity=ParserExecutionIssueSeverity.ERROR, + location=issue.field_name, + ) + for issue in validation_result.issues + ), + parser_metadata=_parser_metadata(), + ) + + if content_input.source_family != "ghg_protocol": + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="GHG_PROTOCOL_CONTENT_SOURCE_FAMILY_MISMATCH", + message="GHG Protocol content parser only accepts ghg_protocol source_family.", + severity=ParserExecutionIssueSeverity.ERROR, + location="source_family", + ), + ), + parser_metadata=_parser_metadata(), + ) + + content_text = _content_text(content_input) + if content_text is None: + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="GHG_PROTOCOL_CONTENT_BYTES_DECODE_FAILED", + message="GHG Protocol bytes content must be UTF-8 CSV text.", + severity=ParserExecutionIssueSeverity.ERROR, + location="content", + ), + ), + parser_metadata=_parser_metadata(), + ) + + return _parse_normalized_csv(content_text, parser_input) + + +def _parse_normalized_csv(content_text: str, parser_input) -> ParserExecutionResult: + reader = csv.DictReader(StringIO(content_text)) + if tuple(reader.fieldnames or ()) != GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER: + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="GHG_PROTOCOL_CONTENT_INVALID_HEADER", + message=( + "GHG Protocol normalized content header must match the " + "declared parser contract." + ), + severity=ParserExecutionIssueSeverity.ERROR, + location="header", + context={ + "expected_header": GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER, + }, + ), + ), + parser_metadata=_parser_metadata(), + ) + + issues: list[ParserExecutionIssue] = [] + raw_records = [] + parsed_record_count = 0 + skipped_record_count = 0 + + for row_number, row in enumerate(reader, start=2): + if None in row: + issues.append( + ParserExecutionIssue( + code="GHG_PROTOCOL_CONTENT_INVALID_ROW", + message="GHG Protocol content row has an unexpected column count.", + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}]", + context={"row_number": row_number}, + ), + ) + continue + + normalized_row = _trim_row(row) + if not any(normalized_row.values()): + continue + + if normalized_row["record_type"] != "emission_factor": + skipped_record_count += 1 + issues.append( + ParserExecutionIssue( + code="GHG_PROTOCOL_CONTENT_UNSUPPORTED_ROW_SKIPPED", + message="GHG Protocol content row was skipped because record_type is unsupported.", + severity=ParserExecutionIssueSeverity.WARNING, + location=f"row[{row_number}].record_type", + context={ + "row_number": row_number, + "record_type": normalized_row["record_type"], + }, + ), + ) + continue + + row_issues = _row_issues(normalized_row, row_number) + issues.extend(row_issues) + if row_issues: + continue + + parsed_record_count += 1 + raw_records.append( + create_parsed_raw_record( + source_family=parser_input.source_family, + source_id=parser_input.source_id, + record_index=parsed_record_count, + row_number=row_number, + raw_fields=_normalized_raw_fields( + normalized_row, + parser_input, + row_number, + ), + parser_metadata=_parser_metadata(skipped_record_count), + source_context=_source_context(parser_input, row_number), + ), + ) + + if any(issue.severity == ParserExecutionIssueSeverity.ERROR for issue in issues): + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + issues=tuple(issues), + parser_metadata=_parser_metadata(skipped_record_count), + ) + + if parsed_record_count == 0: + return create_parser_execution_result( + status=ParserExecutionResultStatus.NO_RECORDS, + parser_input=parser_input, + issues=( + *tuple(issues), + ParserExecutionIssue( + code="GHG_PROTOCOL_CONTENT_NO_RECORDS", + message="GHG Protocol normalized content included no parseable emission factor rows.", + severity=ParserExecutionIssueSeverity.WARNING, + location="content", + ), + ), + parser_metadata=_parser_metadata(skipped_record_count), + ) + + return create_parser_execution_result( + status=ParserExecutionResultStatus.SUCCESS, + parser_input=parser_input, + parsed_record_count=parsed_record_count, + issues=tuple(issues), + parser_metadata=_parser_metadata(skipped_record_count), + raw_record_payload=create_parsed_raw_record_payload( + source_family=parser_input.source_family, + source_id=parser_input.source_id, + records=tuple(raw_records), + parser_metadata=_parser_metadata(skipped_record_count), + source_context={ + "artifact_reference": parser_input.artifact_reference, + "checksum_sha256": parser_input.checksum_sha256, + }, + ), + ) + + +def _trim_row(row: dict[str | None, str | None]) -> dict[str, str]: + return { + field_name: (row.get(field_name) or "").strip() + for field_name in GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER + } + + +def _row_issues(row: dict[str, str], row_number: int) -> tuple[ParserExecutionIssue, ...]: + issues: list[ParserExecutionIssue] = [] + for field_name in _REQUIRED_FIELDS: + if not row[field_name]: + issues.append( + ParserExecutionIssue( + code="GHG_PROTOCOL_CONTENT_MISSING_REQUIRED_FIELD", + message=( + "GHG Protocol emission factor row is missing required " + f"field: {field_name}." + ), + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}].{field_name}", + context={"row_number": row_number, "field_name": field_name}, + ), + ) + + try: + source_year = int(row["source_year"]) + except ValueError: + source_year = 0 + if source_year < 1: + issues.append( + ParserExecutionIssue( + code="GHG_PROTOCOL_CONTENT_INVALID_SOURCE_YEAR", + message="GHG Protocol source_year must be a positive integer.", + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}].source_year", + context={"row_number": row_number}, + ), + ) + + try: + Decimal(row["factor_value"]) + except InvalidOperation: + issues.append( + ParserExecutionIssue( + code="GHG_PROTOCOL_CONTENT_INVALID_FACTOR_VALUE", + message="GHG Protocol factor_value must be a decimal number.", + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}].factor_value", + context={"row_number": row_number}, + ), + ) + + return tuple(issues) + + +def _normalized_raw_fields( + row: dict[str, str], + parser_input, + row_number: int, +) -> dict[str, object]: + master_id = f"ghg_master_{row['source_year']}_{row['source_version']}_{row['factor_id']}" + detail_id = f"ghg_detail_{row['source_year']}_{row['source_version']}_{row['factor_id']}" + + return { + "source_family": parser_input.source_family, + "source_id": parser_input.source_id, + "source_year": int(row["source_year"]), + "source_version": row["source_version"], + "factor_id": row["factor_id"], + "factor_name": row["factor_name"], + "factor_value": Decimal(row["factor_value"]), + "unit": row["unit"], + "category": row["category"], + "subcategory": row["subcategory"] or None, + "scope": row["scope"] or None, + "gas": row["gas"] or None, + "provenance_note": row["provenance_note"] or None, + "provenance_artifact_reference": parser_input.artifact_reference, + "provenance_checksum_algorithm": "sha256" + if parser_input.checksum_sha256 + else None, + "provenance_checksum_value": parser_input.checksum_sha256, + "provenance_row_number": row_number, + "source_family_master_id": master_id, + "source_family_detail_id": detail_id, + "master_external_key": ( + f"{row['source_year']}:{row['source_version']}:{row['factor_id']}" + ), + "detail_external_key": f"{row['factor_id']}:{row['unit']}", + } + + +def _content_text(content_input: ParserFileContentInput) -> str | None: + if isinstance(content_input.content, str): + return content_input.content + try: + return content_input.content.decode("utf-8") + except UnicodeDecodeError: + return None + + +def _only_missing_content(validation_result) -> bool: + return tuple(issue.code for issue in validation_result.issues) == ( + "PARSER_FILE_CONTENT_MISSING_CONTENT", + ) + + +def _parser_input_from_content_input(content_input: ParserFileContentInput): + return create_parser_input_contract( + source_family=content_input.source_family, + source_id=content_input.source_id, + acquisition_status="content_loaded", + artifact_reference=content_input.artifact_reference or "memory://parser-file-content-input", + checksum_sha256=content_input.checksum_sha256, + content_type=content_input.content_type, + format_hint=content_input.format_hint, + ) + + +def _source_context(parser_input, row_number: int) -> dict[str, object]: + return { + "artifact_reference": parser_input.artifact_reference, + "checksum_sha256": parser_input.checksum_sha256, + "row_number": row_number, + } + + +def _parser_metadata(skipped_record_count: int = 0) -> dict[str, object]: + return { + "parser_kind": "ghg_protocol_normalized_content", + "is_real_source_parser": True, + "normalization_executed": True, + "skipped_record_count": skipped_record_count, + } diff --git a/src/carbonfactor_parser/parsers/ipcc_efdb_adapter.py b/src/carbonfactor_parser/parsers/ipcc_efdb_adapter.py new file mode 100644 index 00000000..3c1eecfa --- /dev/null +++ b/src/carbonfactor_parser/parsers/ipcc_efdb_adapter.py @@ -0,0 +1,66 @@ +"""IPCC EFDB parser adapter.""" + +from __future__ import annotations + +from carbonfactor_parser.parsers.execution_result import ( + ParserExecutionIssue, + ParserExecutionIssueSeverity, + ParserExecutionResult, + ParserExecutionResultStatus, + create_parser_execution_result, +) +from carbonfactor_parser.parsers.file_content_input import ParserFileContentInput +from carbonfactor_parser.parsers.input_contract import ParserInputContract +from carbonfactor_parser.parsers.ipcc_efdb_content_parser import ( + parse_ipcc_efdb_file_content, +) + + +class IpccEfdbParserAdapter: + """IPCC EFDB parser adapter for already-loaded content parsing.""" + + source_family = "ipcc_efdb" + supported_content_types = ("text/csv",) + supported_format_hints = ("csv",) + + def can_parse(self, parser_input: ParserInputContract) -> bool: + """Return True for matching IPCC EFDB parser input metadata.""" + + if parser_input.source_family != self.source_family: + return False + if parser_input.content_type in self.supported_content_types: + return True + return parser_input.format_hint in self.supported_format_hints + + def parse(self, parser_input: ParserInputContract) -> ParserExecutionResult: + """Return unsupported until artifact file loading is explicitly wired.""" + + return create_parser_execution_result( + status=ParserExecutionResultStatus.UNSUPPORTED, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="IPCC_EFDB_PARSER_REQUIRES_LOADED_CONTENT", + message=( + "IPCC EFDB parser adapter requires caller-provided " + "content via parse_content." + ), + severity=ParserExecutionIssueSeverity.WARNING, + location="ipcc_efdb_parser_adapter", + ), + ), + parser_metadata={ + "adapter_kind": "source_specific_content_parser", + "is_real_source_parser": True, + "real_parsing_implemented": True, + "requires_loaded_content": True, + }, + ) + + def parse_content( + self, + content_input: ParserFileContentInput, + ) -> ParserExecutionResult: + """Parse already-loaded IPCC EFDB content.""" + + return parse_ipcc_efdb_file_content(content_input) diff --git a/src/carbonfactor_parser/parsers/ipcc_efdb_adapter_contract.py b/src/carbonfactor_parser/parsers/ipcc_efdb_adapter_contract.py index 0272ba5b..0c18e7e2 100644 --- a/src/carbonfactor_parser/parsers/ipcc_efdb_adapter_contract.py +++ b/src/carbonfactor_parser/parsers/ipcc_efdb_adapter_contract.py @@ -22,6 +22,7 @@ class ParserAdapterSkeletonReadiness(str, Enum): """Runtime-passive parser adapter skeleton readiness values.""" CONTRACT_ONLY = "contract_only" + CONTENT_PARSER_READY = "content_parser_ready" @dataclass(frozen=True) @@ -39,7 +40,7 @@ class ParserAdapterCapability: @dataclass(frozen=True) class IpccEfdbParserAdapterDescriptor: - """Runtime-passive IPCC EFDB parser adapter skeleton descriptor.""" + """Runtime-passive IPCC EFDB parser adapter descriptor.""" source_family: str parser_key: str @@ -49,22 +50,22 @@ class IpccEfdbParserAdapterDescriptor: def describe_ipcc_efdb_parser_adapter() -> IpccEfdbParserAdapterDescriptor: - """Return deterministic IPCC EFDB parser adapter skeleton metadata only.""" + """Return deterministic IPCC EFDB parser adapter metadata only.""" capability = ParserAdapterCapability( source_family=IPCC_EFDB_SOURCE_FAMILY, parser_key=IPCC_EFDB_PARSER_KEY, parser_source_format=ParserSourceFormat.DISCOVERY_REFERENCE, - format_hint="discovery", - supports_parser_execution=False, + format_hint="csv", + supports_parser_execution=True, supports_file_reads=False, - supports_content_inspection=False, + supports_content_inspection=True, ) return IpccEfdbParserAdapterDescriptor( source_family=IPCC_EFDB_SOURCE_FAMILY, parser_key=IPCC_EFDB_PARSER_KEY, - readiness=ParserAdapterSkeletonReadiness.CONTRACT_ONLY, + readiness=ParserAdapterSkeletonReadiness.CONTENT_PARSER_READY, capability=capability, mode=SourceAcquisitionPlanMode.DRY_RUN, ) diff --git a/src/carbonfactor_parser/parsers/ipcc_efdb_content_parser.py b/src/carbonfactor_parser/parsers/ipcc_efdb_content_parser.py new file mode 100644 index 00000000..9c4e55e9 --- /dev/null +++ b/src/carbonfactor_parser/parsers/ipcc_efdb_content_parser.py @@ -0,0 +1,426 @@ +"""IPCC EFDB content parser for deterministic normalized factor rows.""" + +from __future__ import annotations + +import csv +from decimal import Decimal, InvalidOperation +from io import StringIO + +from carbonfactor_parser.parsers.execution_result import ( + ParserExecutionIssue, + ParserExecutionIssueSeverity, + ParserExecutionResult, + ParserExecutionResultStatus, + create_parser_execution_result, +) +from carbonfactor_parser.parsers.file_content_input import ( + ParserFileContentInput, + validate_parser_file_content_input, +) +from carbonfactor_parser.parsers.input_contract import create_parser_input_contract +from carbonfactor_parser.parsers.raw_record import ( + create_parsed_raw_record, + create_parsed_raw_record_payload, +) + + +IPCC_EFDB_NORMALIZED_CONTENT_HEADER = ( + "record_type", + "source_year", + "source_version", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", + "subcategory", + "ipcc_sector", + "gas", + "region", + "technology", + "provenance", +) + +_REQUIRED_FIELDS = ( + "source_year", + "source_version", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", + "ipcc_sector", + "gas", + "provenance", +) + + +def parse_ipcc_efdb_file_content( + content_input: ParserFileContentInput, +) -> ParserExecutionResult: + """Parse already-loaded IPCC EFDB normalized CSV extraction content.""" + + parser_input = _parser_input_from_content_input(content_input) + validation_result = validate_parser_file_content_input(content_input) + if not validation_result.is_valid: + if _only_missing_content(validation_result): + return create_parser_execution_result( + status=ParserExecutionResultStatus.NO_RECORDS, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="IPCC_EFDB_CONTENT_EMPTY", + message="IPCC EFDB content input did not include parseable content.", + severity=ParserExecutionIssueSeverity.WARNING, + location="content", + ), + ), + parser_metadata=_parser_metadata(), + ) + + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + issues=tuple( + ParserExecutionIssue( + code=issue.code, + message=issue.message, + severity=ParserExecutionIssueSeverity.ERROR, + location=issue.field_name, + ) + for issue in validation_result.issues + ), + parser_metadata=_parser_metadata(), + ) + + if content_input.source_family != "ipcc_efdb": + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="IPCC_EFDB_CONTENT_SOURCE_FAMILY_MISMATCH", + message="IPCC EFDB content parser only accepts ipcc_efdb source_family.", + severity=ParserExecutionIssueSeverity.ERROR, + location="source_family", + ), + ), + parser_metadata=_parser_metadata(), + ) + + content_text = _content_text(content_input) + if content_text is None: + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="IPCC_EFDB_CONTENT_BYTES_DECODE_FAILED", + message="IPCC EFDB bytes content must be UTF-8 CSV text.", + severity=ParserExecutionIssueSeverity.ERROR, + location="content", + ), + ), + parser_metadata=_parser_metadata(), + ) + + return _parse_normalized_csv(content_text, parser_input) + + +def _parse_normalized_csv(content_text: str, parser_input) -> ParserExecutionResult: + reader = csv.DictReader(StringIO(content_text)) + if tuple(reader.fieldnames or ()) != IPCC_EFDB_NORMALIZED_CONTENT_HEADER: + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + issues=( + ParserExecutionIssue( + code="IPCC_EFDB_CONTENT_INVALID_HEADER", + message="IPCC EFDB normalized content header must match the declared parser contract.", + severity=ParserExecutionIssueSeverity.ERROR, + location="header", + context={ + "expected_header": IPCC_EFDB_NORMALIZED_CONTENT_HEADER, + }, + ), + ), + parser_metadata=_parser_metadata(), + ) + + issues: list[ParserExecutionIssue] = [] + raw_records = [] + parsed_record_count = 0 + skipped_record_count = 0 + + for row_number, row in enumerate(reader, start=2): + if None in row: + issues.append( + ParserExecutionIssue( + code="IPCC_EFDB_CONTENT_INVALID_ROW", + message="IPCC EFDB content row has an unexpected column count.", + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}]", + context={"row_number": row_number}, + ), + ) + continue + + normalized_row = _trim_row(row) + if not any(normalized_row.values()): + continue + + if normalized_row["record_type"] != "emission_factor": + skipped_record_count += 1 + issues.append( + ParserExecutionIssue( + code="IPCC_EFDB_CONTENT_UNSUPPORTED_ROW_SKIPPED", + message="IPCC EFDB content row was skipped because record_type is unsupported.", + severity=ParserExecutionIssueSeverity.WARNING, + location=f"row[{row_number}].record_type", + context={ + "row_number": row_number, + "record_type": normalized_row["record_type"], + }, + ), + ) + continue + + row_issues = _row_issues(normalized_row, row_number) + issues.extend(row_issues) + if row_issues: + continue + + parsed_record_count += 1 + raw_records.append( + create_parsed_raw_record( + source_family=parser_input.source_family, + source_id=parser_input.source_id, + record_index=parsed_record_count, + row_number=row_number, + raw_fields=_normalized_raw_fields( + normalized_row, + parser_input, + row_number, + ), + parser_metadata=_parser_metadata(skipped_record_count), + source_context=_source_context(parser_input, normalized_row, row_number), + ), + ) + + if any(issue.severity == ParserExecutionIssueSeverity.ERROR for issue in issues): + return create_parser_execution_result( + status=ParserExecutionResultStatus.FAILED, + parser_input=parser_input, + parsed_record_count=parsed_record_count, + issues=tuple(issues), + parser_metadata=_parser_metadata(skipped_record_count), + raw_record_payload=( + create_parsed_raw_record_payload( + source_family=parser_input.source_family, + source_id=parser_input.source_id, + records=tuple(raw_records), + parser_metadata=_parser_metadata(skipped_record_count), + source_context={ + "artifact_reference": parser_input.artifact_reference, + "checksum_sha256": parser_input.checksum_sha256, + }, + ) + if raw_records + else None + ), + ) + + if parsed_record_count == 0: + return create_parser_execution_result( + status=ParserExecutionResultStatus.NO_RECORDS, + parser_input=parser_input, + issues=( + *tuple(issues), + ParserExecutionIssue( + code="IPCC_EFDB_CONTENT_NO_RECORDS", + message="IPCC EFDB normalized content included no parseable emission factor rows.", + severity=ParserExecutionIssueSeverity.WARNING, + location="content", + ), + ), + parser_metadata=_parser_metadata(skipped_record_count), + ) + + return create_parser_execution_result( + status=ParserExecutionResultStatus.SUCCESS, + parser_input=parser_input, + parsed_record_count=parsed_record_count, + issues=tuple(issues), + parser_metadata=_parser_metadata(skipped_record_count), + raw_record_payload=create_parsed_raw_record_payload( + source_family=parser_input.source_family, + source_id=parser_input.source_id, + records=tuple(raw_records), + parser_metadata=_parser_metadata(skipped_record_count), + source_context={ + "artifact_reference": parser_input.artifact_reference, + "checksum_sha256": parser_input.checksum_sha256, + }, + ), + ) + + +def _trim_row(row: dict[str | None, str | None]) -> dict[str, str]: + return { + field_name: (row.get(field_name) or "").strip() + for field_name in IPCC_EFDB_NORMALIZED_CONTENT_HEADER + } + + +def _row_issues(row: dict[str, str], row_number: int) -> tuple[ParserExecutionIssue, ...]: + issues: list[ParserExecutionIssue] = [] + for field_name in _REQUIRED_FIELDS: + if not row[field_name]: + issues.append( + ParserExecutionIssue( + code="IPCC_EFDB_CONTENT_MISSING_REQUIRED_FIELD", + message=( + "IPCC EFDB emission factor row is missing required " + f"field: {field_name}." + ), + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}].{field_name}", + context={"row_number": row_number, "field_name": field_name}, + ), + ) + + source_year = int(row["source_year"]) if row["source_year"].isdecimal() else 0 + if source_year < 1: + issues.append( + ParserExecutionIssue( + code="IPCC_EFDB_CONTENT_INVALID_SOURCE_YEAR", + message="IPCC EFDB source_year must be a positive integer.", + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}].source_year", + context={ + "row_number": row_number, + "field_name": "source_year", + "raw_value": row["source_year"], + }, + ), + ) + + parsed_factor_value = _parse_factor_decimal(row["factor_value"]) + if parsed_factor_value is None or not parsed_factor_value.is_finite(): + issues.append( + ParserExecutionIssue( + code="IPCC_EFDB_CONTENT_INVALID_FACTOR_VALUE", + message="IPCC EFDB factor_value must be a decimal number.", + severity=ParserExecutionIssueSeverity.ERROR, + location=f"row[{row_number}].factor_value", + context={ + "row_number": row_number, + "field_name": "factor_value", + "raw_value": row["factor_value"], + }, + ), + ) + + return tuple(issues) + + +def _parse_factor_decimal(raw_value: str) -> Decimal | None: + if "e" in raw_value.lower(): + return None + try: + return Decimal(raw_value.replace(",", "")) + except InvalidOperation: + return None + + +def _normalized_raw_fields( + row: dict[str, str], + parser_input, + row_number: int, +) -> dict[str, object]: + source_year = row["source_year"] + source_version = row["source_version"] + factor_id = row["factor_id"] + unit = row["unit"] + gas = row["gas"] + master_id = f"ipcc_master_{source_year}_{source_version}_{factor_id}" + detail_id = f"ipcc_detail_{source_year}_{source_version}_{factor_id}" + + return { + "source_family": parser_input.source_family, + "source_id": parser_input.source_id, + "source_year": int(source_year), + "source_version": source_version, + "factor_id": factor_id, + "factor_name": row["factor_name"], + "factor_value": _parse_factor_decimal(row["factor_value"]), + "unit": unit, + "category": row["category"], + "subcategory": row["subcategory"] or None, + "ipcc_sector": row["ipcc_sector"], + "gas": gas, + "region": row["region"] or None, + "technology": row["technology"] or None, + "provenance_artifact_reference": parser_input.artifact_reference, + "provenance_checksum_algorithm": "sha256" + if parser_input.checksum_sha256 + else None, + "provenance_checksum_value": parser_input.checksum_sha256, + "provenance_row_number": row_number, + "provenance": row["provenance"], + "source_family_master_id": master_id, + "source_family_detail_id": detail_id, + "master_external_key": f"{source_year}:{source_version}:{factor_id}", + "detail_external_key": f"{factor_id}:{unit}:{gas}:{row['ipcc_sector']}", + } + + +def _content_text(content_input: ParserFileContentInput) -> str | None: + if isinstance(content_input.content, str): + return content_input.content + try: + return content_input.content.decode("utf-8") + except UnicodeDecodeError: + return None + + +def _only_missing_content(validation_result) -> bool: + return tuple(issue.code for issue in validation_result.issues) == ( + "PARSER_FILE_CONTENT_MISSING_CONTENT", + ) + + +def _parser_input_from_content_input(content_input: ParserFileContentInput): + return create_parser_input_contract( + source_family=content_input.source_family, + source_id=content_input.source_id, + acquisition_status="content_loaded", + artifact_reference=content_input.artifact_reference or "memory://parser-file-content-input", + checksum_sha256=content_input.checksum_sha256, + content_type=content_input.content_type, + format_hint=content_input.format_hint, + ) + + +def _source_context( + parser_input, + row: dict[str, str], + row_number: int, +) -> dict[str, object]: + return { + "artifact_reference": parser_input.artifact_reference, + "checksum_sha256": parser_input.checksum_sha256, + "row_number": row_number, + "source_year": row["source_year"], + "source_version": row["source_version"], + "provenance": row["provenance"], + } + + +def _parser_metadata(skipped_record_count: int = 0) -> dict[str, object]: + return { + "parser_kind": "ipcc_efdb_normalized_content", + "is_real_source_parser": True, + "normalization_executed": True, + "skipped_record_count": skipped_record_count, + } diff --git a/src/carbonfactor_parser/parsers/normalized_output_row_contract.py b/src/carbonfactor_parser/parsers/normalized_output_row_contract.py index bd55b4e1..b1a48d78 100644 --- a/src/carbonfactor_parser/parsers/normalized_output_row_contract.py +++ b/src/carbonfactor_parser/parsers/normalized_output_row_contract.py @@ -18,6 +18,7 @@ class ParserNormalizedOutputRowStatus(str, Enum): """Runtime-passive normalized parser output row status values.""" DECLARED = "declared" + VALIDATED = "validated" @dataclass(frozen=True) @@ -330,7 +331,7 @@ def _validate_row_status( status: ParserNormalizedOutputRowStatus, issues: list[ParserNormalizedOutputRowValidationIssue], ) -> None: - if not isinstance(status, ParserNormalizedOutputRowStatus): + if not _is_parser_normalized_output_row_status(status): issues.append( ParserNormalizedOutputRowValidationIssue( code="PARSER_NORMALIZED_ROW_INVALID_STATUS", @@ -340,6 +341,16 @@ def _validate_row_status( ) +def _is_parser_normalized_output_row_status(status: object) -> bool: + if isinstance(status, ParserNormalizedOutputRowStatus): + return True + return ( + status.__class__.__name__ == ParserNormalizedOutputRowStatus.__name__ + and getattr(status, "value", None) + in {item.value for item in ParserNormalizedOutputRowStatus} + ) + + def _validate_positive_int( value: int | None, field_name: str, diff --git a/src/carbonfactor_parser/parsers/run_repository_contract.py b/src/carbonfactor_parser/parsers/run_repository_contract.py new file mode 100644 index 00000000..8d7f262a --- /dev/null +++ b/src/carbonfactor_parser/parsers/run_repository_contract.py @@ -0,0 +1,123 @@ +"""Runtime-passive repository contract for parser run results.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Protocol, runtime_checkable + +from carbonfactor_parser.parsers.parser_run_contract import ParserRunResult + + +class ParserRunRepositoryPersistStatus(str, Enum): + """Deterministic metadata-only persist status values.""" + + DECLARED = "declared" + FAILED_VALIDATION = "failed_validation" + + +@dataclass(frozen=True) +class ParserRunRepositoryIssue: + """Metadata-only parser run repository contract issue.""" + + code: str + message: str + field_name: str + severity: str = "error" + + +@dataclass(frozen=True) +class ParserRunRepositoryPersistResult: + """Metadata-only parser run repository persist result.""" + + provider_name: str + status: ParserRunRepositoryPersistStatus + persisted_count: int + issues: tuple[ParserRunRepositoryIssue, ...] = () + + +@runtime_checkable +class ParserRunRepository(Protocol): + """Protocol for metadata-only parser run repositories.""" + + @property + def provider_name(self) -> str: + """Human-readable provider name.""" + + def persist_runs( + self, + runs: tuple[ParserRunResult, ...], + ) -> ParserRunRepositoryPersistResult: + """Persist parser run metadata contractually without runtime side effects.""" + + +def create_parser_run_repository_persist_result( + *, + provider_name: str, + runs: tuple[ParserRunResult, ...], + issues: tuple[ParserRunRepositoryIssue, ...] = (), +) -> ParserRunRepositoryPersistResult: + """Create deterministic metadata-only repository persist result.""" + + validation_issues = list( + validate_parser_run_repository_inputs( + provider_name=provider_name, + runs=runs, + ).issues, + ) + validation_issues.extend(tuple(issues)) + + status = ( + ParserRunRepositoryPersistStatus.FAILED_VALIDATION + if validation_issues + else ParserRunRepositoryPersistStatus.DECLARED + ) + + return ParserRunRepositoryPersistResult( + provider_name=provider_name, + status=status, + persisted_count=0 if validation_issues else len(runs), + issues=tuple(validation_issues), + ) + + +@dataclass(frozen=True) +class ParserRunRepositoryValidationResult: + """Validation result for parser run repository metadata.""" + + issues: tuple[ParserRunRepositoryIssue, ...] = () + + @property + def is_valid(self) -> bool: + return not self.issues + + +def validate_parser_run_repository_inputs( + *, + provider_name: str, + runs: tuple[ParserRunResult, ...], +) -> ParserRunRepositoryValidationResult: + """Validate repository inputs without runtime side effects.""" + + issues: list[ParserRunRepositoryIssue] = [] + + if not isinstance(provider_name, str) or not provider_name.strip(): + issues.append( + ParserRunRepositoryIssue( + code="PARSER_RUN_REPOSITORY_MISSING_PROVIDER_NAME", + message="provider_name must be a non-empty string.", + field_name="provider_name", + ), + ) + + for index, run in enumerate(runs): + if not isinstance(run, ParserRunResult): + issues.append( + ParserRunRepositoryIssue( + code="PARSER_RUN_REPOSITORY_INVALID_RUN", + message="runs must contain ParserRunResult instances.", + field_name=f"runs[{index}]", + ), + ) + + return ParserRunRepositoryValidationResult(issues=tuple(issues)) diff --git a/src/carbonfactor_parser/persistence/__init__.py b/src/carbonfactor_parser/persistence/__init__.py index 0c01dc9b..7dac8b66 100644 --- a/src/carbonfactor_parser/persistence/__init__.py +++ b/src/carbonfactor_parser/persistence/__init__.py @@ -30,6 +30,14 @@ PostgreSQLInsertStatement, build_postgresql_insert_statement, ) +from carbonfactor_parser.persistence.parsed_factor_persistence_writer import ( + ParsedFactorPersistenceCommand, + ParsedFactorPersistenceIssue, + ParsedFactorPersistenceStatus, + ParsedFactorPersistenceWriterResult, + build_parsed_factor_persistence_command, + persist_parsed_factor_records, +) from carbonfactor_parser.persistence.postgresql_connection_session_contract import ( PostgreSQLConnectionSession, PostgreSQLConnectionSessionContractDescription, @@ -75,6 +83,15 @@ build_postgresql_repository_disabled_execution_preview, describe_postgresql_repository_disabled_execution_preview, ) +from carbonfactor_parser.persistence.postgresql_runtime_config_gate import ( + PostgreSQLRuntimeConfigGate, + PostgreSQLRuntimeConfigGateDecision, + PostgreSQLRuntimeConfigGateDescription, + PostgreSQLRuntimeConfigGateIssue, + PostgreSQLRuntimeConfigGateStatus, + describe_postgresql_runtime_config_gate, + evaluate_postgresql_runtime_config_gate, +) from carbonfactor_parser.persistence.postgresql_runtime_execution_gate import ( PostgreSQLRuntimeExecutionGate, PostgreSQLRuntimeExecutionGateDecision, @@ -167,6 +184,27 @@ PersistenceResultStatus, create_persistence_result, ) +from carbonfactor_parser.persistence.source_document_repository import ( + SourceDocumentRepository, + SourceDocumentRepositoryIssue, + SourceDocumentRepositoryPersistResult, + SourceDocumentRepositoryPersistStatus, + SourceDocumentRepositoryValidationResult, + create_source_document_repository_persist_result, + validate_source_document_repository_inputs, +) +from carbonfactor_parser.persistence.source_family_repository import ( + SourceFamilyDetailRecord, + SourceFamilyMasterRecord, + SourceFamilyRepository, + SourceFamilyRepositoryIssue, + SourceFamilyRepositoryPersistResult, + SourceFamilyRepositoryPersistStatus, + SourceFamilyRepositoryValidationResult, + create_source_family_repository_persist_result, + source_family_repository_table_names, + validate_source_family_repository_inputs, +) from carbonfactor_parser.persistence.postgresql_repository import ( PostgreSQLPersistenceRepository, PostgreSQLRepositoryRuntimeSafetyGate, @@ -209,6 +247,22 @@ "PersistenceRepository", "PersistenceResult", "PersistenceResultStatus", + "ParsedFactorPersistenceCommand", + "ParsedFactorPersistenceIssue", + "ParsedFactorPersistenceStatus", + "ParsedFactorPersistenceWriterResult", + "SourceDocumentRepository", + "SourceDocumentRepositoryIssue", + "SourceDocumentRepositoryPersistResult", + "SourceDocumentRepositoryPersistStatus", + "SourceDocumentRepositoryValidationResult", + "SourceFamilyDetailRecord", + "SourceFamilyMasterRecord", + "SourceFamilyRepository", + "SourceFamilyRepositoryIssue", + "SourceFamilyRepositoryPersistResult", + "SourceFamilyRepositoryPersistStatus", + "SourceFamilyRepositoryValidationResult", "PsycopgPostgreSQLSessionAdapter", "PsycopgPostgreSQLSessionAdapterBoundaryResult", "PsycopgPostgreSQLSessionAdapterMetadata", @@ -267,6 +321,11 @@ "PostgreSQLRepositoryRuntimeSafetyGateDescription", "PostgreSQLRepositoryRuntimeSafetyGateIssue", "PostgreSQLRepositoryRuntimeSafetyGateStatus", + "PostgreSQLRuntimeConfigGate", + "PostgreSQLRuntimeConfigGateDecision", + "PostgreSQLRuntimeConfigGateDescription", + "PostgreSQLRuntimeConfigGateIssue", + "PostgreSQLRuntimeConfigGateStatus", "PostgreSQLRuntimeExecutionGate", "PostgreSQLRuntimeExecutionGateDecision", "PostgreSQLRuntimeExecutionGateDescription", @@ -300,6 +359,7 @@ "PostgreSQLTransactionPolicyValidationResult", "PostgreSQLTransactionRuntimeBoundary", "build_persistence_input_from_normalization_result", + "build_parsed_factor_persistence_command", "build_default_postgresql_transaction_policy", "build_default_postgresql_idempotency_conflict_strategy", "build_disabled_postgresql_execution_result", @@ -316,6 +376,8 @@ "build_postgresql_transaction_plan", "build_default_postgresql_schema_isolation_strategy", "create_persistence_result", + "create_source_document_repository_persist_result", + "create_source_family_repository_persist_result", "create_postgresql_connection_session_runtime_contract", "create_postgresql_integration_test_boundary", "create_postgresql_persistence_options", @@ -326,15 +388,21 @@ "describe_postgresql_idempotency_conflict_strategy_boundary", "describe_postgresql_repository_disabled_execution_preview", "describe_postgresql_repository_runtime_safety_gate", + "describe_postgresql_runtime_config_gate", "describe_postgresql_runtime_execution_gate", "describe_postgresql_schema_isolation_strategy", "describe_postgresql_transaction_policy_boundary", + "evaluate_postgresql_runtime_config_gate", "evaluate_postgresql_runtime_execution_gate", "evaluate_postgresql_repository_runtime_safety_gate", "evaluate_postgresql_integration_test_opt_in_config", "get_normalized_record_postgresql_schema", "render_postgresql_ddl_preview", + "persist_parsed_factor_records", "should_skip_postgresql_integration_tests", + "source_family_repository_table_names", + "validate_source_document_repository_inputs", + "validate_source_family_repository_inputs", "validate_psycopg_session_adapter_boundary", "validate_postgresql_connection_session_runtime_contract", "validate_postgresql_persistence_options", diff --git a/src/carbonfactor_parser/persistence/ingestion_run_history.py b/src/carbonfactor_parser/persistence/ingestion_run_history.py new file mode 100644 index 00000000..454a1bf7 --- /dev/null +++ b/src/carbonfactor_parser/persistence/ingestion_run_history.py @@ -0,0 +1,405 @@ +"""Parser ingestion run-history persistence contracts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from decimal import Decimal +from enum import Enum +import json +from typing import Mapping, Protocol, runtime_checkable + +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text + +_INGESTION_SOURCE_FAMILIES = frozenset(("ghg_protocol", "defra_desnz", "ipcc_efdb")) +_ISSUE_SOURCE_FAMILIES = _INGESTION_SOURCE_FAMILIES | frozenset(("configured_runner",)) +_REDACTED_VALUE = "***" +_SENSITIVE_METADATA_KEYS = frozenset( + ( + "password", + "passwd", + "pwd", + "token", + "secret", + "key", + "api_key", + "apikey", + "access_key", + "accesskey", + "private_key", + "privatekey", + "dsn", + "connection_string", + "connectionstring", + "connection_uri", + "connectionuri", + "database_url", + "databaseurl", + ) +) +_SENSITIVE_COMPACT_METADATA_KEYS = frozenset( + key.replace("_", "") for key in _SENSITIVE_METADATA_KEYS +) + + +class ParserIngestionRunHistoryStatus(str, Enum): + """Status values for run-history persistence attempts.""" + + DECLARED = "declared" + FAILED_VALIDATION = "failed_validation" + FAILED_DATABASE = "failed_database" + + +@dataclass(frozen=True) +class ParserIngestionRunHistoryIssue: + """Validation or database issue raised by the run-history boundary.""" + + code: str + message: str + field_name: str | None = None + severity: str = "error" + + +@dataclass(frozen=True) +class ParserIngestionRunRecord: + """Top-level parser ingestion run history record.""" + + run_id: str + started_at: datetime + status: str + finished_at: datetime | None = None + trigger_type: str = "operator" + config_hash: str | None = None + enabled_source_families: tuple[str, ...] = () + initial_year: int | None = None + cycle_count: int | None = None + total_parsed_rows: int = 0 + total_inserted_count: int = 0 + total_skipped_duplicate_count: int = 0 + failure_count: int = 0 + metadata: Mapping[str, object] | None = None + + +@dataclass(frozen=True) +class ParserIngestionSourceResultRecord: + """Per-source parser ingestion result history record.""" + + run_id: str + source_family: str + status: str + target_year: int | None = None + latest_year: int | None = None + download_status: str | None = None + parse_status: str | None = None + validation_status: str | None = None + insert_status: str | None = None + parsed_rows: int = 0 + master_inserted: int = 0 + master_skipped: int = 0 + detail_inserted: int = 0 + detail_skipped: int = 0 + issue_count: int = 0 + metadata: Mapping[str, object] | None = None + + +@dataclass(frozen=True) +class ParserIngestionIssueRecord: + """Parser ingestion operational issue history record.""" + + run_id: str + stage: str + code: str + message: str + source_family: str | None = None + target_year: int | None = None + severity: str = "error" + field_name: str | None = None + metadata: Mapping[str, object] | None = None + + +@dataclass(frozen=True) +class ParserIngestionRunHistoryCommand: + """Command payload for persisting one parser ingestion run history snapshot.""" + + run: ParserIngestionRunRecord + source_results: tuple[ParserIngestionSourceResultRecord, ...] = () + issues: tuple[ParserIngestionIssueRecord, ...] = () + + +@dataclass(frozen=True) +class ParserIngestionRunHistoryPersistResult: + """Result returned after a run-history persistence attempt.""" + + provider_name: str + status: ParserIngestionRunHistoryStatus + persisted_run_count: int = 0 + persisted_source_result_count: int = 0 + persisted_issue_count: int = 0 + validation_failure_count: int = 0 + issues: tuple[ParserIngestionRunHistoryIssue, ...] = () + + +@runtime_checkable +class ParserIngestionRunHistoryRepository(Protocol): + """Repository boundary for parser ingestion run-history persistence.""" + + @property + def provider_name(self) -> str: + """Return the repository provider name.""" + + def persist_ingestion_run_history( + self, + command: ParserIngestionRunHistoryCommand, + ) -> ParserIngestionRunHistoryPersistResult: + """Persist a parser ingestion run history command.""" + + +def validate_ingestion_run_history_command( + command: ParserIngestionRunHistoryCommand, +) -> tuple[ParserIngestionRunHistoryIssue, ...]: + """Validate a parser ingestion run-history command without persistence.""" + + issues: list[ParserIngestionRunHistoryIssue] = [] + run = command.run + if not str(run.run_id or "").strip(): + issues.append(_issue("INGESTION_RUN_HISTORY_RUN_ID_REQUIRED", "run_id is required.", "run_id")) + if run.started_at is None: + issues.append(_issue("INGESTION_RUN_HISTORY_STARTED_AT_REQUIRED", "started_at is required.", "started_at")) + if not str(run.status or "").strip(): + issues.append(_issue("INGESTION_RUN_HISTORY_STATUS_REQUIRED", "status is required.", "status")) + + _validate_positive_optional("initial_year", run.initial_year, issues) + _validate_non_negative_optional("cycle_count", run.cycle_count, issues) + for field_name in ( + "total_parsed_rows", + "total_inserted_count", + "total_skipped_duplicate_count", + "failure_count", + ): + _validate_non_negative(field_name, getattr(run, field_name), issues) + + for source_family in run.enabled_source_families: + _validate_ingestion_source_family(source_family, "enabled_source_families", issues) + _validate_json_metadata(run.metadata or {}, "metadata", issues) + + for index, source_result in enumerate(command.source_results): + prefix = f"source_results[{index}]" + if source_result.run_id != run.run_id: + issues.append(_issue("INGESTION_RUN_HISTORY_SOURCE_RUN_ID_MISMATCH", "source result run_id must match command run_id.", f"{prefix}.run_id")) + if not str(source_result.status or "").strip(): + issues.append(_issue("INGESTION_RUN_HISTORY_SOURCE_STATUS_REQUIRED", "source result status is required.", f"{prefix}.status")) + _validate_ingestion_source_family(source_result.source_family, f"{prefix}.source_family", issues) + _validate_source_result_target_year(f"{prefix}.target_year", source_result.target_year, issues) + _validate_positive_optional(f"{prefix}.latest_year", source_result.latest_year, issues) + for field_name in ( + "parsed_rows", + "master_inserted", + "master_skipped", + "detail_inserted", + "detail_skipped", + "issue_count", + ): + _validate_non_negative(f"{prefix}.{field_name}", getattr(source_result, field_name), issues) + _validate_json_metadata(source_result.metadata or {}, f"{prefix}.metadata", issues) + + for index, issue_record in enumerate(command.issues): + prefix = f"issues[{index}]" + if issue_record.run_id != run.run_id: + issues.append(_issue("INGESTION_RUN_HISTORY_ISSUE_RUN_ID_MISMATCH", "issue run_id must match command run_id.", f"{prefix}.run_id")) + if issue_record.source_family is not None: + _validate_issue_source_family(issue_record.source_family, f"{prefix}.source_family", issues) + _validate_positive_optional(f"{prefix}.target_year", issue_record.target_year, issues) + if not str(issue_record.stage or "").strip(): + issues.append(_issue("INGESTION_RUN_HISTORY_ISSUE_STAGE_REQUIRED", "issue stage is required.", f"{prefix}.stage")) + if not str(issue_record.code or "").strip(): + issues.append(_issue("INGESTION_RUN_HISTORY_ISSUE_CODE_REQUIRED", "issue code is required.", f"{prefix}.code")) + if not str(issue_record.message or "").strip(): + issues.append(_issue("INGESTION_RUN_HISTORY_ISSUE_MESSAGE_REQUIRED", "issue message is required.", f"{prefix}.message")) + _validate_json_metadata(issue_record.metadata or {}, f"{prefix}.metadata", issues) + + return tuple(issues) + + +def sanitized_ingestion_run_history_command( + command: ParserIngestionRunHistoryCommand, +) -> ParserIngestionRunHistoryCommand: + """Return a JSON-safe command with text content redacted for persistence.""" + + run = command.run + sanitized_run = ParserIngestionRunRecord( + run_id=run.run_id, + started_at=run.started_at, + status=run.status, + finished_at=run.finished_at, + trigger_type=run.trigger_type, + config_hash=run.config_hash, + enabled_source_families=tuple(run.enabled_source_families), + initial_year=run.initial_year, + cycle_count=run.cycle_count, + total_parsed_rows=run.total_parsed_rows, + total_inserted_count=run.total_inserted_count, + total_skipped_duplicate_count=run.total_skipped_duplicate_count, + failure_count=run.failure_count, + metadata=_json_safe_redacted(run.metadata or {}), + ) + source_results = tuple( + ParserIngestionSourceResultRecord( + run_id=record.run_id, + source_family=record.source_family, + status=record.status, + target_year=record.target_year, + latest_year=record.latest_year, + download_status=record.download_status, + parse_status=record.parse_status, + validation_status=record.validation_status, + insert_status=record.insert_status, + parsed_rows=record.parsed_rows, + master_inserted=record.master_inserted, + master_skipped=record.master_skipped, + detail_inserted=record.detail_inserted, + detail_skipped=record.detail_skipped, + issue_count=record.issue_count, + metadata=_json_safe_redacted(record.metadata or {}), + ) + for record in command.source_results + ) + issues = tuple( + ParserIngestionIssueRecord( + run_id=record.run_id, + source_family=record.source_family, + target_year=record.target_year, + stage=record.stage, + code=record.code, + severity=record.severity, + field_name=record.field_name, + message=redact_sensitive_text(record.message), + metadata=_json_safe_redacted(record.metadata or {}), + ) + for record in command.issues + ) + return ParserIngestionRunHistoryCommand( + run=sanitized_run, + source_results=source_results, + issues=issues, + ) + + +def json_payload(value: object) -> str: + """Serialize a JSON-safe, redacted value for PostgreSQL jsonb parameters.""" + + return json.dumps(_json_safe_redacted(value), sort_keys=True, separators=(",", ":")) + + +def _issue(code: str, message: str, field_name: str) -> ParserIngestionRunHistoryIssue: + return ParserIngestionRunHistoryIssue(code=code, message=message, field_name=field_name) + + +def _validate_ingestion_source_family( + source_family: str, + field_name: str, + issues: list[ParserIngestionRunHistoryIssue], +) -> None: + if source_family not in _INGESTION_SOURCE_FAMILIES: + issues.append(_issue("INGESTION_RUN_HISTORY_SOURCE_FAMILY_UNSUPPORTED", "source_family is unsupported.", field_name)) + + +def _validate_issue_source_family( + source_family: str, + field_name: str, + issues: list[ParserIngestionRunHistoryIssue], +) -> None: + if source_family not in _ISSUE_SOURCE_FAMILIES: + issues.append(_issue("INGESTION_RUN_HISTORY_SOURCE_FAMILY_UNSUPPORTED", "source_family is unsupported.", field_name)) + + +def _validate_source_result_target_year( + field_name: str, + value: int | None, + issues: list[ParserIngestionRunHistoryIssue], +) -> None: + if value is None: + issues.append(_issue("INGESTION_RUN_HISTORY_SOURCE_TARGET_YEAR_REQUIRED", "source result target_year is required for idempotent upsert semantics.", field_name)) + elif value <= 0: + issues.append(_issue("INGESTION_RUN_HISTORY_POSITIVE_INTEGER_REQUIRED", f"{field_name} must be positive when provided.", field_name)) + + +def _validate_positive_optional( + field_name: str, + value: int | None, + issues: list[ParserIngestionRunHistoryIssue], +) -> None: + if value is not None and value <= 0: + issues.append(_issue("INGESTION_RUN_HISTORY_POSITIVE_INTEGER_REQUIRED", f"{field_name} must be positive when provided.", field_name)) + + +def _validate_non_negative_optional( + field_name: str, + value: int | None, + issues: list[ParserIngestionRunHistoryIssue], +) -> None: + if value is not None: + _validate_non_negative(field_name, value, issues) + + +def _validate_non_negative( + field_name: str, + value: int, + issues: list[ParserIngestionRunHistoryIssue], +) -> None: + if value < 0: + issues.append(_issue("INGESTION_RUN_HISTORY_NON_NEGATIVE_COUNT_REQUIRED", f"{field_name} must be non-negative.", field_name)) + + +def _validate_json_metadata( + value: Mapping[str, object], + field_name: str, + issues: list[ParserIngestionRunHistoryIssue], +) -> None: + try: + json.dumps(_json_safe_redacted(value), sort_keys=True) + except (TypeError, ValueError) as exc: + issues.append(_issue("INGESTION_RUN_HISTORY_METADATA_JSON_REQUIRED", f"metadata must be JSON serializable: {redact_sensitive_text(str(exc))}", field_name)) + + +def _is_sensitive_metadata_key(key: str) -> bool: + normalized = key.strip().lower().replace("-", "_") + return ( + normalized in _SENSITIVE_METADATA_KEYS + or normalized.replace("_", "") in _SENSITIVE_COMPACT_METADATA_KEYS + ) + + +def _json_safe_redacted(value: object) -> object: + if isinstance(value, Decimal): + return str(value) + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, Mapping): + redacted_mapping: dict[str, object] = {} + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])): + key_text = str(key) + if _is_sensitive_metadata_key(key_text): + redacted_mapping[key_text] = _REDACTED_VALUE + else: + redacted_mapping[key_text] = _json_safe_redacted(item) + return redacted_mapping + if isinstance(value, tuple | list): + return [_json_safe_redacted(item) for item in value] + if isinstance(value, str): + return redact_sensitive_text(value) + return value + + +__all__ = ( + "ParserIngestionIssueRecord", + "ParserIngestionRunHistoryCommand", + "ParserIngestionRunHistoryIssue", + "ParserIngestionRunHistoryPersistResult", + "ParserIngestionRunHistoryRepository", + "ParserIngestionRunHistoryStatus", + "ParserIngestionRunRecord", + "ParserIngestionSourceResultRecord", + "json_payload", + "sanitized_ingestion_run_history_command", + "validate_ingestion_run_history_command", +) diff --git a/src/carbonfactor_parser/persistence/ingestion_run_history_mapping.py b/src/carbonfactor_parser/persistence/ingestion_run_history_mapping.py new file mode 100644 index 00000000..798fbc89 --- /dev/null +++ b/src/carbonfactor_parser/persistence/ingestion_run_history_mapping.py @@ -0,0 +1,166 @@ +"""Mapping from configured ingestion cycles to run-history commands.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Mapping + +from carbonfactor_parser.diagnostics.ingestion_runtime_events import ( + configured_download_status_value, + configured_parse_status_value, + iter_deduplicated_ingestion_issues, +) +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text +from carbonfactor_parser.persistence.ingestion_run_history import ( + ParserIngestionIssueRecord, + ParserIngestionRunHistoryCommand, + ParserIngestionRunRecord, + ParserIngestionSourceResultRecord, +) +if TYPE_CHECKING: + from carbonfactor_parser.pipeline.configured_cycle_models import ConfiguredCycleResult + + +def build_ingestion_run_history_command_from_configured_cycle( + cycle: "ConfiguredCycleResult", + *, + started_at: datetime | None = None, + finished_at: datetime | None = None, + trigger_type: str = "operator", + metadata: Mapping[str, object] | None = None, +) -> ParserIngestionRunHistoryCommand: + """Build a parser run-history persistence command for a configured cycle.""" + + safe_started_at = started_at or datetime.now(timezone.utc) + safe_finished_at = finished_at or datetime.now(timezone.utc) + result = cycle.result + summary = result.summary + run_metadata: dict[str, object] = { + "requested_family_count": summary.requested_family_count, + "completed_family_count": summary.completed_family_count, + "failed_family_count": summary.failed_family_count, + "no_available_source_year_count": summary.no_available_source_year_count, + } + if metadata: + run_metadata.update(dict(metadata)) + + source_results = tuple( + _source_result_record(cycle.run_id, family) + for family in result.family_results + ) + issues = tuple( + _issue_record( + cycle.run_id, + issue, + target_year_by_source_family=_target_year_by_source_family(result.family_results), + ) + for issue in iter_deduplicated_ingestion_issues(result) + ) + + return ParserIngestionRunHistoryCommand( + run=ParserIngestionRunRecord( + run_id=cycle.run_id, + started_at=safe_started_at, + finished_at=safe_finished_at, + status=result.status.value, + trigger_type=trigger_type, + enabled_source_families=tuple(result.selected_source_families), + initial_year=result.request.initial_year, + cycle_count=cycle.cycle_number, + total_parsed_rows=summary.parsed_row_count, + total_inserted_count=summary.inserted_count, + total_skipped_duplicate_count=summary.skipped_duplicate_count, + failure_count=summary.failure_count, + metadata=run_metadata, + ), + source_results=source_results, + issues=issues, + ) + + +def _source_result_record( + run_id: str, + family: object, +) -> ParserIngestionSourceResultRecord: + year_state = getattr(family, "year_state") + validation_result = getattr(family, "validation_result", None) + insert_summary = getattr(family, "insert_summary", None) + metadata = { + "selection_status": getattr( + getattr(year_state, "selection_status", None), + "value", + getattr(year_state, "selection_status", None), + ), + "recorded_ingested_year": getattr(family, "recorded_ingested_year", None), + } + return ParserIngestionSourceResultRecord( + run_id=run_id, + source_family=getattr(family, "source_family"), + target_year=getattr(year_state, "target_year"), + latest_year=getattr(year_state, "latest_year"), + status=_status_value(getattr(family, "status", None)), + download_status=configured_download_status_value( + getattr(family, "download_result", None), + ), + parse_status=configured_parse_status_value(family), + validation_status=( + _status_value(getattr(validation_result, "status", None)) + if validation_result is not None + else None + ), + insert_status=( + _status_value(getattr(insert_summary, "status", None)) + if insert_summary is not None + else None + ), + parsed_rows=getattr(family, "parsed_row_count", 0), + master_inserted=getattr(insert_summary, "master_inserted", 0), + master_skipped=getattr(insert_summary, "master_skipped", 0), + detail_inserted=getattr(insert_summary, "detail_inserted", 0), + detail_skipped=getattr(insert_summary, "detail_skipped", 0), + issue_count=len(tuple(getattr(family, "failures", ()))), + metadata={key: value for key, value in metadata.items() if value is not None}, + ) + + +def _status_value(status: object | None) -> str | None: + if status is None: + return None + return str(getattr(status, "value", status)) + + +def _issue_record( + run_id: str, + issue: object, + *, + target_year_by_source_family: Mapping[str, int], +) -> ParserIngestionIssueRecord: + source_family = getattr(issue, "source_family", None) + return ParserIngestionIssueRecord( + run_id=run_id, + source_family=source_family, + target_year=( + target_year_by_source_family.get(source_family) + if source_family is not None + else None + ), + stage=str(getattr(issue, "stage")), + code=str(getattr(issue, "code")), + severity=str(getattr(issue, "severity", "error")), + field_name=getattr(issue, "field_name", None), + message=redact_sensitive_text(str(getattr(issue, "message"))), + ) + + +def _target_year_by_source_family(family_results: object) -> dict[str, int]: + target_years: dict[str, int] = {} + for family in family_results: + year_state = getattr(family, "year_state") + target_years[getattr(family, "source_family")] = getattr( + year_state, + "target_year", + ) + return target_years + + +__all__ = ("build_ingestion_run_history_command_from_configured_cycle",) diff --git a/src/carbonfactor_parser/persistence/ingestion_run_history_reader.py b/src/carbonfactor_parser/persistence/ingestion_run_history_reader.py new file mode 100644 index 00000000..130d90f5 --- /dev/null +++ b/src/carbonfactor_parser/persistence/ingestion_run_history_reader.py @@ -0,0 +1,451 @@ +"""Read-side boundary for parser ingestion run history.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +import json +from typing import Protocol, runtime_checkable + + +_RUN_COLUMNS = ( + "run_id", + "started_at", + "finished_at", + "status", + "trigger_type", + "config_hash", + "enabled_source_families", + "initial_year", + "cycle_count", + "total_parsed_rows", + "total_inserted_count", + "total_skipped_duplicate_count", + "failure_count", + "metadata", +) +_SOURCE_RESULT_COLUMNS = ( + "run_id", + "source_family", + "target_year", + "latest_year", + "status", + "download_status", + "parse_status", + "validation_status", + "insert_status", + "parsed_rows", + "master_inserted", + "master_skipped", + "detail_inserted", + "detail_skipped", + "issue_count", + "metadata", +) +_ISSUE_COLUMNS = ( + "run_id", + "source_family", + "target_year", + "stage", + "code", + "severity", + "field_name", + "message", + "metadata", + "created_at", +) +_MIN_RECENT_RUN_LIMIT = 1 +_MAX_RECENT_RUN_LIMIT = 100 +_DEFAULT_RECENT_RUN_LIMIT = 20 + + +@dataclass(frozen=True) +class ParserIngestionRunReadModel: + """Read model for one parser ingestion run history row.""" + + run_id: str + started_at: datetime | object + finished_at: datetime | object | None + status: str + trigger_type: str + config_hash: str | None + enabled_source_families: tuple[str, ...] + initial_year: int | None + cycle_count: int | None + total_parsed_rows: int + total_inserted_count: int + total_skipped_duplicate_count: int + failure_count: int + metadata: Mapping[str, object] + + +@dataclass(frozen=True) +class ParserIngestionSourceResultReadModel: + """Read model for one source result in a parser ingestion run.""" + + run_id: str + source_family: str + target_year: int + latest_year: int | None + status: str + download_status: str | None + parse_status: str | None + validation_status: str | None + insert_status: str | None + parsed_rows: int + master_inserted: int + master_skipped: int + detail_inserted: int + detail_skipped: int + issue_count: int + metadata: Mapping[str, object] + + +@dataclass(frozen=True) +class ParserIngestionIssueReadModel: + """Read model for one operational issue in a parser ingestion run.""" + + run_id: str + source_family: str | None + target_year: int | None + stage: str + code: str + severity: str + field_name: str | None + message: str + metadata: Mapping[str, object] + created_at: datetime | object + + +@dataclass(frozen=True) +class ParserIngestionRunDetailReadModel: + """Read model containing a run and its source results and issues.""" + + run: ParserIngestionRunReadModel + source_results: tuple[ParserIngestionSourceResultReadModel, ...] + issues: tuple[ParserIngestionIssueReadModel, ...] + + +@runtime_checkable +class ParserIngestionRunHistoryReader(Protocol): + """Read repository boundary for parser ingestion run history.""" + + @property + def provider_name(self) -> str: + """Return the reader provider name.""" + + def get_latest_ingestion_run(self) -> ParserIngestionRunReadModel | None: + """Return the most recent parser ingestion run, if present.""" + + def get_ingestion_run_by_id( + self, + run_id: str, + ) -> ParserIngestionRunDetailReadModel | None: + """Return one parser ingestion run with source results and issues.""" + + def list_recent_ingestion_runs( + self, + limit: int = _DEFAULT_RECENT_RUN_LIMIT, + ) -> tuple[ParserIngestionRunReadModel, ...]: + """Return recent parser ingestion runs.""" + + def list_ingestion_run_source_results( + self, + run_id: str, + ) -> tuple[ParserIngestionSourceResultReadModel, ...]: + """Return source results for one parser ingestion run.""" + + def list_ingestion_run_issues( + self, + run_id: str, + ) -> tuple[ParserIngestionIssueReadModel, ...]: + """Return issues for one parser ingestion run.""" + + +class PostgreSQLIngestionRunHistoryReader: + """Query parser ingestion run-history records from PostgreSQL.""" + + def __init__(self, connection: object) -> None: + if connection is None: + raise ValueError("connection must be provided.") + self._connection = connection + + @property + def provider_name(self) -> str: + """Return the reader provider name.""" + + return "postgresql" + + def get_latest_ingestion_run(self) -> ParserIngestionRunReadModel | None: + """Return the latest parser ingestion run, if one exists.""" + + cursor = _execute(self._connection, _latest_run_sql(), (1,)) + row = _fetchone(cursor) + if row is None: + return None + return _run_read_model(row) + + def get_ingestion_run_by_id( + self, + run_id: str, + ) -> ParserIngestionRunDetailReadModel | None: + """Return a parser ingestion run with source results and issues.""" + + validated_run_id = _validated_run_id(run_id) + cursor = _execute(self._connection, _run_by_id_sql(), (validated_run_id,)) + row = _fetchone(cursor) + if row is None: + return None + return ParserIngestionRunDetailReadModel( + run=_run_read_model(row), + source_results=self.list_ingestion_run_source_results(validated_run_id), + issues=self.list_ingestion_run_issues(validated_run_id), + ) + + def list_recent_ingestion_runs( + self, + limit: int = _DEFAULT_RECENT_RUN_LIMIT, + ) -> tuple[ParserIngestionRunReadModel, ...]: + """Return recent parser ingestion runs in deterministic order.""" + + validated_limit = _validated_limit(limit) + cursor = _execute(self._connection, _recent_runs_sql(), (validated_limit,)) + return tuple(_run_read_model(row) for row in _fetchall(cursor)) + + def list_ingestion_run_source_results( + self, + run_id: str, + ) -> tuple[ParserIngestionSourceResultReadModel, ...]: + """Return source results for one parser ingestion run.""" + + validated_run_id = _validated_run_id(run_id) + cursor = _execute(self._connection, _source_results_sql(), (validated_run_id,)) + return tuple(_source_result_read_model(row) for row in _fetchall(cursor)) + + def list_ingestion_run_issues( + self, + run_id: str, + ) -> tuple[ParserIngestionIssueReadModel, ...]: + """Return issues for one parser ingestion run.""" + + validated_run_id = _validated_run_id(run_id) + cursor = _execute(self._connection, _issues_sql(), (validated_run_id,)) + return tuple(_issue_read_model(row) for row in _fetchall(cursor)) + + +def _latest_run_sql() -> str: + return f""" + SELECT {_run_select_list()} + FROM parser_ingestion_runs + ORDER BY started_at DESC, run_id DESC + LIMIT %s + """ + + +def _recent_runs_sql() -> str: + return _latest_run_sql() + + +def _run_by_id_sql() -> str: + return f""" + SELECT {_run_select_list()} + FROM parser_ingestion_runs + WHERE run_id = %s + """ + + +def _source_results_sql() -> str: + return f""" + SELECT {_source_result_select_list()} + FROM parser_ingestion_source_results + WHERE run_id = %s + ORDER BY source_family ASC, target_year ASC + """ + + +def _issues_sql() -> str: + return f""" + SELECT {_issue_select_list()} + FROM parser_ingestion_issues + WHERE run_id = %s + ORDER BY created_at ASC, source_family ASC NULLS LAST, code ASC + """ + + +def _run_select_list() -> str: + return ", ".join(_RUN_COLUMNS) + + +def _source_result_select_list() -> str: + return ", ".join(_SOURCE_RESULT_COLUMNS) + + +def _issue_select_list() -> str: + return ", ".join(_ISSUE_COLUMNS) + + +def _run_read_model(row: object) -> ParserIngestionRunReadModel: + return ParserIngestionRunReadModel( + run_id=_string_value(row, "run_id", _RUN_COLUMNS), + started_at=_row_value(row, "started_at", _RUN_COLUMNS), + finished_at=_row_value(row, "finished_at", _RUN_COLUMNS), + status=_string_value(row, "status", _RUN_COLUMNS), + trigger_type=_string_value(row, "trigger_type", _RUN_COLUMNS), + config_hash=_optional_string_value(row, "config_hash", _RUN_COLUMNS), + enabled_source_families=_source_families_value( + _row_value(row, "enabled_source_families", _RUN_COLUMNS) + ), + initial_year=_optional_int_value(row, "initial_year", _RUN_COLUMNS), + cycle_count=_optional_int_value(row, "cycle_count", _RUN_COLUMNS), + total_parsed_rows=_int_value(row, "total_parsed_rows", _RUN_COLUMNS), + total_inserted_count=_int_value(row, "total_inserted_count", _RUN_COLUMNS), + total_skipped_duplicate_count=_int_value( + row, + "total_skipped_duplicate_count", + _RUN_COLUMNS, + ), + failure_count=_int_value(row, "failure_count", _RUN_COLUMNS), + metadata=_metadata_value(_row_value(row, "metadata", _RUN_COLUMNS)), + ) + + +def _source_result_read_model(row: object) -> ParserIngestionSourceResultReadModel: + return ParserIngestionSourceResultReadModel( + run_id=_string_value(row, "run_id", _SOURCE_RESULT_COLUMNS), + source_family=_string_value(row, "source_family", _SOURCE_RESULT_COLUMNS), + target_year=_int_value(row, "target_year", _SOURCE_RESULT_COLUMNS), + latest_year=_optional_int_value(row, "latest_year", _SOURCE_RESULT_COLUMNS), + status=_string_value(row, "status", _SOURCE_RESULT_COLUMNS), + download_status=_optional_string_value(row, "download_status", _SOURCE_RESULT_COLUMNS), + parse_status=_optional_string_value(row, "parse_status", _SOURCE_RESULT_COLUMNS), + validation_status=_optional_string_value(row, "validation_status", _SOURCE_RESULT_COLUMNS), + insert_status=_optional_string_value(row, "insert_status", _SOURCE_RESULT_COLUMNS), + parsed_rows=_int_value(row, "parsed_rows", _SOURCE_RESULT_COLUMNS), + master_inserted=_int_value(row, "master_inserted", _SOURCE_RESULT_COLUMNS), + master_skipped=_int_value(row, "master_skipped", _SOURCE_RESULT_COLUMNS), + detail_inserted=_int_value(row, "detail_inserted", _SOURCE_RESULT_COLUMNS), + detail_skipped=_int_value(row, "detail_skipped", _SOURCE_RESULT_COLUMNS), + issue_count=_int_value(row, "issue_count", _SOURCE_RESULT_COLUMNS), + metadata=_metadata_value(_row_value(row, "metadata", _SOURCE_RESULT_COLUMNS)), + ) + + +def _issue_read_model(row: object) -> ParserIngestionIssueReadModel: + return ParserIngestionIssueReadModel( + run_id=_string_value(row, "run_id", _ISSUE_COLUMNS), + source_family=_optional_string_value(row, "source_family", _ISSUE_COLUMNS), + target_year=_optional_int_value(row, "target_year", _ISSUE_COLUMNS), + stage=_string_value(row, "stage", _ISSUE_COLUMNS), + code=_string_value(row, "code", _ISSUE_COLUMNS), + severity=_string_value(row, "severity", _ISSUE_COLUMNS), + field_name=_optional_string_value(row, "field_name", _ISSUE_COLUMNS), + message=_string_value(row, "message", _ISSUE_COLUMNS), + metadata=_metadata_value(_row_value(row, "metadata", _ISSUE_COLUMNS)), + created_at=_row_value(row, "created_at", _ISSUE_COLUMNS), + ) + + +def _execute(connection: object, statement: str, parameters: tuple[object, ...]) -> object: + return getattr(connection, "execute")(statement, parameters) + + +def _fetchone(cursor: object) -> object | None: + return getattr(cursor, "fetchone")() + + +def _fetchall(cursor: object) -> tuple[object, ...]: + rows = getattr(cursor, "fetchall")() + return tuple(rows or ()) + + +def _row_value(row: object, column_name: str, column_names: Sequence[str]) -> object: + if isinstance(row, Mapping): + return row[column_name] + if hasattr(row, column_name): + return getattr(row, column_name) + return row[column_names.index(column_name)] # type: ignore[index] + + +def _string_value(row: object, column_name: str, column_names: Sequence[str]) -> str: + return str(_row_value(row, column_name, column_names)) + + +def _optional_string_value( + row: object, + column_name: str, + column_names: Sequence[str], +) -> str | None: + value = _row_value(row, column_name, column_names) + if value is None: + return None + return str(value) + + +def _int_value(row: object, column_name: str, column_names: Sequence[str]) -> int: + return int(_row_value(row, column_name, column_names)) + + +def _optional_int_value( + row: object, + column_name: str, + column_names: Sequence[str], +) -> int | None: + value = _row_value(row, column_name, column_names) + if value is None: + return None + return int(value) + + +def _metadata_value(value: object) -> Mapping[str, object]: + decoded = _json_decoded_value(value) + if isinstance(decoded, Mapping): + return decoded + return {} + + +def _source_families_value(value: object) -> tuple[str, ...]: + decoded = _json_decoded_value(value) + if isinstance(decoded, str): + return (decoded,) + if isinstance(decoded, Sequence): + return tuple(str(item) for item in decoded) + return () + + +def _json_decoded_value(value: object) -> object: + if isinstance(value, str): + try: + return json.loads(value) + except json.JSONDecodeError: + return value + return value + + +def _validated_limit(limit: int) -> int: + if not isinstance(limit, int): + raise ValueError("limit must be an integer.") + if limit < _MIN_RECENT_RUN_LIMIT or limit > _MAX_RECENT_RUN_LIMIT: + raise ValueError("limit must be between 1 and 100.") + return limit + + +def _validated_run_id(run_id: str) -> str: + if not isinstance(run_id, str) or not run_id.strip(): + raise ValueError("run_id must be a non-empty string.") + return run_id + + +__all__ = ( + "ParserIngestionIssueReadModel", + "ParserIngestionRunDetailReadModel", + "ParserIngestionRunHistoryReader", + "ParserIngestionRunReadModel", + "ParserIngestionSourceResultReadModel", + "PostgreSQLIngestionRunHistoryReader", + "_issues_sql", + "_latest_run_sql", + "_recent_runs_sql", + "_run_by_id_sql", + "_source_results_sql", +) diff --git a/src/carbonfactor_parser/persistence/parsed_factor_persistence_writer.py b/src/carbonfactor_parser/persistence/parsed_factor_persistence_writer.py new file mode 100644 index 00000000..d2398a36 --- /dev/null +++ b/src/carbonfactor_parser/persistence/parsed_factor_persistence_writer.py @@ -0,0 +1,723 @@ +"""Parsed emission factor persistence writer boundary.""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from enum import Enum +import hashlib +import json +from typing import TYPE_CHECKING, Mapping, Sequence + +from carbonfactor_parser.parsers.raw_record import ( + ParsedRawRecord, + ParsedRawRecordPayload, + validate_parsed_raw_record_payload, +) +from carbonfactor_parser.persistence.postgresql_schema_catalog import SourceFamily +from carbonfactor_parser.persistence.source_document_mapping import ( + DRY_RUN_TIMESTAMP_LABEL, +) +from carbonfactor_parser.persistence.source_family_repository import ( + SourceFamilyDetailRecord, + SourceFamilyMasterRecord, + SourceFamilyRepository, + SourceFamilyRepositoryIssue, + SourceFamilyRepositoryPersistStatus, + validate_source_family_repository_inputs, +) + +if TYPE_CHECKING: + from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputBatch, + ParserNormalizedOutputRow, + ) + ParsedFactorOutput = ParsedRawRecordPayload | ParserNormalizedOutputBatch +else: + ParsedFactorOutput = object + + +class ParsedFactorPersistenceStatus(str, Enum): + """Status for parsed factor persistence writer outcomes.""" + + DECLARED = "declared" + FAILED_VALIDATION = "failed_validation" + NO_RECORDS = "no_records" + + +@dataclass(frozen=True) +class ParsedFactorPersistenceIssue: + """Validation or repository issue from parsed factor persistence mapping.""" + + code: str + message: str + field_name: str + severity: str = "error" + + +@dataclass(frozen=True) +class ParsedFactorPersistenceCommand: + """Source-family master/detail records ready for repository persistence.""" + + master_records: tuple[SourceFamilyMasterRecord, ...] + detail_records: tuple[SourceFamilyDetailRecord, ...] + skipped_duplicate_count: int = 0 + issues: tuple[ParsedFactorPersistenceIssue, ...] = () + + +@dataclass(frozen=True) +class ParsedFactorPersistenceWriterResult: + """Result of building and submitting parsed factor persistence commands.""" + + provider_name: str + status: ParsedFactorPersistenceStatus + attempted_master_count: int + attempted_detail_count: int + persisted_master_count: int + persisted_detail_count: int + skipped_duplicate_count: int = 0 + skipped_master_count: int = 0 + skipped_detail_count: int = 0 + validation_failure_count: int = 0 + issues: tuple[ParsedFactorPersistenceIssue, ...] = () + command: ParsedFactorPersistenceCommand | None = None + + @property + def master_inserted_count(self) -> int: + """Return user-visible inserted master count.""" + + return self.persisted_master_count + + @property + def detail_inserted_count(self) -> int: + """Return user-visible inserted detail count.""" + + return self.persisted_detail_count + + @property + def master_skipped_count(self) -> int: + """Return user-visible skipped master count.""" + + return self.skipped_master_count + + @property + def detail_skipped_count(self) -> int: + """Return user-visible skipped detail count.""" + + return self.skipped_detail_count + + @property + def final_status(self) -> str: + """Return user-visible final persistence status.""" + + return self.status.value + + +_SOURCE_FAMILY_ALIASES: Mapping[str, SourceFamily] = { + "ghg": SourceFamily.GHG, + "ghg_protocol": SourceFamily.GHG, + "defra": SourceFamily.DEFRA, + "defra_desnz": SourceFamily.DEFRA, + "desnz": SourceFamily.DEFRA, + "ipcc": SourceFamily.IPCC, + "ipcc_efdb": SourceFamily.IPCC, +} + + +def build_parsed_factor_persistence_command( + parsed_output: ParsedFactorOutput, + *, + source_document_id: str | None = None, + lifecycle_status: str = "active", + timestamp_label: str = DRY_RUN_TIMESTAMP_LABEL, +) -> ParsedFactorPersistenceCommand: + """Map parsed factor output into source-family master/detail records.""" + + rows, shape_issues = _rows_from_output(parsed_output) + if not rows and not shape_issues: + return ParsedFactorPersistenceCommand( + master_records=(), + detail_records=(), + issues=( + ParsedFactorPersistenceIssue( + code="PARSED_FACTOR_PERSISTENCE_NO_RECORDS", + message="parsed output must include records before persistence.", + field_name="records", + severity="warning", + ), + ), + ) + + issues: list[ParsedFactorPersistenceIssue] = list(shape_issues) + masters: dict[tuple[SourceFamily, str], SourceFamilyMasterRecord] = {} + details: dict[tuple[SourceFamily, str, str], SourceFamilyDetailRecord] = {} + skipped_duplicate_count = 0 + + for position, row in enumerate(rows, start=1): + mapped = _map_row( + row, + position=position, + source_document_id=source_document_id, + lifecycle_status=lifecycle_status, + timestamp_label=timestamp_label, + ) + issues.extend(mapped.issues) + if mapped.master_record is None or mapped.detail_record is None: + continue + + master_key = ( + mapped.master_record.source_family, + mapped.master_record.source_family_master_id, + ) + existing_master = masters.get(master_key) + if existing_master is None: + masters[master_key] = mapped.master_record + elif existing_master != mapped.master_record: + issues.append( + ParsedFactorPersistenceIssue( + code="PARSED_FACTOR_PERSISTENCE_DUPLICATE_MASTER_CONFLICT", + message=( + "duplicate source-family master identity maps to " + "different record content." + ), + field_name=f"records[{position}].source_family_master_id", + ), + ) + + detail_key = ( + mapped.detail_record.source_family, + mapped.detail_record.source_family_master_id, + mapped.detail_record.detail_external_key, + ) + existing_detail = details.get(detail_key) + if existing_detail is None: + details[detail_key] = mapped.detail_record + elif existing_detail == mapped.detail_record: + skipped_duplicate_count += 1 + else: + issues.append( + ParsedFactorPersistenceIssue( + code="PARSED_FACTOR_PERSISTENCE_DUPLICATE_DETAIL_CONFLICT", + message=( + "duplicate factor identity maps to different detail " + "record content." + ), + field_name=f"records[{position}].detail_external_key", + ), + ) + + command = ParsedFactorPersistenceCommand( + master_records=tuple(masters.values()), + detail_records=tuple(details.values()), + skipped_duplicate_count=skipped_duplicate_count, + issues=tuple(issues), + ) + repository_validation = validate_source_family_repository_inputs( + provider_name="parsed_factor_persistence_command", + master_records=command.master_records, + detail_records=command.detail_records, + ) + if repository_validation.issues: + return ParsedFactorPersistenceCommand( + master_records=command.master_records, + detail_records=command.detail_records, + skipped_duplicate_count=command.skipped_duplicate_count, + issues=( + *command.issues, + *tuple( + _from_repository_issue(issue) + for issue in repository_validation.issues + ), + ), + ) + + return command + + +def persist_parsed_factor_records( + parsed_output: ParsedFactorOutput, + repository: SourceFamilyRepository, + *, + source_document_id: str | None = None, + lifecycle_status: str = "active", + timestamp_label: str = DRY_RUN_TIMESTAMP_LABEL, +) -> ParsedFactorPersistenceWriterResult: + """Build parsed factor records and submit them to a repository protocol.""" + + command = build_parsed_factor_persistence_command( + parsed_output, + source_document_id=source_document_id, + lifecycle_status=lifecycle_status, + timestamp_label=timestamp_label, + ) + if command.issues: + status = ( + ParsedFactorPersistenceStatus.NO_RECORDS + if _only_no_records(command.issues) + else ParsedFactorPersistenceStatus.FAILED_VALIDATION + ) + return ParsedFactorPersistenceWriterResult( + provider_name=repository.provider_name, + status=status, + attempted_master_count=len(command.master_records), + attempted_detail_count=len(command.detail_records), + persisted_master_count=0, + persisted_detail_count=0, + skipped_duplicate_count=command.skipped_duplicate_count, + validation_failure_count=len(command.issues), + issues=command.issues, + command=command, + ) + + repository_result = repository.persist_source_family_records( + command.master_records, + command.detail_records, + ) + repository_issues = tuple( + _from_repository_issue(issue) for issue in repository_result.issues + ) + status = ( + ParsedFactorPersistenceStatus.DECLARED + if repository_result.status is SourceFamilyRepositoryPersistStatus.DECLARED + else ParsedFactorPersistenceStatus.FAILED_VALIDATION + ) + return ParsedFactorPersistenceWriterResult( + provider_name=repository_result.provider_name, + status=status, + attempted_master_count=len(command.master_records), + attempted_detail_count=len(command.detail_records), + persisted_master_count=repository_result.persisted_master_count, + persisted_detail_count=repository_result.persisted_detail_count, + skipped_duplicate_count=command.skipped_duplicate_count, + skipped_master_count=repository_result.skipped_master_count, + skipped_detail_count=repository_result.skipped_detail_count, + validation_failure_count=repository_result.validation_failure_count, + issues=repository_issues, + command=command, + ) + + +@dataclass(frozen=True) +class _PersistenceRow: + source_family: str + source_id: str + row_id: str + fields: Mapping[str, object] + artifact_reference: str | None + artifact_checksum_sha256: str | None + source_row_number: int | None + honor_explicit_master_external_key: bool = False + + +@dataclass(frozen=True) +class _MappedRow: + master_record: SourceFamilyMasterRecord | None + detail_record: SourceFamilyDetailRecord | None + issues: tuple[ParsedFactorPersistenceIssue, ...] = () + + +def _rows_from_output( + parsed_output: ParsedFactorOutput, +) -> tuple[tuple[_PersistenceRow, ...], tuple[ParsedFactorPersistenceIssue, ...]]: + from carbonfactor_parser.parsers.normalized_output_row_contract import ( + validate_parser_normalized_output_batch, + ) + + if _looks_like_raw_record_payload(parsed_output): + validation = validate_parsed_raw_record_payload(parsed_output) + issues = tuple( + ParsedFactorPersistenceIssue( + code=issue.code, + message=issue.message, + field_name=issue.field_name, + severity=issue.severity, + ) + for issue in validation.issues + ) + return ( + tuple(_row_from_raw_record(record) for record in parsed_output.records), + issues, + ) + + if _looks_like_normalized_output_batch(parsed_output): + validation = validate_parser_normalized_output_batch(parsed_output) + issues = tuple( + ParsedFactorPersistenceIssue( + code=issue.code, + message=issue.message, + field_name=issue.field_name, + severity=issue.severity, + ) + for issue in validation.issues + ) + return ( + tuple(_row_from_normalized_row(row) for row in parsed_output.rows), + issues, + ) + + return ( + (), + ( + ParsedFactorPersistenceIssue( + code="PARSED_FACTOR_PERSISTENCE_INVALID_OUTPUT", + message=( + "parsed_output must be ParsedRawRecordPayload or " + "ParserNormalizedOutputBatch." + ), + field_name="parsed_output", + ), + ), + ) + + +def _looks_like_raw_record_payload(value: object) -> bool: + return hasattr(value, "records") and all( + hasattr(record, "raw_fields") for record in getattr(value, "records", ()) + ) + + +def _looks_like_normalized_output_batch(value: object) -> bool: + return hasattr(value, "rows") and all( + hasattr(row, "normalized_fields") for row in getattr(value, "rows", ()) + ) + + +def _row_from_raw_record(record: ParsedRawRecord) -> _PersistenceRow: + return _PersistenceRow( + source_family=record.source_family, + source_id=record.source_id, + row_id=f"record-{record.record_index}", + fields=dict(record.raw_fields), + artifact_reference=_text_or_none( + (record.source_context or {}).get("artifact_reference") + ), + artifact_checksum_sha256=_text_or_none( + (record.source_context or {}).get("checksum_sha256") + ), + source_row_number=record.row_number, + ) + + +def _row_from_normalized_row(row: "ParserNormalizedOutputRow") -> _PersistenceRow: + fields = dict(row.normalized_fields) + fields.setdefault("parser_key", row.parser_key) + return _PersistenceRow( + source_family=row.source_family, + source_id=row.source_key, + row_id=row.row_id, + fields=fields, + artifact_reference=row.artifact_reference, + artifact_checksum_sha256=_text_or_none( + _field(fields, "source_checksum_sha256", "checksum_sha256") + ), + source_row_number=row.source_row_number, + honor_explicit_master_external_key=True, + ) + + +def _map_row( + row: _PersistenceRow, + *, + position: int, + source_document_id: str | None, + lifecycle_status: str, + timestamp_label: str, +) -> _MappedRow: + issues: list[ParsedFactorPersistenceIssue] = [] + source_family = _source_family(row.source_family) + if source_family is None: + issues.append( + ParsedFactorPersistenceIssue( + code="PARSED_FACTOR_PERSISTENCE_UNSUPPORTED_SOURCE_FAMILY", + message="source_family must map to GHG, DEFRA/DESNZ, or IPCC.", + field_name=f"records[{position}].source_family", + ), + ) + return _MappedRow(None, None, tuple(issues)) + + resolved_source_document_id = _source_document_id(row, source_document_id) + required_values = { + "source_document_id": resolved_source_document_id, + "factor_value": _field(row.fields, "factor_value", "value"), + "factor_unit": _field(row.fields, "factor_unit", "unit"), + } + for field_name, value in required_values.items(): + if _text_or_none(value) is None: + issues.append( + ParsedFactorPersistenceIssue( + code="PARSED_FACTOR_PERSISTENCE_MISSING_REQUIRED_FIELD", + message="parsed factor persistence requires a non-empty value.", + field_name=f"records[{position}].{field_name}", + ), + ) + + if issues: + return _MappedRow(None, None, tuple(issues)) + + explicit_master_external_key = ( + _text_or_none(_field(row.fields, "master_external_key")) + if row.honor_explicit_master_external_key + else None + ) + master_external_key = explicit_master_external_key or _default_master_external_key(row) + detail_external_key = _text_or_none( + _field(row.fields, "detail_external_key") + ) or _default_detail_external_key(row) + master_id = ( + f"{source_family.value}_master_" + f"{_stable_digest(source_family.value, master_external_key)[:16]}" + ) + detail_id = _text_or_none( + _field(row.fields, "source_family_detail_id") + ) or ( + f"{source_family.value}_detail_" + f"{_stable_digest(source_family.value, master_id, detail_external_key)[:16]}" + ) + source_year = _master_source_year(row) + if source_year is None: + issues.append( + ParsedFactorPersistenceIssue( + code="PARSED_FACTOR_PERSISTENCE_MISSING_REQUIRED_FIELD", + message="parsed factor persistence requires a source_year value.", + field_name=f"records[{position}].source_year", + ), + ) + return _MappedRow(None, None, tuple(issues)) + source_version = _text_or_none(_field(row.fields, "source_version")) or "unknown" + artifact_checksum_sha256 = row.artifact_checksum_sha256 or _text_or_none( + _field(row.fields, "provenance_checksum_value", "source_checksum_sha256") + ) + + master_record = SourceFamilyMasterRecord( + source_family=source_family, + source_family_master_id=master_id, + source_year=source_year, + source_version=source_version, + source_release=_text_or_none(_field(row.fields, "source_release")), + source_document_id=resolved_source_document_id or "", + ingestion_run_id=_text_or_none(_field(row.fields, "ingestion_run_id")), + run_id=_text_or_none(_field(row.fields, "run_id")), + master_external_key=master_external_key, + status=lifecycle_status, + artifact_reference=row.artifact_reference, + artifact_checksum_sha256=artifact_checksum_sha256, + archive_reference=_text_or_none(_field(row.fields, "archive_reference")), + archive_checksum_sha256=_text_or_none( + _field(row.fields, "archive_checksum_sha256") + ), + effective_from=_text_or_none(_field(row.fields, "effective_from")), + effective_to=_text_or_none(_field(row.fields, "effective_to")), + record_checksum_sha256=_record_checksum( + "master", + source_family.value, + resolved_source_document_id, + master_external_key, + lifecycle_status, + ), + metadata={ + "parser_key": _text_or_none(_field(row.fields, "parser_key")), + "source_document_id": resolved_source_document_id, + }, + created_at=timestamp_label, + updated_at=timestamp_label, + ) + detail_record = SourceFamilyDetailRecord( + source_family=source_family, + source_family_detail_id=detail_id, + source_family_master_id=master_id, + detail_external_key=detail_external_key, + source_row_number=row.source_row_number, + factor_id=_text_or_none(_field(row.fields, "factor_id")), + factor_name=_text_or_none(_field(row.fields, "factor_name")), + factor_value=_text_or_none(required_values["factor_value"]) or "", + factor_unit=_text_or_none(required_values["factor_unit"]) or "", + status=lifecycle_status, + record_checksum_sha256=_record_checksum( + "detail", + source_family.value, + master_id, + detail_external_key, + required_values["factor_value"], + required_values["factor_unit"], + ), + raw_fields=dict(row.fields), + normalized_fields=dict(row.fields), + created_at=timestamp_label, + updated_at=timestamp_label, + ) + return _MappedRow(master_record, detail_record) + + +def _source_family(value: str) -> SourceFamily | None: + if not isinstance(value, str): + return None + return _SOURCE_FAMILY_ALIASES.get(value.strip().lower()) + + +def _source_document_id( + row: _PersistenceRow, + explicit_source_document_id: str | None, +) -> str | None: + explicit = _text_or_none(explicit_source_document_id) + if explicit is not None: + return explicit + field_value = _text_or_none(_field(row.fields, "source_document_id")) + if field_value is not None: + return field_value + artifact_reference = _text_or_none( + _field(row.fields, "provenance_artifact_reference", "artifact_reference") + ) or row.artifact_reference + checksum = _text_or_none( + _field(row.fields, "provenance_checksum_value", "source_checksum_sha256") + ) + if artifact_reference is None and checksum is None: + return None + digest = _stable_digest( + row.source_family, + row.source_id, + artifact_reference, + checksum, + ) + return f"source_document_{digest[:24]}" + + +def _default_master_external_key(row: _PersistenceRow) -> str: + source_year = _master_source_year_text(row) + source_version = ( + _text_or_none(_field(row.fields, "source_version")) or "unknown-version" + ) + artifact_reference = ( + row.artifact_reference + or _text_or_none( + _field( + row.fields, + "source_artifact_reference", + "artifact_reference", + "provenance_artifact_reference", + ) + ) + or "artifact-unavailable" + ) + checksum = ( + row.artifact_checksum_sha256 + or _text_or_none( + _field( + row.fields, + "source_checksum_sha256", + "checksum_sha256", + "provenance_checksum_value", + ) + ) + or "checksum-unavailable" + ) + return f"{source_year}:{source_version}:{artifact_reference}:{checksum}" + + +def _master_source_year(row: _PersistenceRow) -> int | None: + return _int_or_none(_master_source_year_text(row)) + + +def _master_source_year_text(row: _PersistenceRow) -> str: + explicit_artifact_year = _text_or_none( + _field(row.fields, "source_artifact_year", "artifact_year", "reporting_year") + ) + if explicit_artifact_year is not None: + return explicit_artifact_year + + source_version = _text_or_none(_field(row.fields, "source_version")) + if source_version is not None: + version_year = _year_from_text(source_version) + if version_year is not None: + return version_year + + return _text_or_none(_field(row.fields, "source_year")) or "unknown-year" + + +def _year_from_text(value: str) -> str | None: + for index in range(0, len(value) - 3): + candidate = value[index : index + 4] + if candidate.isdecimal() and 1900 <= int(candidate) <= 2999: + return candidate + return None + + +def _default_detail_external_key(row: _PersistenceRow) -> str: + factor_id = _text_or_none(_field(row.fields, "factor_id")) or row.row_id + factor_unit = ( + _text_or_none(_field(row.fields, "factor_unit", "unit")) or "unknown-unit" + ) + gas = _text_or_none(_field(row.fields, "greenhouse_gas", "gas")) + if gas is None: + return f"{factor_id}:{factor_unit}" + return f"{factor_id}:{factor_unit}:{gas}" + + +def _field(fields: Mapping[str, object], *names: str) -> object | None: + for name in names: + if name in fields: + return fields[name] + return None + + +def _text_or_none(value: object | None) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _int_or_none(value: object | None) -> int | None: + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return value + text = _text_or_none(value) + if text is None: + return None + try: + return int(text) + except ValueError: + return None + + +def _record_checksum(*values: object) -> str: + return _stable_digest(*values) + + +def _stable_digest(*values: object) -> str: + payload = json.dumps( + [_json_safe(value) for value in values], + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _json_safe(value: object) -> object: + if isinstance(value, Decimal): + return str(value) + if isinstance(value, Mapping): + return { + str(key): _json_safe(item) + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + } + if isinstance(value, Sequence) and not isinstance(value, str): + return [_json_safe(item) for item in value] + return value + + +def _from_repository_issue( + issue: SourceFamilyRepositoryIssue, +) -> ParsedFactorPersistenceIssue: + return ParsedFactorPersistenceIssue( + code=issue.code, + message=issue.message, + field_name=issue.field_name, + severity=issue.severity, + ) + + +def _only_no_records(issues: tuple[ParsedFactorPersistenceIssue, ...]) -> bool: + return tuple(issue.code for issue in issues) == ( + "PARSED_FACTOR_PERSISTENCE_NO_RECORDS", + ) diff --git a/src/carbonfactor_parser/persistence/postgresql_ddl_renderer.py b/src/carbonfactor_parser/persistence/postgresql_ddl_renderer.py index e66448f2..1bd56574 100644 --- a/src/carbonfactor_parser/persistence/postgresql_ddl_renderer.py +++ b/src/carbonfactor_parser/persistence/postgresql_ddl_renderer.py @@ -83,6 +83,8 @@ def render_create_table_statement(table_definition: TableDefinition) -> str: for column in table_definition.columns: column_name = _render_identifier(column.name, "column") column_sql = f"{column_name} {_DATA_TYPE_SQL[column.data_type]}" + if column.default_sql is not None: + column_sql += f" DEFAULT {column.default_sql}" if not column.nullable: column_sql += " NOT NULL" lines.append(column_sql) @@ -115,7 +117,7 @@ def render_create_table_statement(table_definition: TableDefinition) -> str: ) inner = ",\n ".join(lines) - return f"CREATE TABLE {table_name} (\n {inner}\n);" + return f"CREATE TABLE IF NOT EXISTS {table_name} (\n {inner}\n);" def render_create_index_statements(table_definition: TableDefinition) -> tuple[str, ...]: @@ -131,7 +133,7 @@ def render_create_index_statements(table_definition: TableDefinition) -> tuple[s columns.append(_render_identifier(column_name, "column")) unique_prefix = "UNIQUE " if index.unique else "" statements.append( - f"CREATE {unique_prefix}INDEX {index_name} ON {table_name} ({', '.join(columns)});" + f"CREATE {unique_prefix}INDEX IF NOT EXISTS {index_name} ON {table_name} ({', '.join(columns)});" ) return tuple(statements) diff --git a/src/carbonfactor_parser/persistence/postgresql_execution.py b/src/carbonfactor_parser/persistence/postgresql_execution.py new file mode 100644 index 00000000..683dc98d --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_execution.py @@ -0,0 +1,42 @@ +"""PostgreSQL connection execution helpers.""" + +from __future__ import annotations + + +def execute( + connection: object, + statement: str, + parameters: object | None = None, +) -> object: + """Execute a statement against a connection-like object.""" + + execute_method = getattr(connection, "execute") + if parameters is None: + return execute_method(statement) + return execute_method(statement, parameters) + + +def fetchone(cursor: object) -> object | None: + """Fetch one row from a cursor-like object.""" + + fetchone_method = getattr(cursor, "fetchone") + return fetchone_method() + + +def commit(connection: object) -> None: + """Commit a connection-like object when it supports commit.""" + + commit_method = getattr(connection, "commit", None) + if commit_method is not None: + commit_method() + + +def rollback(connection: object) -> None: + """Rollback a connection-like object when it supports rollback.""" + + rollback_method = getattr(connection, "rollback", None) + if rollback_method is not None: + rollback_method() + + +__all__ = ("commit", "execute", "fetchone", "rollback") diff --git a/src/carbonfactor_parser/persistence/postgresql_ingestion_run_history_repository.py b/src/carbonfactor_parser/persistence/postgresql_ingestion_run_history_repository.py new file mode 100644 index 00000000..1bfde961 --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_ingestion_run_history_repository.py @@ -0,0 +1,307 @@ +"""PostgreSQL parser ingestion run-history repository.""" + +from __future__ import annotations + +import json +import uuid + +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text +from carbonfactor_parser.persistence.ingestion_run_history import ( + ParserIngestionIssueRecord, + ParserIngestionRunHistoryCommand, + ParserIngestionRunHistoryIssue, + ParserIngestionRunHistoryPersistResult, + ParserIngestionRunHistoryStatus, + ParserIngestionRunRecord, + ParserIngestionSourceResultRecord, + json_payload, + sanitized_ingestion_run_history_command, + validate_ingestion_run_history_command, +) + + +class PostgreSQLIngestionRunHistoryRepository: + """Persist parser ingestion run-history records with idempotent upserts.""" + + def __init__(self, connection: object) -> None: + if connection is None: + raise ValueError("connection must be provided.") + self._connection = connection + + @property + def provider_name(self) -> str: + """Return the repository provider name.""" + + return "postgresql" + + def persist_ingestion_run_history( + self, + command: ParserIngestionRunHistoryCommand, + ) -> ParserIngestionRunHistoryPersistResult: + """Persist one parser ingestion run-history command.""" + + validation_issues = validate_ingestion_run_history_command(command) + if validation_issues: + return ParserIngestionRunHistoryPersistResult( + provider_name=self.provider_name, + status=ParserIngestionRunHistoryStatus.FAILED_VALIDATION, + validation_failure_count=len(validation_issues), + issues=validation_issues, + ) + + sanitized = sanitized_ingestion_run_history_command(command) + try: + _execute(self._connection, _run_upsert_sql(), _run_parameters(sanitized.run)) + for source_result in sanitized.source_results: + _execute( + self._connection, + _source_result_upsert_sql(), + _source_result_parameters(source_result), + ) + for issue in sanitized.issues: + _execute( + self._connection, + _issue_insert_sql(), + _issue_parameters(issue), + ) + _commit(self._connection) + except Exception as exc: # pragma: no cover - driver type varies + _rollback(self._connection) + return ParserIngestionRunHistoryPersistResult( + provider_name=self.provider_name, + status=ParserIngestionRunHistoryStatus.FAILED_DATABASE, + issues=( + ParserIngestionRunHistoryIssue( + code="POSTGRESQL_INGESTION_RUN_HISTORY_DATABASE_ERROR", + message=redact_sensitive_text(str(exc)), + field_name="database", + ), + ), + ) + + return ParserIngestionRunHistoryPersistResult( + provider_name=self.provider_name, + status=ParserIngestionRunHistoryStatus.DECLARED, + persisted_run_count=1, + persisted_source_result_count=len(sanitized.source_results), + persisted_issue_count=len(sanitized.issues), + ) + + +def _run_upsert_sql() -> str: + return """ + INSERT INTO parser_ingestion_runs ( + run_id, + started_at, + finished_at, + status, + trigger_type, + config_hash, + enabled_source_families, + initial_year, + cycle_count, + total_parsed_rows, + total_inserted_count, + total_skipped_duplicate_count, + failure_count, + metadata, + created_at, + updated_at + ) + VALUES ( + %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s, %s, + %s, %s, %s, %s::jsonb, NOW(), NOW() + ) + ON CONFLICT (run_id) DO UPDATE SET + started_at = EXCLUDED.started_at, + finished_at = EXCLUDED.finished_at, + status = EXCLUDED.status, + trigger_type = EXCLUDED.trigger_type, + config_hash = EXCLUDED.config_hash, + enabled_source_families = EXCLUDED.enabled_source_families, + initial_year = EXCLUDED.initial_year, + cycle_count = EXCLUDED.cycle_count, + total_parsed_rows = EXCLUDED.total_parsed_rows, + total_inserted_count = EXCLUDED.total_inserted_count, + total_skipped_duplicate_count = EXCLUDED.total_skipped_duplicate_count, + failure_count = EXCLUDED.failure_count, + metadata = EXCLUDED.metadata, + updated_at = NOW() + """ + + +def _source_result_upsert_sql() -> str: + return """ + INSERT INTO parser_ingestion_source_results ( + parser_ingestion_source_result_id, + run_id, + source_family, + target_year, + latest_year, + status, + download_status, + parse_status, + validation_status, + insert_status, + parsed_rows, + master_inserted, + master_skipped, + detail_inserted, + detail_skipped, + issue_count, + metadata, + created_at, + updated_at + ) + VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s::jsonb, NOW(), NOW() + ) + ON CONFLICT (run_id, source_family, target_year) DO UPDATE SET + latest_year = EXCLUDED.latest_year, + status = EXCLUDED.status, + download_status = EXCLUDED.download_status, + parse_status = EXCLUDED.parse_status, + validation_status = EXCLUDED.validation_status, + insert_status = EXCLUDED.insert_status, + parsed_rows = EXCLUDED.parsed_rows, + master_inserted = EXCLUDED.master_inserted, + master_skipped = EXCLUDED.master_skipped, + detail_inserted = EXCLUDED.detail_inserted, + detail_skipped = EXCLUDED.detail_skipped, + issue_count = EXCLUDED.issue_count, + metadata = EXCLUDED.metadata, + updated_at = NOW() + """ + + +def _issue_insert_sql() -> str: + return """ + INSERT INTO parser_ingestion_issues ( + parser_ingestion_issue_id, + run_id, + source_family, + target_year, + stage, + code, + severity, + field_name, + message, + metadata, + created_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, NOW()) + ON CONFLICT (parser_ingestion_issue_id) DO NOTHING + """ + + +def _run_parameters(record: ParserIngestionRunRecord) -> tuple[object, ...]: + return ( + record.run_id, + record.started_at, + record.finished_at, + record.status, + record.trigger_type, + record.config_hash, + json_payload(tuple(record.enabled_source_families)), + record.initial_year, + record.cycle_count, + record.total_parsed_rows, + record.total_inserted_count, + record.total_skipped_duplicate_count, + record.failure_count, + json_payload(record.metadata or {}), + ) + + +def _source_result_parameters(record: ParserIngestionSourceResultRecord) -> tuple[object, ...]: + return ( + str(stable_source_result_uuid(record)), + record.run_id, + record.source_family, + record.target_year, + record.latest_year, + record.status, + record.download_status, + record.parse_status, + record.validation_status, + record.insert_status, + record.parsed_rows, + record.master_inserted, + record.master_skipped, + record.detail_inserted, + record.detail_skipped, + record.issue_count, + json_payload(record.metadata or {}), + ) + + +def _issue_parameters(record: ParserIngestionIssueRecord) -> tuple[object, ...]: + return ( + str(stable_issue_uuid(record)), + record.run_id, + record.source_family, + record.target_year, + record.stage, + record.code, + record.severity, + record.field_name, + record.message, + json_payload(record.metadata or {}), + ) + + +def stable_source_result_uuid(record: ParserIngestionSourceResultRecord) -> uuid.UUID: + """Return a deterministic source result id for the natural upsert key.""" + + return _stable_uuid("parser_ingestion_source_result", record.run_id, record.source_family, record.target_year) + + +def stable_issue_uuid(record: ParserIngestionIssueRecord) -> uuid.UUID: + """Return a deterministic issue id from the sanitized issue identity.""" + + sanitized_message = redact_sensitive_text(record.message) + return _stable_uuid( + "parser_ingestion_issue", + record.run_id, + record.source_family, + record.target_year, + record.stage, + record.code, + sanitized_message, + ) + + +def _stable_uuid(*values: object) -> uuid.UUID: + payload = json.dumps(tuple(str(value) for value in values), separators=(",", ":")) + return uuid.uuid5(uuid.NAMESPACE_URL, payload) + + +def _execute(connection: object, statement: str, parameters: object | None = None) -> object: + execute = getattr(connection, "execute") + if parameters is None: + return execute(statement) + return execute(statement, parameters) + + +def _commit(connection: object) -> None: + commit = getattr(connection, "commit", None) + if commit is not None: + commit() + + +def _rollback(connection: object) -> None: + rollback = getattr(connection, "rollback", None) + if rollback is not None: + rollback() + + +__all__ = ( + "PostgreSQLIngestionRunHistoryRepository", + "stable_issue_uuid", + "stable_source_result_uuid", + "_issue_insert_sql", + "_issue_parameters", + "_run_upsert_sql", + "_source_result_upsert_sql", +) diff --git a/src/carbonfactor_parser/persistence/postgresql_normalized_factor_repository.py b/src/carbonfactor_parser/persistence/postgresql_normalized_factor_repository.py new file mode 100644 index 00000000..8da77bac --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_normalized_factor_repository.py @@ -0,0 +1,557 @@ +"""PostgreSQL runtime insert repository for parser normalized factor records.""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from enum import Enum +import hashlib +import json +import re +from typing import TYPE_CHECKING, Callable, Mapping, Sequence + +from carbonfactor_parser.persistence.postgresql_runtime_config import ( + PostgreSQLRuntimeConfig, + PostgreSQLRuntimeConfigLoadResult, +) + +if TYPE_CHECKING: + from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputBatch, + ParserNormalizedOutputRow, + ) +else: + ParserNormalizedOutputBatch = object + ParserNormalizedOutputRow = object + + +NORMALIZED_FACTOR_RECORDS_TABLE_NAME = "normalized_factor_records" + + +class PostgreSQLNormalizedFactorInsertStatus(str, Enum): + """Status values for PostgreSQL normalized factor inserts.""" + + INSERTED = "inserted" + FAILED_VALIDATION = "failed_validation" + FAILED_DATABASE = "failed_database" + NO_RECORDS = "no_records" + + +@dataclass(frozen=True) +class PostgreSQLNormalizedFactorInsertIssue: + """Safe structured insert issue.""" + + code: str + message: str + field_name: str | None = None + severity: str = "error" + + +@dataclass(frozen=True) +class PostgreSQLNormalizedFactorInsertSummary: + """Deterministic insert summary for normalized factor records.""" + + status: PostgreSQLNormalizedFactorInsertStatus + attempted: int + inserted: int + skipped_duplicate: int + failed: int + validation_error_count: int + provider_name: str = "postgresql" + issues: tuple[PostgreSQLNormalizedFactorInsertIssue, ...] = () + + +@dataclass(frozen=True) +class _InsertRecord: + normalized_factor_record_id: str + idempotency_key_sha256: str + source_family: str + source_id: str + source_year: int | None + source_version: str | None + record_id: str + source_row_number: int | None + source_document_reference: str | None + source_artifact_reference: str | None + source_checksum_sha256: str | None + factor_id: str | None + factor_name: str | None + factor_value: Decimal + factor_unit: str + validation_status: str + run_id: str | None + parser_key: str + metadata_json: str + normalized_fields_json: str + warnings_json: str + errors_json: str + + +class PostgreSQLNormalizedFactorRuntimeRepository: + """Runtime PostgreSQL repository using a caller-provided connection.""" + + def __init__(self, connection: object) -> None: + if connection is None: + raise ValueError("connection must be provided.") + self._connection = connection + + @property + def provider_name(self) -> str: + """Return the repository provider name.""" + + return "postgresql" + + def insert_normalized_factor_records( + self, + batch: ParserNormalizedOutputBatch, + ) -> PostgreSQLNormalizedFactorInsertSummary: + """Insert normalized factor rows with idempotent conflict handling.""" + + records, validation_issues = _build_insert_records(batch) + if validation_issues: + return PostgreSQLNormalizedFactorInsertSummary( + status=PostgreSQLNormalizedFactorInsertStatus.FAILED_VALIDATION, + attempted=len(batch.rows), + inserted=0, + skipped_duplicate=0, + failed=len(batch.rows), + validation_error_count=len(validation_issues), + issues=validation_issues, + ) + if not records: + return PostgreSQLNormalizedFactorInsertSummary( + status=PostgreSQLNormalizedFactorInsertStatus.NO_RECORDS, + attempted=0, + inserted=0, + skipped_duplicate=0, + failed=0, + validation_error_count=0, + issues=( + PostgreSQLNormalizedFactorInsertIssue( + code="POSTGRESQL_NORMALIZED_FACTOR_NO_RECORDS", + message="batch must include records before insert.", + field_name="batch.rows", + severity="warning", + ), + ), + ) + + inserted = 0 + try: + for record in records: + cursor = _execute(self._connection, _INSERT_SQL, _parameters(record)) + if _fetchone(cursor) is None: + continue + inserted += 1 + _commit(self._connection) + except Exception as exc: # pragma: no cover - exact driver type varies + _rollback(self._connection) + return PostgreSQLNormalizedFactorInsertSummary( + status=PostgreSQLNormalizedFactorInsertStatus.FAILED_DATABASE, + attempted=len(records), + inserted=0, + skipped_duplicate=0, + failed=len(records), + validation_error_count=0, + issues=( + PostgreSQLNormalizedFactorInsertIssue( + code="POSTGRESQL_NORMALIZED_FACTOR_DATABASE_ERROR", + message=_redact_sensitive_text(str(exc)), + field_name="database", + ), + ), + ) + + skipped = len(records) - inserted + return PostgreSQLNormalizedFactorInsertSummary( + status=PostgreSQLNormalizedFactorInsertStatus.INSERTED, + attempted=len(records), + inserted=inserted, + skipped_duplicate=skipped, + failed=0, + validation_error_count=0, + ) + + +def insert_postgresql_normalized_factor_records( + batch: ParserNormalizedOutputBatch, + *, + config_result: PostgreSQLRuntimeConfigLoadResult, + connection_factory: Callable[[PostgreSQLRuntimeConfig], object], +) -> PostgreSQLNormalizedFactorInsertSummary: + """Insert via explicit config and connection factory, failing closed.""" + + if not config_result.is_ready or config_result.config is None: + return PostgreSQLNormalizedFactorInsertSummary( + status=PostgreSQLNormalizedFactorInsertStatus.FAILED_VALIDATION, + attempted=len(batch.rows), + inserted=0, + skipped_duplicate=0, + failed=len(batch.rows), + validation_error_count=len(config_result.issues), + issues=tuple( + PostgreSQLNormalizedFactorInsertIssue( + code=issue.code, + message=issue.message, + field_name=issue.field_name, + severity=issue.severity, + ) + for issue in config_result.issues + ), + ) + + try: + connection = connection_factory(config_result.config) + except Exception as exc: # pragma: no cover - exact factory failures vary + return PostgreSQLNormalizedFactorInsertSummary( + status=PostgreSQLNormalizedFactorInsertStatus.FAILED_DATABASE, + attempted=len(batch.rows), + inserted=0, + skipped_duplicate=0, + failed=len(batch.rows), + validation_error_count=0, + issues=( + PostgreSQLNormalizedFactorInsertIssue( + code="POSTGRESQL_NORMALIZED_FACTOR_CONNECTION_ERROR", + message=_redact_sensitive_text(str(exc)), + field_name="database", + ), + ), + ) + + return PostgreSQLNormalizedFactorRuntimeRepository( + connection, + ).insert_normalized_factor_records(batch) + + +def _build_insert_records( + batch: ParserNormalizedOutputBatch, +) -> tuple[ + tuple[_InsertRecord, ...], + tuple[PostgreSQLNormalizedFactorInsertIssue, ...], +]: + from carbonfactor_parser.parsers.normalized_output_row_contract import ( + validate_parser_normalized_output_batch, + ) + + issues: list[PostgreSQLNormalizedFactorInsertIssue] = [] + for issue in validate_parser_normalized_output_batch(batch).issues: + issues.append( + PostgreSQLNormalizedFactorInsertIssue( + code=issue.code, + message=issue.message, + field_name=issue.field_name, + severity=issue.severity, + ), + ) + + records: list[_InsertRecord] = [] + for position, row in enumerate(batch.rows, start=1): + record, row_issues = _map_row(row, position) + issues.extend(row_issues) + if record is not None: + records.append(record) + + if issues: + return (), tuple(issues) + return tuple(records), () + + +def _map_row( + row: ParserNormalizedOutputRow, + position: int, +) -> tuple[_InsertRecord | None, tuple[PostgreSQLNormalizedFactorInsertIssue, ...]]: + from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputRowStatus, + ) + + fields = dict(row.normalized_fields) + issues: list[PostgreSQLNormalizedFactorInsertIssue] = [] + if not _is_validated_row_status(row.status, ParserNormalizedOutputRowStatus): + issues.append( + PostgreSQLNormalizedFactorInsertIssue( + code="POSTGRESQL_NORMALIZED_FACTOR_INVALID_ROW_STATUS", + message="normalized factor row status must be validated before insert.", + field_name=f"rows[{position}].status", + ), + ) + + factor_value, factor_value_issue = _required_decimal( + fields, + ("factor_value", "value"), + f"rows[{position}].factor_value", + ) + if factor_value_issue is not None: + issues.append(factor_value_issue) + + factor_unit = _text_or_none(_first_field(fields, "factor_unit", "unit")) + if factor_unit is None: + issues.append( + PostgreSQLNormalizedFactorInsertIssue( + code="POSTGRESQL_NORMALIZED_FACTOR_MISSING_FACTOR_UNIT", + message="normalized factor row must include factor_unit or unit.", + field_name=f"rows[{position}].factor_unit", + ), + ) + + source_year = _positive_int_or_none( + _first_field(fields, "source_year", "reporting_year"), + ) + if source_year is None: + source_year = row.reporting_year + source_document_reference = _text_or_none( + _first_field(fields, "source_document_id", "source_document_reference"), + ) + source_artifact_reference = _text_or_none( + _first_field(fields, "source_artifact_reference", "artifact_reference"), + ) or row.artifact_reference + source_checksum_sha256 = _text_or_none( + _first_field(fields, "source_checksum_sha256", "checksum_sha256"), + ) + + if issues or factor_value is None or factor_unit is None: + return None, tuple(issues) + + source_version = _text_or_none(_first_field(fields, "source_version")) + factor_id = _text_or_none(_first_field(fields, "factor_id")) + factor_name = _text_or_none(_first_field(fields, "factor_name", "name")) + run_id = _text_or_none(_first_field(fields, "run_id", "ingestion_run_id")) + validation_status = _text_or_none( + _first_field(fields, "validation_status"), + ) or _row_status_value(row.status) + idempotency_key = _idempotency_key( + row.source_family, + row.source_key, + source_year, + source_document_reference or source_artifact_reference, + source_checksum_sha256, + row.row_id, + ) + metadata = { + "artifact_identifier": row.artifact_identifier, + "parser_key": row.parser_key, + "reporting_year": row.reporting_year, + "source_row_number": row.source_row_number, + "status": _row_status_value(row.status), + } + + return ( + _InsertRecord( + normalized_factor_record_id=f"nfr_{idempotency_key[:32]}", + idempotency_key_sha256=idempotency_key, + source_family=row.source_family, + source_id=row.source_key, + source_year=source_year, + source_version=source_version, + record_id=row.row_id, + source_row_number=row.source_row_number, + source_document_reference=source_document_reference, + source_artifact_reference=source_artifact_reference, + source_checksum_sha256=source_checksum_sha256, + factor_id=factor_id, + factor_name=factor_name, + factor_value=factor_value, + factor_unit=factor_unit, + validation_status=validation_status, + run_id=run_id, + parser_key=row.parser_key, + metadata_json=_json_dumps(metadata), + normalized_fields_json=_json_dumps(fields), + warnings_json=_json_dumps(row.warnings), + errors_json=_json_dumps(row.errors), + ), + (), + ) + + +def _idempotency_key( + source_family: str, + source_id: str, + source_year: int | None, + source_document_identity: str | None, + source_checksum_sha256: str | None, + record_id: str, +) -> str: + payload = "\x1f".join( + ( + source_family.strip().lower(), + source_id.strip().lower(), + "" if source_year is None else str(source_year), + source_document_identity or "", + source_checksum_sha256 or "", + record_id, + ), + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _required_decimal( + fields: Mapping[str, object], + names: Sequence[str], + field_name: str, +) -> tuple[Decimal | None, PostgreSQLNormalizedFactorInsertIssue | None]: + value = _first_field(fields, *names) + if value is None or _text_or_none(value) is None: + return None, PostgreSQLNormalizedFactorInsertIssue( + code="POSTGRESQL_NORMALIZED_FACTOR_MISSING_FACTOR_VALUE", + message="normalized factor row must include factor_value or value.", + field_name=field_name, + ) + try: + return Decimal(str(value)), None + except (InvalidOperation, ValueError): + return None, PostgreSQLNormalizedFactorInsertIssue( + code="POSTGRESQL_NORMALIZED_FACTOR_INVALID_FACTOR_VALUE", + message="factor_value must be numeric.", + field_name=field_name, + ) + + +def _first_field(fields: Mapping[str, object], *names: str) -> object | None: + for name in names: + if name in fields: + return fields[name] + return None + + +def _text_or_none(value: object | None) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _positive_int_or_none(value: object | None) -> int | None: + if value is None or value == "": + return None + try: + parsed = int(str(value)) + except (TypeError, ValueError): + return None + return parsed if parsed > 0 else None + + +def _row_status_value(status: object) -> str: + return str(getattr(status, "value", status)) + + +def _is_validated_row_status(status: object, status_enum: object) -> bool: + validated = getattr(status_enum, "VALIDATED") + if isinstance(status, status_enum): + return status is validated + return ( + status.__class__.__name__ == status_enum.__name__ + and getattr(status, "value", None) == getattr(validated, "value", None) + ) + + +def _json_dumps(value: object) -> str: + return json.dumps(value, sort_keys=True, default=str, separators=(",", ":")) + + +def _execute( + connection: object, + statement: str, + parameters: object | None = None, +) -> object: + execute = getattr(connection, "execute") + if parameters is None: + return execute(statement) + return execute(statement, parameters) + + +def _fetchone(cursor: object) -> object | None: + fetchone = getattr(cursor, "fetchone") + return fetchone() + + +def _commit(connection: object) -> None: + commit = getattr(connection, "commit", None) + if commit is not None: + commit() + + +def _rollback(connection: object) -> None: + rollback = getattr(connection, "rollback", None) + if rollback is not None: + rollback() + + +def _redact_sensitive_text(value: str) -> str: + redacted = _DSN_PATTERN.sub(r"\1***@", value) + redacted = _BARE_USERINFO_PATTERN.sub(r"***@\3", redacted) + for pattern in _SECRET_PATTERNS: + redacted = pattern.sub(r"\1=***", redacted) + return redacted + + +_DSN_PATTERN = re.compile(r"([a-z][a-z0-9+.-]*://)([^\s/?#]*@)") +_BARE_USERINFO_PATTERN = re.compile( + r"(? tuple[object, ...]: + return ( + record.normalized_factor_record_id, + record.idempotency_key_sha256, + record.source_family, + record.source_id, + record.source_year, + record.source_version, + record.record_id, + record.source_row_number, + record.source_document_reference, + record.source_artifact_reference, + record.source_checksum_sha256, + record.factor_id, + record.factor_name, + record.factor_value, + record.factor_unit, + record.validation_status, + record.run_id, + record.parser_key, + record.metadata_json, + record.normalized_fields_json, + record.warnings_json, + record.errors_json, + ) diff --git a/src/carbonfactor_parser/persistence/postgresql_runtime.py b/src/carbonfactor_parser/persistence/postgresql_runtime.py new file mode 100644 index 00000000..bde4ccce --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_runtime.py @@ -0,0 +1,79 @@ +"""PostgreSQL runtime startup helpers for Phase 1 ingestion state.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from carbonfactor_parser.persistence.postgresql_runtime_config import ( + PostgreSQLRuntimeConfig, + PostgreSQLRuntimeConfigIssue, + PostgreSQLRuntimeConfigLoadResult, +) +from carbonfactor_parser.persistence.postgresql_runtime_schema_bootstrap import ( + PostgreSQLRuntimeSchemaBootstrapResult, + bootstrap_postgresql_phase1_schema, +) +from carbonfactor_parser.persistence.postgresql_year_state_repository import ( + PostgreSQLSourceFamilyYearStateRepository, +) + + +@dataclass(frozen=True) +class PostgreSQLRuntimeStartupResult: + """Started PostgreSQL runtime components.""" + + connection: object + schema_bootstrap: PostgreSQLRuntimeSchemaBootstrapResult + year_state_repository: PostgreSQLSourceFamilyYearStateRepository + + +def connect_postgresql_runtime(config: PostgreSQLRuntimeConfig) -> object: + """Open a psycopg PostgreSQL connection from validated runtime config.""" + + import psycopg + + if config.uses_dsn: + return psycopg.connect(config.dsn) + + kwargs: dict[str, object] = { + "host": config.host, + "port": config.port, + "dbname": config.database, + "user": config.username, + "password": config.password, + } + if config.ssl_mode is not None: + kwargs["sslmode"] = config.ssl_mode + if config.application_name is not None: + kwargs["application_name"] = config.application_name + return psycopg.connect(**kwargs) + + +def start_postgresql_runtime( + config_result: PostgreSQLRuntimeConfigLoadResult, +) -> PostgreSQLRuntimeStartupResult: + """Fail closed on config issues, then connect and bootstrap schema.""" + + if not config_result.is_ready or config_result.config is None: + raise PostgreSQLRuntimeStartupBlockedError(config_result.issues) + + connection = connect_postgresql_runtime(config_result.config) + schema_bootstrap = bootstrap_postgresql_phase1_schema(connection) + repository = PostgreSQLSourceFamilyYearStateRepository( + connection, + initial_year=config_result.config.initial_year, + ) + return PostgreSQLRuntimeStartupResult( + connection=connection, + schema_bootstrap=schema_bootstrap, + year_state_repository=repository, + ) + + +class PostgreSQLRuntimeStartupBlockedError(RuntimeError): + """Raised when PostgreSQL runtime startup is blocked by safe config checks.""" + + def __init__(self, issues: tuple[PostgreSQLRuntimeConfigIssue, ...]) -> None: + self.issues = issues + codes = ", ".join(issue.code for issue in issues) or "unknown" + super().__init__(f"PostgreSQL runtime startup blocked: {codes}") diff --git a/src/carbonfactor_parser/persistence/postgresql_runtime_config.py b/src/carbonfactor_parser/persistence/postgresql_runtime_config.py new file mode 100644 index 00000000..55e1731f --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_runtime_config.py @@ -0,0 +1,247 @@ +"""Explicit PostgreSQL runtime configuration loading for local/test startup.""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from dataclasses import dataclass, field +from enum import Enum + + +POSTGRESQL_RUNTIME_DSN_ENV_VAR = "CARBONOPS_POSTGRESQL_DSN" +POSTGRESQL_RUNTIME_HOST_ENV_VAR = "CARBONOPS_POSTGRESQL_HOST" +POSTGRESQL_RUNTIME_PORT_ENV_VAR = "CARBONOPS_POSTGRESQL_PORT" +POSTGRESQL_RUNTIME_DATABASE_ENV_VAR = "CARBONOPS_POSTGRESQL_DATABASE" +POSTGRESQL_RUNTIME_USERNAME_ENV_VAR = "CARBONOPS_POSTGRESQL_USERNAME" +POSTGRESQL_RUNTIME_PASSWORD_ENV_VAR = "CARBONOPS_POSTGRESQL_PASSWORD" +POSTGRESQL_RUNTIME_SSL_MODE_ENV_VAR = "CARBONOPS_POSTGRESQL_SSL_MODE" +POSTGRESQL_RUNTIME_APPLICATION_NAME_ENV_VAR = ( + "CARBONOPS_POSTGRESQL_APPLICATION_NAME" +) +POSTGRESQL_RUNTIME_INITIAL_YEAR_ENV_VAR = ( + "CARBONOPS_POSTGRESQL_INITIAL_YEAR" +) +POSTGRESQL_RUNTIME_DEFAULT_PORT = 5432 +POSTGRESQL_RUNTIME_DEFAULT_INITIAL_YEAR = 2024 + + +class PostgreSQLRuntimeConfigStatus(str, Enum): + """Runtime configuration loading status.""" + + READY = "ready" + BLOCKED = "blocked" + + +@dataclass(frozen=True) +class PostgreSQLRuntimeConfigIssue: + """Safe runtime configuration issue without secret values.""" + + code: str + message: str + field_name: str + severity: str = "error" + + +@dataclass(frozen=True) +class PostgreSQLRuntimeConfig: + """Validated PostgreSQL runtime configuration.""" + + dsn: str | None = field(default=None, repr=False) + host: str | None = None + port: int = POSTGRESQL_RUNTIME_DEFAULT_PORT + database: str | None = None + username: str | None = None + password: str | None = field(default=None, repr=False) + ssl_mode: str | None = None + application_name: str | None = None + initial_year: int = POSTGRESQL_RUNTIME_DEFAULT_INITIAL_YEAR + + @property + def uses_dsn(self) -> bool: + """Return whether the connection should use a DSN string.""" + + return bool(self.dsn and self.dsn.strip()) + + @property + def password_configured(self) -> bool: + """Return a safe password-presence marker.""" + + return bool(self.password) + + +@dataclass(frozen=True) +class PostgreSQLRuntimeConfigLoadResult: + """Result of explicit/env PostgreSQL runtime configuration loading.""" + + status: PostgreSQLRuntimeConfigStatus + config: PostgreSQLRuntimeConfig | None + issues: tuple[PostgreSQLRuntimeConfigIssue, ...] = () + loaded_from_environment: bool = False + loaded_from_explicit_values: bool = False + + @property + def is_ready(self) -> bool: + """Return whether the config is complete enough to open PostgreSQL.""" + + return self.status is PostgreSQLRuntimeConfigStatus.READY + + +def load_postgresql_runtime_config_from_environment( + environ: Mapping[str, str] | None = None, +) -> PostgreSQLRuntimeConfigLoadResult: + """Load runtime config from environment variables only when called.""" + + return load_postgresql_runtime_config( + os.environ if environ is None else environ, + loaded_from_environment=True, + ) + + +def load_postgresql_runtime_config( + values: Mapping[str, object] | None = None, + *, + loaded_from_environment: bool = False, +) -> PostgreSQLRuntimeConfigLoadResult: + """Load runtime config from caller-provided values and fail closed.""" + + source = values or {} + dsn = _optional_text(source.get(POSTGRESQL_RUNTIME_DSN_ENV_VAR)) + host = _optional_text(source.get(POSTGRESQL_RUNTIME_HOST_ENV_VAR)) + database = _optional_text(source.get(POSTGRESQL_RUNTIME_DATABASE_ENV_VAR)) + username = _optional_text(source.get(POSTGRESQL_RUNTIME_USERNAME_ENV_VAR)) + password = _optional_text(source.get(POSTGRESQL_RUNTIME_PASSWORD_ENV_VAR)) + ssl_mode = _optional_text(source.get(POSTGRESQL_RUNTIME_SSL_MODE_ENV_VAR)) + application_name = _optional_text( + source.get(POSTGRESQL_RUNTIME_APPLICATION_NAME_ENV_VAR), + ) + port, port_issue = _parse_int( + source.get(POSTGRESQL_RUNTIME_PORT_ENV_VAR), + default=POSTGRESQL_RUNTIME_DEFAULT_PORT, + field_name=POSTGRESQL_RUNTIME_PORT_ENV_VAR, + code="POSTGRESQL_RUNTIME_CONFIG_INVALID_PORT", + message="PostgreSQL runtime port must be an integer between 1 and 65535.", + ) + initial_year, initial_year_issue = _parse_int( + source.get(POSTGRESQL_RUNTIME_INITIAL_YEAR_ENV_VAR), + default=POSTGRESQL_RUNTIME_DEFAULT_INITIAL_YEAR, + field_name=POSTGRESQL_RUNTIME_INITIAL_YEAR_ENV_VAR, + code="POSTGRESQL_RUNTIME_CONFIG_INVALID_INITIAL_YEAR", + message="PostgreSQL runtime initial year must be a positive integer.", + ) + + issues: list[PostgreSQLRuntimeConfigIssue] = [] + if port_issue is not None: + issues.append(port_issue) + elif port < 1 or port > 65535: + issues.append( + PostgreSQLRuntimeConfigIssue( + code="POSTGRESQL_RUNTIME_CONFIG_INVALID_PORT", + message="PostgreSQL runtime port must be between 1 and 65535.", + field_name=POSTGRESQL_RUNTIME_PORT_ENV_VAR, + ), + ) + if initial_year_issue is not None: + issues.append(initial_year_issue) + elif initial_year < 1: + issues.append( + PostgreSQLRuntimeConfigIssue( + code="POSTGRESQL_RUNTIME_CONFIG_INVALID_INITIAL_YEAR", + message="PostgreSQL runtime initial year must be positive.", + field_name=POSTGRESQL_RUNTIME_INITIAL_YEAR_ENV_VAR, + ), + ) + + if not dsn: + _append_missing_required( + issues, + host, + POSTGRESQL_RUNTIME_HOST_ENV_VAR, + "POSTGRESQL_RUNTIME_CONFIG_MISSING_HOST", + ) + _append_missing_required( + issues, + database, + POSTGRESQL_RUNTIME_DATABASE_ENV_VAR, + "POSTGRESQL_RUNTIME_CONFIG_MISSING_DATABASE", + ) + _append_missing_required( + issues, + username, + POSTGRESQL_RUNTIME_USERNAME_ENV_VAR, + "POSTGRESQL_RUNTIME_CONFIG_MISSING_USERNAME", + ) + _append_missing_required( + issues, + password, + POSTGRESQL_RUNTIME_PASSWORD_ENV_VAR, + "POSTGRESQL_RUNTIME_CONFIG_MISSING_PASSWORD", + ) + + config = PostgreSQLRuntimeConfig( + dsn=dsn, + host=host, + port=port, + database=database, + username=username, + password=password, + ssl_mode=ssl_mode, + application_name=application_name, + initial_year=initial_year, + ) + status = ( + PostgreSQLRuntimeConfigStatus.BLOCKED + if issues + else PostgreSQLRuntimeConfigStatus.READY + ) + return PostgreSQLRuntimeConfigLoadResult( + status=status, + config=None if issues else config, + issues=tuple(issues), + loaded_from_environment=loaded_from_environment, + loaded_from_explicit_values=not loaded_from_environment, + ) + + +def _append_missing_required( + issues: list[PostgreSQLRuntimeConfigIssue], + value: str | None, + field_name: str, + code: str, +) -> None: + if value: + return + issues.append( + PostgreSQLRuntimeConfigIssue( + code=code, + message=f"PostgreSQL runtime configuration is missing {field_name}.", + field_name=field_name, + ), + ) + + +def _optional_text(value: object) -> str | None: + if not isinstance(value, str): + return None + stripped = value.strip() + return stripped or None + + +def _parse_int( + value: object, + *, + default: int, + field_name: str, + code: str, + message: str, +) -> tuple[int, PostgreSQLRuntimeConfigIssue | None]: + if value is None or value == "": + return default, None + try: + parsed = int(str(value)) + except (TypeError, ValueError): + return default, PostgreSQLRuntimeConfigIssue( + code=code, + message=message, + field_name=field_name, + ) + return parsed, None diff --git a/src/carbonfactor_parser/persistence/postgresql_runtime_config_gate.py b/src/carbonfactor_parser/persistence/postgresql_runtime_config_gate.py new file mode 100644 index 00000000..cea86582 --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_runtime_config_gate.py @@ -0,0 +1,217 @@ +"""PostgreSQL runtime configuration gate metadata without configuration loading.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + + +class PostgreSQLRuntimeConfigGateStatus(str, Enum): + """Status values for runtime configuration gate decisions.""" + + DISABLED = "disabled" + BLOCKED = "blocked" + NOT_ENABLED = "not_enabled" + + +@dataclass(frozen=True) +class PostgreSQLRuntimeConfigGateIssue: + """Issue describing a runtime configuration gate decision.""" + + code: str + message: str + field_name: str | None = None + severity: str = "warning" + + +@dataclass(frozen=True) +class PostgreSQLRuntimeConfigGate: + """Caller-provided runtime configuration intent metadata only.""" + + requested: bool = False + safety_gate_approved: bool = False + options_contract_available: bool = False + explicit_runtime_opt_in: bool = False + secret_source_approved: bool = False + + +@dataclass(frozen=True) +class PostgreSQLRuntimeConfigGateDecision: + """Structured runtime configuration gate decision without runtime effects.""" + + status: PostgreSQLRuntimeConfigGateStatus + requested: bool + reason: str + config_loading_enabled: bool + runtime_enabled: bool + loads_environment: bool + loads_config_files: bool + loads_credentials: bool + required_future_components: tuple[str, ...] + safe_operational_notes: tuple[str, ...] + issues: tuple[PostgreSQLRuntimeConfigGateIssue, ...] = () + + def __post_init__(self) -> None: + object.__setattr__( + self, + "required_future_components", + tuple(self.required_future_components), + ) + object.__setattr__( + self, + "safe_operational_notes", + tuple(self.safe_operational_notes), + ) + object.__setattr__(self, "issues", tuple(self.issues)) + + +@dataclass(frozen=True) +class PostgreSQLRuntimeConfigGateDescription: + """Side-effect-free description of the runtime configuration gate.""" + + default_status: PostgreSQLRuntimeConfigGateStatus + disabled_by_default: bool + accepts_caller_intent: bool + loads_environment: bool + loads_config_files: bool + loads_credentials: bool + opens_connection: bool + runs_sql: bool + notes: tuple[str, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "notes", tuple(self.notes)) + + +def evaluate_postgresql_runtime_config_gate( + gate: PostgreSQLRuntimeConfigGate | None = None, +) -> PostgreSQLRuntimeConfigGateDecision: + """Evaluate runtime configuration intent without configuration loading.""" + + active_gate = gate or PostgreSQLRuntimeConfigGate() + required_components = _required_future_components(active_gate) + + if not active_gate.requested: + return PostgreSQLRuntimeConfigGateDecision( + status=PostgreSQLRuntimeConfigGateStatus.DISABLED, + requested=False, + reason="PostgreSQL runtime configuration loading is disabled by default.", + config_loading_enabled=False, + runtime_enabled=False, + loads_environment=False, + loads_config_files=False, + loads_credentials=False, + required_future_components=required_components, + safe_operational_notes=_safe_operational_notes(), + issues=( + PostgreSQLRuntimeConfigGateIssue( + code="POSTGRESQL_RUNTIME_CONFIG_DISABLED_BY_DEFAULT", + message=( + "Runtime PostgreSQL configuration loading requires explicit " + "future enablement and remains disabled." + ), + ), + ), + ) + + ready_metadata_only = not required_components + return PostgreSQLRuntimeConfigGateDecision( + status=( + PostgreSQLRuntimeConfigGateStatus.NOT_ENABLED + if ready_metadata_only + else PostgreSQLRuntimeConfigGateStatus.BLOCKED + ), + requested=True, + reason=_requested_reason(ready_metadata_only), + config_loading_enabled=False, + runtime_enabled=False, + loads_environment=False, + loads_config_files=False, + loads_credentials=False, + required_future_components=required_components, + safe_operational_notes=_safe_operational_notes(), + issues=tuple(_requested_issues(required_components)), + ) + + +def describe_postgresql_runtime_config_gate() -> PostgreSQLRuntimeConfigGateDescription: + """Describe runtime configuration gate behavior without side effects.""" + + return PostgreSQLRuntimeConfigGateDescription( + default_status=PostgreSQLRuntimeConfigGateStatus.DISABLED, + disabled_by_default=True, + accepts_caller_intent=True, + loads_environment=False, + loads_config_files=False, + loads_credentials=False, + opens_connection=False, + runs_sql=False, + notes=( + "Runtime configuration gate metadata only.", + "Default decision is disabled/no-loading.", + "Requested runtime configuration remains blocked in this boundary.", + "No environment/config file/credential loading occurs.", + ), + ) + + +def _required_future_components(gate: PostgreSQLRuntimeConfigGate) -> tuple[str, ...]: + required_components: list[str] = [] + if not gate.safety_gate_approved: + required_components.append("postgresql_implementation_safety_gate") + if not gate.options_contract_available: + required_components.append("postgresql_persistence_options_contract") + if not gate.explicit_runtime_opt_in: + required_components.append("explicit_runtime_configuration_opt_in") + if not gate.secret_source_approved: + required_components.append("approved_secret_source") + return tuple(required_components) + + +def _requested_issues( + required_components: tuple[str, ...], +) -> list[PostgreSQLRuntimeConfigGateIssue]: + if required_components: + return [ + PostgreSQLRuntimeConfigGateIssue( + code="POSTGRESQL_RUNTIME_CONFIG_BLOCKED", + message=( + "Runtime PostgreSQL configuration loading remains blocked until " + "future safety-gated components are complete." + ), + field_name="requested", + ), + ] + + return [ + PostgreSQLRuntimeConfigGateIssue( + code="POSTGRESQL_RUNTIME_CONFIG_NOT_ENABLED", + message=( + "All supplied gate metadata is marked complete, but this " + "boundary still does not enable runtime config loading." + ), + field_name="requested", + ), + ] + + +def _requested_reason(ready_metadata_only: bool) -> str: + if ready_metadata_only: + return ( + "PostgreSQL runtime configuration was requested, but this boundary " + "does not enable runtime config loading." + ) + return ( + "PostgreSQL runtime configuration was requested, but this boundary does " + "not enable config loading and required future components are not complete." + ) + + +def _safe_operational_notes() -> tuple[str, ...]: + return ( + "No environment variables are loaded.", + "No config files are read.", + "No credentials are loaded.", + "No PostgreSQL connection is opened.", + "No SQL runtime is created.", + ) diff --git a/src/carbonfactor_parser/persistence/postgresql_runtime_schema_bootstrap.py b/src/carbonfactor_parser/persistence/postgresql_runtime_schema_bootstrap.py new file mode 100644 index 00000000..fcdb1c88 --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_runtime_schema_bootstrap.py @@ -0,0 +1,127 @@ +"""PostgreSQL runtime schema bootstrap for Phase 1 tables.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + get_required_table_names, +) +from carbonfactor_parser.persistence.postgresql_schema_ddl import ( + render_postgresql_phase1_create_table_ddl, + render_postgresql_phase1_index_ddl, +) + + +@dataclass(frozen=True) +class PostgreSQLRuntimeSchemaBootstrapResult: + """Result of runtime Phase 1 schema bootstrap.""" + + required_table_names: tuple[str, ...] + present_table_names: tuple[str, ...] + missing_table_names: tuple[str, ...] + created_table_names: tuple[str, ...] + statement_count: int + + +def bootstrap_postgresql_phase1_schema( + connection: object, +) -> PostgreSQLRuntimeSchemaBootstrapResult: + """Create missing Phase 1 PostgreSQL tables with idempotent DDL.""" + + required_table_names = get_required_table_names() + present_before = _fetch_present_table_names(connection, required_table_names) + + statements = _idempotent_schema_statements() + for statement in statements: + _execute(connection, statement) + _commit(connection) + + present_after = _fetch_present_table_names(connection, required_table_names) + created = tuple( + table_name + for table_name in required_table_names + if table_name in present_after and table_name not in present_before + ) + missing_after = tuple( + table_name + for table_name in required_table_names + if table_name not in present_after + ) + return PostgreSQLRuntimeSchemaBootstrapResult( + required_table_names=required_table_names, + present_table_names=present_after, + missing_table_names=missing_after, + created_table_names=created, + statement_count=len(statements), + ) + + +def _fetch_present_table_names( + connection: object, + required_table_names: tuple[str, ...], +) -> tuple[str, ...]: + cursor = _execute( + connection, + """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name = ANY(%s) + ORDER BY table_name + """, + (list(required_table_names),), + ) + rows = _fetchall(cursor) + found = {str(row[0]) for row in rows} + return tuple( + table_name for table_name in required_table_names if table_name in found + ) + + +def _idempotent_schema_statements() -> tuple[str, ...]: + statements: list[str] = [] + for statement in render_postgresql_phase1_create_table_ddl(): + if statement.startswith("CREATE TABLE IF NOT EXISTS "): + statements.append(statement) + else: + statements.append( + statement.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ", 1), + ) + for statement in render_postgresql_phase1_index_ddl(): + if statement.startswith("CREATE INDEX IF NOT EXISTS ") or statement.startswith( + "CREATE UNIQUE INDEX IF NOT EXISTS " + ): + statements.append(statement) + elif statement.startswith("CREATE UNIQUE INDEX "): + statements.append( + statement.replace("CREATE UNIQUE INDEX ", "CREATE UNIQUE INDEX IF NOT EXISTS ", 1), + ) + else: + statements.append( + statement.replace("CREATE INDEX ", "CREATE INDEX IF NOT EXISTS ", 1), + ) + return tuple(statements) + + +def _execute( + connection: object, + statement: str, + parameters: object | None = None, +) -> object: + execute = getattr(connection, "execute") + if parameters is None: + return execute(statement) + return execute(statement, parameters) + + +def _fetchall(cursor: object) -> list[object]: + fetchall = getattr(cursor, "fetchall") + rows = fetchall() + return list(rows) + + +def _commit(connection: object) -> None: + commit = getattr(connection, "commit", None) + if commit is not None: + commit() diff --git a/src/carbonfactor_parser/persistence/postgresql_schema_bootstrap_planner.py b/src/carbonfactor_parser/persistence/postgresql_schema_bootstrap_planner.py index fae1fff4..4f215e31 100644 --- a/src/carbonfactor_parser/persistence/postgresql_schema_bootstrap_planner.py +++ b/src/carbonfactor_parser/persistence/postgresql_schema_bootstrap_planner.py @@ -18,7 +18,7 @@ POSTGRESQL_PHASE1_SCHEMA_MARKER = "phase1" POSTGRESQL_SCHEMA_BOOTSTRAP_EXECUTION_SCOPE = "planning_only_no_execution" _CREATE_INDEX_IDENTIFIER_PATTERN = re.compile( - r"\bCREATE (?:UNIQUE )?INDEX ([a-z][a-z0-9_]*)" + r"\bCREATE (?:UNIQUE )?INDEX (?:IF NOT EXISTS )?([a-z][a-z0-9_]*)" ) _CONSTRAINT_IDENTIFIER_PATTERN = re.compile(r"\bCONSTRAINT ([a-z][a-z0-9_]*)") diff --git a/src/carbonfactor_parser/persistence/postgresql_schema_catalog.py b/src/carbonfactor_parser/persistence/postgresql_schema_catalog.py index a7a4cfc7..a030a913 100644 --- a/src/carbonfactor_parser/persistence/postgresql_schema_catalog.py +++ b/src/carbonfactor_parser/persistence/postgresql_schema_catalog.py @@ -15,6 +15,29 @@ class SourceFamily(str, Enum): IPCC = "ipcc" +_SOURCE_FAMILY_ALIASES: Mapping[str, SourceFamily] = { + SourceFamily.GHG.value: SourceFamily.GHG, + "ghg_protocol": SourceFamily.GHG, + SourceFamily.DEFRA.value: SourceFamily.DEFRA, + "defra_desnz": SourceFamily.DEFRA, + "desnz": SourceFamily.DEFRA, + SourceFamily.IPCC.value: SourceFamily.IPCC, + "ipcc_efdb": SourceFamily.IPCC, +} + +_SOURCE_FAMILY_POSTGRESQL_VALUES: Mapping[SourceFamily, str] = { + SourceFamily.GHG: "ghg_protocol", + SourceFamily.DEFRA: "defra_desnz", + SourceFamily.IPCC: "ipcc_efdb", +} + +_SOURCE_FAMILY_TABLE_PREFIXES: Mapping[SourceFamily, str] = { + SourceFamily.GHG: "ghg", + SourceFamily.DEFRA: "defra", + SourceFamily.IPCC: "ipcc", +} + + class PostgreSQLDataType(str, Enum): """Conceptual PostgreSQL-oriented data types used by catalog definitions.""" @@ -33,6 +56,7 @@ class ColumnDefinition: data_type: PostgreSQLDataType nullable: bool is_primary_key: bool = False + default_sql: str | None = None @dataclass(frozen=True) @@ -149,11 +173,214 @@ def _build_shared_tables() -> tuple[TableDefinition, ...]: ), ), ), + TableDefinition( + name="parser_ingestion_runs", + columns=( + ColumnDefinition("run_id", PostgreSQLDataType.TEXT, nullable=False, is_primary_key=True), + ColumnDefinition("started_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False), + ColumnDefinition("finished_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=True), + ColumnDefinition("status", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("trigger_type", PostgreSQLDataType.TEXT, nullable=False, default_sql="'operator'"), + ColumnDefinition("config_hash", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("enabled_source_families", PostgreSQLDataType.JSONB, nullable=False, default_sql="'[]'::jsonb"), + ColumnDefinition("initial_year", PostgreSQLDataType.INTEGER, nullable=True), + ColumnDefinition("cycle_count", PostgreSQLDataType.INTEGER, nullable=True), + ColumnDefinition("total_parsed_rows", PostgreSQLDataType.INTEGER, nullable=False, default_sql="0"), + ColumnDefinition("total_inserted_count", PostgreSQLDataType.INTEGER, nullable=False, default_sql="0"), + ColumnDefinition("total_skipped_duplicate_count", PostgreSQLDataType.INTEGER, nullable=False, default_sql="0"), + ColumnDefinition("failure_count", PostgreSQLDataType.INTEGER, nullable=False, default_sql="0"), + ColumnDefinition("metadata", PostgreSQLDataType.JSONB, nullable=False, default_sql="'{}'::jsonb"), + ColumnDefinition("created_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False, default_sql="now()"), + ColumnDefinition("updated_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False, default_sql="now()"), + ), + ), + TableDefinition( + name="parser_ingestion_source_results", + columns=( + ColumnDefinition("parser_ingestion_source_result_id", PostgreSQLDataType.UUID, nullable=False, is_primary_key=True), + ColumnDefinition("run_id", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("source_family", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("target_year", PostgreSQLDataType.INTEGER, nullable=True), + ColumnDefinition("latest_year", PostgreSQLDataType.INTEGER, nullable=True), + ColumnDefinition("status", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("download_status", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("parse_status", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("validation_status", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("insert_status", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("parsed_rows", PostgreSQLDataType.INTEGER, nullable=False, default_sql="0"), + ColumnDefinition("master_inserted", PostgreSQLDataType.INTEGER, nullable=False, default_sql="0"), + ColumnDefinition("master_skipped", PostgreSQLDataType.INTEGER, nullable=False, default_sql="0"), + ColumnDefinition("detail_inserted", PostgreSQLDataType.INTEGER, nullable=False, default_sql="0"), + ColumnDefinition("detail_skipped", PostgreSQLDataType.INTEGER, nullable=False, default_sql="0"), + ColumnDefinition("issue_count", PostgreSQLDataType.INTEGER, nullable=False, default_sql="0"), + ColumnDefinition("metadata", PostgreSQLDataType.JSONB, nullable=False, default_sql="'{}'::jsonb"), + ColumnDefinition("created_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False, default_sql="now()"), + ColumnDefinition("updated_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False, default_sql="now()"), + ), + foreign_keys=( + ForeignKeyDefinition("run_id", "parser_ingestion_runs", "run_id"), + ), + unique_constraints=( + UniqueConstraintDefinition( + name="uq_parser_ingestion_source_results_run_family_year", + column_names=("run_id", "source_family", "target_year"), + ), + ), + ), + TableDefinition( + name="parser_ingestion_issues", + columns=( + ColumnDefinition("parser_ingestion_issue_id", PostgreSQLDataType.UUID, nullable=False, is_primary_key=True), + ColumnDefinition("run_id", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("source_family", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("target_year", PostgreSQLDataType.INTEGER, nullable=True), + ColumnDefinition("stage", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("code", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("severity", PostgreSQLDataType.TEXT, nullable=False, default_sql="'error'"), + ColumnDefinition("field_name", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("message", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("metadata", PostgreSQLDataType.JSONB, nullable=False, default_sql="'{}'::jsonb"), + ColumnDefinition("created_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False, default_sql="now()"), + ), + foreign_keys=( + ForeignKeyDefinition("run_id", "parser_ingestion_runs", "run_id"), + ), + indexes=( + IndexDefinition(name="idx_parser_ingestion_issues_run_id", column_names=("run_id",)), + IndexDefinition(name="idx_parser_ingestion_issues_family_year", column_names=("source_family", "target_year")), + IndexDefinition(name="idx_parser_ingestion_issues_code", column_names=("code",)), + ), + ), + TableDefinition( + name="source_family_year_states", + columns=( + ColumnDefinition( + "source_family_year_state_id", + PostgreSQLDataType.UUID, + nullable=False, + is_primary_key=True, + ), + ColumnDefinition("source_family", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("ingested_year", PostgreSQLDataType.INTEGER, nullable=False), + ColumnDefinition("created_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False), + ColumnDefinition("updated_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False), + ), + unique_constraints=( + UniqueConstraintDefinition( + name="uq_source_family_year_states_family_year", + column_names=("source_family", "ingested_year"), + ), + ), + indexes=( + IndexDefinition( + name="idx_source_family_year_states_family_year", + column_names=("source_family", "ingested_year"), + ), + ), + ), + TableDefinition( + name="normalized_factor_records", + columns=( + ColumnDefinition( + "normalized_factor_record_id", + PostgreSQLDataType.TEXT, + nullable=False, + is_primary_key=True, + ), + ColumnDefinition( + "idempotency_key_sha256", + PostgreSQLDataType.TEXT, + nullable=False, + ), + ColumnDefinition( + "source_family", + PostgreSQLDataType.TEXT, + nullable=False, + ), + ColumnDefinition("source_id", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition( + "source_year", + PostgreSQLDataType.INTEGER, + nullable=True, + ), + ColumnDefinition( + "source_version", + PostgreSQLDataType.TEXT, + nullable=True, + ), + ColumnDefinition("record_id", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition( + "source_row_number", + PostgreSQLDataType.INTEGER, + nullable=True, + ), + ColumnDefinition( + "source_document_reference", + PostgreSQLDataType.TEXT, + nullable=True, + ), + ColumnDefinition( + "source_artifact_reference", + PostgreSQLDataType.TEXT, + nullable=True, + ), + ColumnDefinition( + "source_checksum_sha256", + PostgreSQLDataType.TEXT, + nullable=True, + ), + ColumnDefinition("factor_id", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("factor_name", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition( + "factor_value", + PostgreSQLDataType.NUMERIC, + nullable=False, + ), + ColumnDefinition("factor_unit", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition( + "validation_status", + PostgreSQLDataType.TEXT, + nullable=False, + ), + ColumnDefinition("run_id", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("parser_key", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("metadata", PostgreSQLDataType.JSONB, nullable=False), + ColumnDefinition( + "normalized_fields", + PostgreSQLDataType.JSONB, + nullable=False, + ), + ColumnDefinition("warnings", PostgreSQLDataType.JSONB, nullable=False), + ColumnDefinition("errors", PostgreSQLDataType.JSONB, nullable=False), + ColumnDefinition( + "created_at", + PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, + nullable=False, + ), + ColumnDefinition( + "updated_at", + PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, + nullable=False, + ), + ), + unique_constraints=( + UniqueConstraintDefinition( + name="uq_normalized_factor_records_idempotency_key", + column_names=("idempotency_key_sha256",), + ), + ), + indexes=( + IndexDefinition( + name="idx_normalized_factor_records_source_year", + column_names=("source_family", "source_id", "source_year"), + ), + ), + ), ) def _build_source_family_tables(source_family: SourceFamily) -> tuple[TableDefinition, TableDefinition]: - family = source_family.value + family = source_family_table_prefix(source_family) master_table_name = f"{family}_emission_factor_masters" detail_table_name = f"{family}_emission_factor_details" master_id = f"{family}_emission_factor_master_id" @@ -163,22 +390,49 @@ def _build_source_family_tables(source_family: SourceFamily) -> tuple[TableDefin name=master_table_name, columns=( ColumnDefinition(master_id, PostgreSQLDataType.UUID, nullable=False, is_primary_key=True), + ColumnDefinition("source_family", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("source_year", PostgreSQLDataType.INTEGER, nullable=False), + ColumnDefinition("source_version", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("source_release", PostgreSQLDataType.TEXT, nullable=True), ColumnDefinition("source_document_id", PostgreSQLDataType.UUID, nullable=False), + ColumnDefinition("ingestion_run_id", PostgreSQLDataType.UUID, nullable=True), + ColumnDefinition("run_id", PostgreSQLDataType.TEXT, nullable=True), ColumnDefinition("master_external_key", PostgreSQLDataType.TEXT, nullable=False), - ColumnDefinition("lifecycle_status", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("status", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("artifact_reference", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("artifact_checksum_sha256", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("archive_reference", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("archive_checksum_sha256", PostgreSQLDataType.TEXT, nullable=True), ColumnDefinition("effective_from", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=True), ColumnDefinition("effective_to", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=True), ColumnDefinition("record_checksum_sha256", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("metadata", PostgreSQLDataType.JSONB, nullable=False), ColumnDefinition("created_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False), ColumnDefinition("updated_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False), ), foreign_keys=( ForeignKeyDefinition("source_document_id", "source_documents", "source_document_id"), + ForeignKeyDefinition("ingestion_run_id", "ingestion_runs", "ingestion_run_id"), ), unique_constraints=( UniqueConstraintDefinition( - name=f"uq_{master_table_name}_external_key", - column_names=("master_external_key",), + name=f"uq_{master_table_name}_family_year_version_key", + column_names=( + "source_family", + "source_year", + "source_version", + "master_external_key", + ), + ), + ), + indexes=( + IndexDefinition( + name=f"idx_{master_table_name}_source_year", + column_names=("source_family", "source_year", "source_version"), + ), + IndexDefinition( + name=f"idx_{master_table_name}_ingestion_run_id", + column_names=("ingestion_run_id",), ), ), ) @@ -189,10 +443,15 @@ def _build_source_family_tables(source_family: SourceFamily) -> tuple[TableDefin ColumnDefinition(detail_id, PostgreSQLDataType.UUID, nullable=False, is_primary_key=True), ColumnDefinition(master_id, PostgreSQLDataType.UUID, nullable=False), ColumnDefinition("detail_external_key", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("source_row_number", PostgreSQLDataType.INTEGER, nullable=True), + ColumnDefinition("factor_id", PostgreSQLDataType.TEXT, nullable=True), + ColumnDefinition("factor_name", PostgreSQLDataType.TEXT, nullable=True), ColumnDefinition("factor_value", PostgreSQLDataType.NUMERIC, nullable=False), ColumnDefinition("factor_unit", PostgreSQLDataType.TEXT, nullable=False), - ColumnDefinition("lifecycle_status", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("status", PostgreSQLDataType.TEXT, nullable=False), ColumnDefinition("record_checksum_sha256", PostgreSQLDataType.TEXT, nullable=False), + ColumnDefinition("raw_fields", PostgreSQLDataType.JSONB, nullable=False), + ColumnDefinition("normalized_fields", PostgreSQLDataType.JSONB, nullable=False), ColumnDefinition("created_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False), ColumnDefinition("updated_at", PostgreSQLDataType.TIMESTAMP_WITH_TIME_ZONE, nullable=False), ), @@ -242,5 +501,28 @@ def get_required_table_names() -> tuple[str, ...]: def get_source_family_table_names(source_family: SourceFamily | str) -> tuple[str, ...]: """Return deterministic master/detail table names for a source family.""" - family = SourceFamily(source_family) + family = coerce_source_family(source_family) return get_postgresql_phase1_schema_catalog().source_family_tables[family] + + +def coerce_source_family(source_family: SourceFamily | str) -> SourceFamily: + """Return the internal table-prefix enum for any supported family alias.""" + + if isinstance(source_family, SourceFamily): + return source_family + try: + return _SOURCE_FAMILY_ALIASES[source_family.strip().lower()] + except KeyError as exc: + raise ValueError(f"Unsupported source family: {source_family}") from exc + + +def source_family_postgresql_value(source_family: SourceFamily | str) -> str: + """Return the persisted PostgreSQL source-family identifier.""" + + return _SOURCE_FAMILY_POSTGRESQL_VALUES[coerce_source_family(source_family)] + + +def source_family_table_prefix(source_family: SourceFamily | str) -> str: + """Return the source-family table/id prefix.""" + + return _SOURCE_FAMILY_TABLE_PREFIXES[coerce_source_family(source_family)] diff --git a/src/carbonfactor_parser/persistence/postgresql_schema_ddl.py b/src/carbonfactor_parser/persistence/postgresql_schema_ddl.py index 5366774d..be0d27c8 100644 --- a/src/carbonfactor_parser/persistence/postgresql_schema_ddl.py +++ b/src/carbonfactor_parser/persistence/postgresql_schema_ddl.py @@ -84,6 +84,8 @@ def render_postgresql_table_create_table_ddl( for column in table_definition.columns: column_name = _render_identifier(column.name, "column") column_sql = f"{column_name} {_DATA_TYPE_SQL[column.data_type]}" + if column.default_sql is not None: + column_sql += f" DEFAULT {column.default_sql}" if not column.nullable: column_sql += " NOT NULL" lines.append(column_sql) @@ -131,7 +133,7 @@ def render_postgresql_table_create_table_ddl( ) inner = ",\n ".join(lines) - return f"CREATE TABLE {table_name} (\n {inner}\n);" + return f"CREATE TABLE IF NOT EXISTS {table_name} (\n {inner}\n);" def render_postgresql_table_index_ddl( @@ -152,7 +154,7 @@ def render_postgresql_table_index_ddl( ) unique_prefix = "UNIQUE " if index.unique else "" statements.append( - f"CREATE {unique_prefix}INDEX {index_name} " + f"CREATE {unique_prefix}INDEX IF NOT EXISTS {index_name} " f"ON {table_name} ({', '.join(index_column_names)});" ) diff --git a/src/carbonfactor_parser/persistence/postgresql_source_family_ids.py b/src/carbonfactor_parser/persistence/postgresql_source_family_ids.py new file mode 100644 index 00000000..f1217427 --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_source_family_ids.py @@ -0,0 +1,66 @@ +"""Stable UUID helpers for PostgreSQL source-family persistence.""" + +from __future__ import annotations + +import json +import uuid + +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + SourceFamily, + source_family_postgresql_value, +) +from carbonfactor_parser.persistence.source_family_repository import ( + SourceFamilyMasterRecord, +) + + +def source_document_uuid(record: SourceFamilyMasterRecord) -> uuid.UUID: + """Return the stable source document UUID for a master record.""" + + return _stable_uuid( + "source_document", + source_family_postgresql_value(record.source_family), + record.source_document_id, + ) + + +def ingestion_run_uuid(record: SourceFamilyMasterRecord) -> uuid.UUID | None: + """Return the stable ingestion run UUID for a master record.""" + + source = record.ingestion_run_id or record.run_id + if source is None: + source = ( + f"{source_family_postgresql_value(record.source_family)}:" + f"{record.source_year}:" + f"{record.source_version}" + ) + return _stable_uuid( + "ingestion_run", + source_family_postgresql_value(record.source_family), + source, + ) + + +def master_uuid(source_family: SourceFamily, master_id: str) -> uuid.UUID: + """Return the stable source-family master UUID.""" + + return _stable_uuid("master", source_family_postgresql_value(source_family), master_id) + + +def detail_uuid(source_family: SourceFamily, detail_id: str) -> uuid.UUID: + """Return the stable source-family detail UUID.""" + + return _stable_uuid("detail", source_family_postgresql_value(source_family), detail_id) + + +def _stable_uuid(*values: object) -> uuid.UUID: + payload = json.dumps(tuple(str(value) for value in values), separators=(",", ":")) + return uuid.uuid5(uuid.NAMESPACE_URL, payload) + + +__all__ = ( + "detail_uuid", + "ingestion_run_uuid", + "master_uuid", + "source_document_uuid", +) diff --git a/src/carbonfactor_parser/persistence/postgresql_source_family_parameters.py b/src/carbonfactor_parser/persistence/postgresql_source_family_parameters.py new file mode 100644 index 00000000..b8eb1566 --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_source_family_parameters.py @@ -0,0 +1,95 @@ +"""PostgreSQL source-family master/detail parameter mapping helpers.""" + +from __future__ import annotations + +from decimal import Decimal +import json +from typing import Mapping + +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + source_family_postgresql_value, +) +from carbonfactor_parser.persistence.postgresql_source_family_ids import ( + detail_uuid, + ingestion_run_uuid, + master_uuid, + source_document_uuid, +) +from carbonfactor_parser.persistence.source_family_repository import ( + SourceFamilyDetailRecord, + SourceFamilyMasterRecord, +) + + +def master_parameters(record: SourceFamilyMasterRecord) -> tuple[object, ...]: + """Return PostgreSQL parameters for a source-family master row.""" + + ingestion_id = ingestion_run_uuid(record) + return ( + str(master_uuid(record.source_family, record.source_family_master_id)), + source_family_postgresql_value(record.source_family), + record.source_year, + record.source_version, + record.source_release, + str(source_document_uuid(record)), + str(ingestion_id) if ingestion_id else None, + record.run_id, + record.master_external_key, + record.status, + record.artifact_reference, + record.artifact_checksum_sha256, + record.archive_reference, + record.archive_checksum_sha256, + record.effective_from, + record.effective_to, + record.record_checksum_sha256, + json_payload(record.metadata), + ) + + +def detail_parameters(record: SourceFamilyDetailRecord) -> tuple[object, ...]: + """Return PostgreSQL parameters for a source-family detail row.""" + + return ( + str(detail_uuid(record.source_family, record.source_family_detail_id)), + str(master_uuid(record.source_family, record.source_family_master_id)), + record.detail_external_key, + record.source_row_number, + record.factor_id, + record.factor_name, + str(Decimal(str(record.factor_value))), + record.factor_unit, + record.status, + record.record_checksum_sha256, + json_payload(record.raw_fields), + json_payload(record.normalized_fields), + ) + + +def json_payload(value: Mapping[str, object]) -> str: + """Return compact, sorted JSON for PostgreSQL JSONB parameters.""" + + return json.dumps(json_safe(value), sort_keys=True, separators=(",", ":")) + + +def json_safe(value: object) -> object: + """Convert values that are not JSON-native while preserving payload shape.""" + + if isinstance(value, Decimal): + return str(value) + if isinstance(value, Mapping): + return { + str(key): json_safe(item) + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + } + if isinstance(value, tuple | list): + return [json_safe(item) for item in value] + return value + + +__all__ = ( + "detail_parameters", + "json_payload", + "json_safe", + "master_parameters", +) diff --git a/src/carbonfactor_parser/persistence/postgresql_source_family_repository.py b/src/carbonfactor_parser/persistence/postgresql_source_family_repository.py new file mode 100644 index 00000000..e0dc6da0 --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_source_family_repository.py @@ -0,0 +1,207 @@ +"""PostgreSQL runtime repository for source-specific master/detail tables.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text + +from carbonfactor_parser.persistence.parsed_factor_persistence_writer import ( + persist_parsed_factor_records, +) +from carbonfactor_parser.persistence.postgresql_execution import ( + commit, + execute, + fetchone, + rollback, +) +from carbonfactor_parser.persistence.postgresql_source_family_parameters import ( + detail_parameters, + master_parameters, +) +from carbonfactor_parser.persistence.postgresql_source_family_sql import ( + detail_insert_sql, + master_insert_sql, +) +from carbonfactor_parser.persistence.postgresql_source_family_upserts import ( + ensure_ingestion_run, + ensure_source_document, +) +from carbonfactor_parser.persistence.source_family_repository import ( + SourceFamilyDetailRecord, + SourceFamilyMasterRecord, + SourceFamilyRepositoryIssue, + SourceFamilyRepositoryPersistResult, + SourceFamilyRepositoryPersistStatus, + validate_source_family_repository_inputs, +) + + +class PostgreSQLSourceSpecificFactorInsertStatus(str, Enum): + """Status values for source-specific PostgreSQL inserts.""" + + INSERTED = "inserted" + FAILED_VALIDATION = "failed_validation" + FAILED_DATABASE = "failed_database" + NO_RECORDS = "no_records" + + +@dataclass(frozen=True) +class PostgreSQLSourceSpecificFactorInsertSummary: + """User-visible source-specific master/detail insert counts.""" + + status: PostgreSQLSourceSpecificFactorInsertStatus + attempted: int + inserted: int + skipped_duplicate: int + failed: int + validation_error_count: int + master_inserted: int = 0 + master_skipped: int = 0 + detail_inserted: int = 0 + detail_skipped: int = 0 + provider_name: str = "postgresql" + issues: tuple[SourceFamilyRepositoryIssue, ...] = () + + +class PostgreSQLSourceFamilyRuntimeRepository: + """Insert parsed factor rows into PH-019 source-family tables.""" + + def __init__(self, connection: object) -> None: + if connection is None: + raise ValueError("connection must be provided.") + self._connection = connection + + @property + def provider_name(self) -> str: + """Return the repository provider name.""" + + return "postgresql" + + def insert_normalized_factor_records( + self, + batch: object, + ) -> PostgreSQLSourceSpecificFactorInsertSummary: + """Build and insert source-specific master/detail rows from a batch.""" + + result = persist_parsed_factor_records(batch, self) + attempted = result.attempted_detail_count + inserted = result.persisted_detail_count + skipped = result.skipped_detail_count + if result.status.value == "no_records": + status = PostgreSQLSourceSpecificFactorInsertStatus.NO_RECORDS + elif any( + issue.code == "POSTGRESQL_SOURCE_FAMILY_DATABASE_ERROR" + for issue in result.issues + ): + status = PostgreSQLSourceSpecificFactorInsertStatus.FAILED_DATABASE + elif result.status.value == "failed_validation": + status = PostgreSQLSourceSpecificFactorInsertStatus.FAILED_VALIDATION + else: + status = PostgreSQLSourceSpecificFactorInsertStatus.INSERTED + return PostgreSQLSourceSpecificFactorInsertSummary( + status=status, + attempted=attempted, + inserted=inserted, + skipped_duplicate=skipped, + failed=attempted if status.name.startswith("FAILED") else 0, + validation_error_count=result.validation_failure_count, + master_inserted=result.persisted_master_count, + master_skipped=result.skipped_master_count, + detail_inserted=result.persisted_detail_count, + detail_skipped=result.skipped_detail_count, + issues=tuple( + SourceFamilyRepositoryIssue( + code=issue.code, + message=issue.message, + field_name=issue.field_name, + severity=issue.severity, + ) + for issue in result.issues + ), + ) + + def persist_source_family_records( + self, + master_records: tuple[SourceFamilyMasterRecord, ...], + detail_records: tuple[SourceFamilyDetailRecord, ...], + ) -> SourceFamilyRepositoryPersistResult: + """Insert source-family records with idempotent conflict handling.""" + + validation = validate_source_family_repository_inputs( + provider_name=self.provider_name, + master_records=master_records, + detail_records=detail_records, + ) + if validation.issues: + return SourceFamilyRepositoryPersistResult( + provider_name=self.provider_name, + status=SourceFamilyRepositoryPersistStatus.FAILED_VALIDATION, + persisted_master_count=0, + persisted_detail_count=0, + validation_failure_count=len(validation.issues), + issues=validation.issues, + ) + + inserted_masters = 0 + inserted_details = 0 + try: + for master in master_records: + ensure_ingestion_run(self._connection, master) + ensure_source_document(self._connection, master) + if fetchone( + execute( + self._connection, + master_insert_sql(master.source_family), + master_parameters(master), + ) + ) is not None: + inserted_masters += 1 + + for detail in detail_records: + if fetchone( + execute( + self._connection, + detail_insert_sql(detail.source_family), + detail_parameters(detail), + ) + ) is not None: + inserted_details += 1 + + commit(self._connection) + except Exception as exc: # pragma: no cover - driver type varies + rollback(self._connection) + return SourceFamilyRepositoryPersistResult( + provider_name=self.provider_name, + status=SourceFamilyRepositoryPersistStatus.FAILED_DATABASE, + persisted_master_count=0, + persisted_detail_count=0, + issues=( + SourceFamilyRepositoryIssue( + code="POSTGRESQL_SOURCE_FAMILY_DATABASE_ERROR", + message=_redact_sensitive_text(str(exc)), + field_name="database", + ), + ), + ) + + return SourceFamilyRepositoryPersistResult( + provider_name=self.provider_name, + status=SourceFamilyRepositoryPersistStatus.DECLARED, + persisted_master_count=inserted_masters, + persisted_detail_count=inserted_details, + skipped_master_count=len(master_records) - inserted_masters, + skipped_detail_count=len(detail_records) - inserted_details, + ) + + +def _redact_sensitive_text(value: str) -> str: + return redact_sensitive_text(value) + + +__all__ = ( + "PostgreSQLSourceFamilyRuntimeRepository", + "PostgreSQLSourceSpecificFactorInsertStatus", + "PostgreSQLSourceSpecificFactorInsertSummary", +) diff --git a/src/carbonfactor_parser/persistence/postgresql_source_family_sql.py b/src/carbonfactor_parser/persistence/postgresql_source_family_sql.py new file mode 100644 index 00000000..67b0dfdf --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_source_family_sql.py @@ -0,0 +1,88 @@ +"""SQL builders for PostgreSQL source-family master/detail inserts.""" + +from __future__ import annotations + +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + SourceFamily, + source_family_table_prefix, +) +from carbonfactor_parser.persistence.source_family_repository import ( + source_family_repository_table_names, +) + + +def master_insert_sql(source_family: SourceFamily) -> str: + """Return the source-family master INSERT statement.""" + + master_table, _detail_table = source_family_repository_table_names(source_family) + family_prefix = source_family_table_prefix(source_family) + master_id = f"{family_prefix}_emission_factor_master_id" + return f""" + INSERT INTO {master_table} ( + {master_id}, + source_family, + source_year, + source_version, + source_release, + source_document_id, + ingestion_run_id, + run_id, + master_external_key, + status, + artifact_reference, + artifact_checksum_sha256, + archive_reference, + archive_checksum_sha256, + effective_from, + effective_to, + record_checksum_sha256, + metadata, + created_at, + updated_at + ) + VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, %s::jsonb, NOW(), NOW() + ) + ON CONFLICT (source_family, source_year, source_version, master_external_key) + DO NOTHING + RETURNING {master_id} + """ + + +def detail_insert_sql(source_family: SourceFamily) -> str: + """Return the source-family detail INSERT statement.""" + + master_table, detail_table = source_family_repository_table_names(source_family) + del master_table + family_prefix = source_family_table_prefix(source_family) + master_id = f"{family_prefix}_emission_factor_master_id" + detail_id = f"{family_prefix}_emission_factor_detail_id" + return f""" + INSERT INTO {detail_table} ( + {detail_id}, + {master_id}, + detail_external_key, + source_row_number, + factor_id, + factor_name, + factor_value, + factor_unit, + status, + record_checksum_sha256, + raw_fields, + normalized_fields, + created_at, + updated_at + ) + VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + %s::jsonb, %s::jsonb, NOW(), NOW() + ) + ON CONFLICT ({master_id}, detail_external_key) + DO NOTHING + RETURNING {detail_id} + """ + + +__all__ = ("detail_insert_sql", "master_insert_sql") diff --git a/src/carbonfactor_parser/persistence/postgresql_source_family_upserts.py b/src/carbonfactor_parser/persistence/postgresql_source_family_upserts.py new file mode 100644 index 00000000..1661943d --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_source_family_upserts.py @@ -0,0 +1,85 @@ +"""PostgreSQL source-family source-document and ingestion-run upserts.""" + +from __future__ import annotations + +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + source_family_postgresql_value, +) +from carbonfactor_parser.persistence.postgresql_source_family_ids import ( + ingestion_run_uuid, + source_document_uuid, +) +from carbonfactor_parser.persistence.source_family_repository import ( + SourceFamilyMasterRecord, +) + + +def ensure_ingestion_run(connection: object, master: SourceFamilyMasterRecord) -> None: + """Ensure the source-family ingestion run row exists.""" + + ingestion_run_id = ingestion_run_uuid(master) + if ingestion_run_id is None: + return + _execute( + connection, + """ + INSERT INTO ingestion_runs ( + ingestion_run_id, + run_status, + created_at, + updated_at + ) + VALUES (%s, %s, NOW(), NOW()) + ON CONFLICT (ingestion_run_id) DO NOTHING + """, + (str(ingestion_run_id), "completed"), + ) + + +def ensure_source_document(connection: object, master: SourceFamilyMasterRecord) -> None: + """Ensure the source-family source document row exists.""" + + _execute( + connection, + """ + INSERT INTO source_documents ( + source_document_id, + ingestion_run_id, + source_family, + source_document_uri, + source_checksum_sha256, + acquisition_status, + acquired_at, + created_at, + updated_at + ) + VALUES (%s, %s, %s, %s, %s, %s, NOW(), NOW(), NOW()) + ON CONFLICT (source_family, source_document_uri, source_checksum_sha256) + DO NOTHING + """, + ( + str(source_document_uuid(master)), + str(ingestion_run_uuid(master)), + source_family_postgresql_value(master.source_family), + master.artifact_reference or master.source_document_id, + master.artifact_checksum_sha256 or "checksum-unavailable", + "downloaded", + ), + ) + + +def _execute( + connection: object, + statement: str, + parameters: object | None = None, +) -> object: + execute = getattr(connection, "execute") + if parameters is None: + return execute(statement) + return execute(statement, parameters) + + +__all__ = ( + "ensure_ingestion_run", + "ensure_source_document", +) diff --git a/src/carbonfactor_parser/persistence/postgresql_year_state_repository.py b/src/carbonfactor_parser/persistence/postgresql_year_state_repository.py new file mode 100644 index 00000000..067cc3c6 --- /dev/null +++ b/src/carbonfactor_parser/persistence/postgresql_year_state_repository.py @@ -0,0 +1,159 @@ +"""PostgreSQL repository for source-family ingestion year state.""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass + +from carbonfactor_parser.persistence.postgresql_runtime_config import ( + POSTGRESQL_RUNTIME_DEFAULT_INITIAL_YEAR, +) +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + SourceFamily, + coerce_source_family, + source_family_postgresql_value, +) + + +SOURCE_FAMILY_YEAR_STATE_TABLE_NAME = "source_family_year_states" + + +@dataclass(frozen=True) +class SourceFamilyYearState: + """Latest and next target year for a source family.""" + + source_family: SourceFamily + latest_year: int | None + next_year: int + initial_year: int + + +class PostgreSQLSourceFamilyYearStateRepository: + """Runtime PostgreSQL year-state repository using a caller connection.""" + + def __init__( + self, + connection: object, + *, + initial_year: int = POSTGRESQL_RUNTIME_DEFAULT_INITIAL_YEAR, + ) -> None: + if initial_year < 1: + raise ValueError("initial_year must be positive.") + self._connection = connection + self._initial_year = initial_year + + @property + def provider_name(self) -> str: + """Return the repository provider name.""" + + return "postgresql" + + @property + def initial_year(self) -> int: + """Return the configured initial target year.""" + + return self._initial_year + + def latest_ingested_year(self, source_family: SourceFamily | str) -> int | None: + """Return the latest ingested year for a source family, if present.""" + + family = coerce_source_family(source_family) + cursor = _execute( + self._connection, + """ + SELECT MAX(ingested_year) + FROM source_family_year_states + WHERE source_family = %s + """, + (source_family_postgresql_value(family),), + ) + row = _fetchone(cursor) + if row is not None and row[0] is not None: + return int(row[0]) + if source_family_postgresql_value(family) != family.value: + legacy_cursor = _execute( + self._connection, + """ + SELECT MAX(ingested_year) + FROM source_family_year_states + WHERE source_family = %s + """, + (family.value,), + ) + legacy_row = _fetchone(legacy_cursor) + if legacy_row is not None and legacy_row[0] is not None: + return int(legacy_row[0]) + return None + + def next_target_year(self, source_family: SourceFamily | str) -> int: + """Return initial year for no data or latest ingested year plus one.""" + + latest_year = self.latest_ingested_year(source_family) + if latest_year is None: + return self._initial_year + return latest_year + 1 + + def get_year_state( + self, + source_family: SourceFamily | str, + ) -> SourceFamilyYearState: + """Return latest and next target year for a source family.""" + + family = coerce_source_family(source_family) + latest_year = self.latest_ingested_year(family) + return SourceFamilyYearState( + source_family=family, + latest_year=latest_year, + next_year=self._initial_year if latest_year is None else latest_year + 1, + initial_year=self._initial_year, + ) + + def record_ingested_year( + self, + source_family: SourceFamily | str, + ingested_year: int, + ) -> None: + """Record a completed ingested year idempotently.""" + + if ingested_year < 1: + raise ValueError("ingested_year must be positive.") + family = coerce_source_family(source_family) + _execute( + self._connection, + """ + INSERT INTO source_family_year_states ( + source_family_year_state_id, + source_family, + ingested_year, + created_at, + updated_at + ) + VALUES (%s, %s, %s, NOW(), NOW()) + ON CONFLICT (source_family, ingested_year) + DO UPDATE SET updated_at = EXCLUDED.updated_at + """, + (str(uuid.uuid4()), source_family_postgresql_value(family), ingested_year), + ) + _commit(self._connection) + + +def _execute( + connection: object, + statement: str, + parameters: object | None = None, +) -> object: + execute = getattr(connection, "execute") + if parameters is None: + return execute(statement) + return execute(statement, parameters) + + +def _fetchone(cursor: object) -> object | None: + fetchone = getattr(cursor, "fetchone") + return fetchone() + + +def _commit(connection: object) -> None: + commit = getattr(connection, "commit", None) + if commit is not None: + commit() diff --git a/src/carbonfactor_parser/persistence/production_config_boundary.py b/src/carbonfactor_parser/persistence/production_config_boundary.py new file mode 100644 index 00000000..01147430 --- /dev/null +++ b/src/carbonfactor_parser/persistence/production_config_boundary.py @@ -0,0 +1,170 @@ +"""Production runtime configuration boundary without environment loading.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Mapping + + +PRODUCTION_CONFIG_REQUIRED_ENV_VARS: tuple[str, ...] = ( + "CARBONOPS_PARSER_ENV", + "CARBONOPS_PARSER_DATABASE_PROVIDER", + "CARBONOPS_PARSER_POSTGRES_HOST", + "CARBONOPS_PARSER_POSTGRES_PORT", + "CARBONOPS_PARSER_POSTGRES_DATABASE", + "CARBONOPS_PARSER_POSTGRES_USERNAME", + "CARBONOPS_PARSER_POSTGRES_PASSWORD", + "CARBONOPS_PARSER_POSTGRES_SCHEMA", + "CARBONOPS_PARSER_RAW_ARCHIVE_PATH", + "CARBONOPS_PARSER_LOG_LEVEL", +) + +PRODUCTION_CONFIG_SECRET_ENV_VARS: tuple[str, ...] = ( + "CARBONOPS_PARSER_POSTGRES_PASSWORD", +) + +_VALID_LOG_LEVELS = frozenset({"debug", "info", "warning", "error", "critical"}) + + +@dataclass(frozen=True) +class ProductionConfigValidationIssue: + """Safe production config validation issue with no configured values.""" + + code: str + message: str + field_name: str + severity: str = "error" + + +@dataclass(frozen=True) +class ProductionConfigValidationResult: + """Side-effect-free validation result for production config mappings.""" + + issues: tuple[ProductionConfigValidationIssue, ...] = () + + @property + def is_valid(self) -> bool: + return not self.issues + + +@dataclass(frozen=True) +class ProductionConfigBoundaryDescription: + """Shared Python/.NET production configuration expectations.""" + + required_env_vars: tuple[str, ...] + secret_env_vars: tuple[str, ...] + provider: str + loads_environment: bool + loads_config_files: bool + loads_credentials: bool + logs_secret_values: bool + notes: tuple[str, ...] + + +def describe_production_config_boundary() -> ProductionConfigBoundaryDescription: + """Describe production config expectations without reading configuration.""" + + return ProductionConfigBoundaryDescription( + required_env_vars=PRODUCTION_CONFIG_REQUIRED_ENV_VARS, + secret_env_vars=PRODUCTION_CONFIG_SECRET_ENV_VARS, + provider="postgres", + loads_environment=False, + loads_config_files=False, + loads_credentials=False, + logs_secret_values=False, + notes=( + "Callers pass an explicit mapping for validation.", + "CARBONOPS_PARSER_POSTGRES_PASSWORD is required but never returned.", + "Connection strings are not accepted as production config values.", + "Validation messages name keys only and do not echo configured values.", + ), + ) + + +def validate_production_config_mapping( + values: Mapping[str, str | None], +) -> ProductionConfigValidationResult: + """Validate required production config keys without loading env or secrets.""" + + issues: list[ProductionConfigValidationIssue] = [] + + for env_var in PRODUCTION_CONFIG_REQUIRED_ENV_VARS: + if not _has_text(values.get(env_var)): + issues.append( + ProductionConfigValidationIssue( + code="PRODUCTION_CONFIG_MISSING_REQUIRED_ENV_VAR", + message=f"{env_var} must be set for production startup.", + field_name=env_var, + ), + ) + + provider = values.get("CARBONOPS_PARSER_DATABASE_PROVIDER") + if _has_text(provider) and provider.strip().lower() != "postgres": + issues.append( + ProductionConfigValidationIssue( + code="PRODUCTION_CONFIG_UNSUPPORTED_DATABASE_PROVIDER", + message="Unsupported database provider. Phase 1 supports postgres only.", + field_name="CARBONOPS_PARSER_DATABASE_PROVIDER", + ), + ) + + _validate_port(values.get("CARBONOPS_PARSER_POSTGRES_PORT"), issues) + _validate_log_level(values.get("CARBONOPS_PARSER_LOG_LEVEL"), issues) + + if _has_text(values.get("CARBONOPS_PARSER_POSTGRES_CONNECTION_STRING")): + issues.append( + ProductionConfigValidationIssue( + code="PRODUCTION_CONFIG_RAW_CONNECTION_STRING_NOT_ALLOWED", + message=( + "Raw PostgreSQL connection strings are not accepted; use split " + "non-secret fields and CARBONOPS_PARSER_POSTGRES_PASSWORD." + ), + field_name="CARBONOPS_PARSER_POSTGRES_CONNECTION_STRING", + ), + ) + + return ProductionConfigValidationResult(issues=tuple(issues)) + + +def _validate_port( + raw_value: str | None, + issues: list[ProductionConfigValidationIssue], +) -> None: + if not _has_text(raw_value): + return + try: + port = int(raw_value) + except ValueError: + port = 0 + if not 1 <= port <= 65535: + issues.append( + ProductionConfigValidationIssue( + code="PRODUCTION_CONFIG_INVALID_POSTGRES_PORT", + message="CARBONOPS_PARSER_POSTGRES_PORT must be an integer between 1 and 65535.", + field_name="CARBONOPS_PARSER_POSTGRES_PORT", + ), + ) + + +def _validate_log_level( + raw_value: str | None, + issues: list[ProductionConfigValidationIssue], +) -> None: + if not _has_text(raw_value): + return + if raw_value.strip().lower() not in _VALID_LOG_LEVELS: + issues.append( + ProductionConfigValidationIssue( + code="PRODUCTION_CONFIG_INVALID_LOG_LEVEL", + message=( + "CARBONOPS_PARSER_LOG_LEVEL must be one of debug, info, " + "warning, error, or critical." + ), + field_name="CARBONOPS_PARSER_LOG_LEVEL", + ), + ) + + +def _has_text(value: str | None) -> bool: + return isinstance(value, str) and bool(value.strip()) + diff --git a/src/carbonfactor_parser/persistence/source_document_repository.py b/src/carbonfactor_parser/persistence/source_document_repository.py new file mode 100644 index 00000000..709911c7 --- /dev/null +++ b/src/carbonfactor_parser/persistence/source_document_repository.py @@ -0,0 +1,135 @@ +"""Runtime-passive source document repository contract.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Protocol, runtime_checkable + +from carbonfactor_parser.source_acquisition.models import ( + SourceDocumentPersistenceRecord, +) + + +class SourceDocumentRepositoryPersistStatus(str, Enum): + """Deterministic metadata-only source document persist status values.""" + + DECLARED = "declared" + FAILED_VALIDATION = "failed_validation" + + +@dataclass(frozen=True) +class SourceDocumentRepositoryIssue: + """Metadata-only source document repository contract issue.""" + + code: str + message: str + field_name: str + severity: str = "error" + + +@dataclass(frozen=True) +class SourceDocumentRepositoryPersistResult: + """Metadata-only source document repository persist result.""" + + provider_name: str + status: SourceDocumentRepositoryPersistStatus + persisted_count: int + issues: tuple[SourceDocumentRepositoryIssue, ...] = () + + +@runtime_checkable +class SourceDocumentRepository(Protocol): + """Protocol for metadata-only source document repositories.""" + + @property + def provider_name(self) -> str: + """Human-readable provider name.""" + + def persist_source_documents( + self, + records: tuple[SourceDocumentPersistenceRecord, ...], + ) -> SourceDocumentRepositoryPersistResult: + """Persist source document metadata contractually without side effects.""" + + +def create_source_document_repository_persist_result( + *, + provider_name: str, + records: ( + tuple[SourceDocumentPersistenceRecord, ...] + | list[SourceDocumentPersistenceRecord] + ), + issues: ( + tuple[SourceDocumentRepositoryIssue, ...] + | list[SourceDocumentRepositoryIssue] + ) = (), +) -> SourceDocumentRepositoryPersistResult: + """Create deterministic metadata-only source document persist result.""" + + record_snapshot = tuple(records) + validation_issues = list( + validate_source_document_repository_inputs( + provider_name=provider_name, + records=record_snapshot, + ).issues, + ) + validation_issues.extend(issues) + + status = ( + SourceDocumentRepositoryPersistStatus.FAILED_VALIDATION + if validation_issues + else SourceDocumentRepositoryPersistStatus.DECLARED + ) + + return SourceDocumentRepositoryPersistResult( + provider_name=provider_name, + status=status, + persisted_count=0 if validation_issues else len(record_snapshot), + issues=tuple(validation_issues), + ) + + +@dataclass(frozen=True) +class SourceDocumentRepositoryValidationResult: + """Validation result for source document repository metadata.""" + + issues: tuple[SourceDocumentRepositoryIssue, ...] = () + + @property + def is_valid(self) -> bool: + return not self.issues + + +def validate_source_document_repository_inputs( + *, + provider_name: str, + records: tuple[SourceDocumentPersistenceRecord, ...], +) -> SourceDocumentRepositoryValidationResult: + """Validate repository inputs without runtime side effects.""" + + issues: list[SourceDocumentRepositoryIssue] = [] + + if not isinstance(provider_name, str) or not provider_name.strip(): + issues.append( + SourceDocumentRepositoryIssue( + code="SOURCE_DOCUMENT_REPOSITORY_MISSING_PROVIDER_NAME", + message="provider_name must be a non-empty string.", + field_name="provider_name", + ), + ) + + for index, record in enumerate(records): + if not isinstance(record, SourceDocumentPersistenceRecord): + issues.append( + SourceDocumentRepositoryIssue( + code="SOURCE_DOCUMENT_REPOSITORY_INVALID_RECORD", + message=( + "records must contain SourceDocumentPersistenceRecord " + "instances." + ), + field_name=f"records[{index}]", + ), + ) + + return SourceDocumentRepositoryValidationResult(issues=tuple(issues)) diff --git a/src/carbonfactor_parser/persistence/source_family_repository.py b/src/carbonfactor_parser/persistence/source_family_repository.py new file mode 100644 index 00000000..f2f98371 --- /dev/null +++ b/src/carbonfactor_parser/persistence/source_family_repository.py @@ -0,0 +1,373 @@ +"""Runtime-passive source-family master/detail repository contract.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Protocol, runtime_checkable + +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + SourceFamily, + get_source_family_table_names, +) + + +class SourceFamilyRepositoryPersistStatus(str, Enum): + """Deterministic metadata-only source-family persist status values.""" + + DECLARED = "declared" + FAILED_VALIDATION = "failed_validation" + FAILED_DATABASE = "failed_database" + + +@dataclass(frozen=True) +class SourceFamilyMasterRecord: + """Runtime-passive source-family master record contract.""" + + source_family: SourceFamily + source_family_master_id: str + source_year: int + source_version: str + source_release: str | None + source_document_id: str + ingestion_run_id: str | None + run_id: str | None + master_external_key: str + status: str + artifact_reference: str | None + artifact_checksum_sha256: str | None + archive_reference: str | None + archive_checksum_sha256: str | None + effective_from: str | None + effective_to: str | None + record_checksum_sha256: str + metadata: dict[str, object] + created_at: str + updated_at: str + + @property + def lifecycle_status(self) -> str: + """Backward-compatible alias for the persisted master status.""" + + return self.status + + +@dataclass(frozen=True) +class SourceFamilyDetailRecord: + """Runtime-passive source-family detail record contract.""" + + source_family: SourceFamily + source_family_detail_id: str + source_family_master_id: str + detail_external_key: str + source_row_number: int | None + factor_id: str | None + factor_name: str | None + factor_value: str + factor_unit: str + status: str + record_checksum_sha256: str + raw_fields: dict[str, object] + normalized_fields: dict[str, object] + created_at: str + updated_at: str + + @property + def lifecycle_status(self) -> str: + """Backward-compatible alias for the persisted detail status.""" + + return self.status + + +@dataclass(frozen=True) +class SourceFamilyRepositoryIssue: + """Metadata-only source-family repository contract issue.""" + + code: str + message: str + field_name: str + severity: str = "error" + + +@dataclass(frozen=True) +class SourceFamilyRepositoryPersistResult: + """Metadata-only source-family master/detail repository persist result.""" + + provider_name: str + status: SourceFamilyRepositoryPersistStatus + persisted_master_count: int + persisted_detail_count: int + skipped_master_count: int = 0 + skipped_detail_count: int = 0 + validation_failure_count: int = 0 + issues: tuple[SourceFamilyRepositoryIssue, ...] = () + + +@runtime_checkable +class SourceFamilyRepository(Protocol): + """Protocol for metadata-only source-family master/detail repositories.""" + + @property + def provider_name(self) -> str: + """Human-readable provider name.""" + + def persist_source_family_records( + self, + master_records: tuple[SourceFamilyMasterRecord, ...], + detail_records: tuple[SourceFamilyDetailRecord, ...], + ) -> SourceFamilyRepositoryPersistResult: + """Persist source-family master/detail records without side effects.""" + + +def create_source_family_repository_persist_result( + *, + provider_name: str, + master_records: ( + tuple[SourceFamilyMasterRecord, ...] | list[SourceFamilyMasterRecord] + ), + detail_records: ( + tuple[SourceFamilyDetailRecord, ...] | list[SourceFamilyDetailRecord] + ), + issues: tuple[SourceFamilyRepositoryIssue, ...] + | list[SourceFamilyRepositoryIssue] = (), +) -> SourceFamilyRepositoryPersistResult: + """Create deterministic metadata-only source-family persist result.""" + + master_snapshot = tuple(master_records) + detail_snapshot = tuple(detail_records) + validation_issues = list( + validate_source_family_repository_inputs( + provider_name=provider_name, + master_records=master_snapshot, + detail_records=detail_snapshot, + ).issues, + ) + validation_issues.extend(issues) + + status = ( + SourceFamilyRepositoryPersistStatus.FAILED_VALIDATION + if validation_issues + else SourceFamilyRepositoryPersistStatus.DECLARED + ) + + return SourceFamilyRepositoryPersistResult( + provider_name=provider_name, + status=status, + persisted_master_count=0 if validation_issues else len(master_snapshot), + persisted_detail_count=0 if validation_issues else len(detail_snapshot), + validation_failure_count=len(validation_issues), + issues=tuple(validation_issues), + ) + + +@dataclass(frozen=True) +class SourceFamilyRepositoryValidationResult: + """Validation result for source-family repository metadata.""" + + issues: tuple[SourceFamilyRepositoryIssue, ...] = () + + @property + def is_valid(self) -> bool: + return not self.issues + + +def validate_source_family_repository_inputs( + *, + provider_name: str, + master_records: tuple[SourceFamilyMasterRecord, ...], + detail_records: tuple[SourceFamilyDetailRecord, ...], +) -> SourceFamilyRepositoryValidationResult: + """Validate source-family repository inputs without runtime side effects.""" + + issues: list[SourceFamilyRepositoryIssue] = [] + + if not isinstance(provider_name, str) or not provider_name.strip(): + issues.append( + SourceFamilyRepositoryIssue( + code="SOURCE_FAMILY_REPOSITORY_MISSING_PROVIDER_NAME", + message="provider_name must be a non-empty string.", + field_name="provider_name", + ), + ) + + master_keys: set[tuple[SourceFamily, str]] = set() + for index, record in enumerate(master_records): + if not isinstance(record, SourceFamilyMasterRecord): + issues.append( + SourceFamilyRepositoryIssue( + code="SOURCE_FAMILY_REPOSITORY_INVALID_MASTER_RECORD", + message=( + "master_records must contain SourceFamilyMasterRecord " + "instances." + ), + field_name=f"master_records[{index}]", + ), + ) + continue + + source_family = _validate_record_source_family( + record.source_family, + f"master_records[{index}].source_family", + issues, + ) + _validate_master_record(record, index, issues) + if _is_non_empty_string(record.source_family_master_id): + if source_family is not None: + master_keys.add((source_family, record.source_family_master_id)) + + for index, record in enumerate(detail_records): + if not isinstance(record, SourceFamilyDetailRecord): + issues.append( + SourceFamilyRepositoryIssue( + code="SOURCE_FAMILY_REPOSITORY_INVALID_DETAIL_RECORD", + message=( + "detail_records must contain SourceFamilyDetailRecord " + "instances." + ), + field_name=f"detail_records[{index}]", + ), + ) + continue + + _validate_detail_record(record, index, master_keys, issues) + + return SourceFamilyRepositoryValidationResult(issues=tuple(issues)) + + +def source_family_repository_table_names( + source_family: SourceFamily | str, +) -> tuple[str, str]: + """Return the master/detail table names owned by a source family.""" + + table_names = get_source_family_table_names(source_family) + return table_names[0], table_names[1] + + +def _validate_master_record( + record: SourceFamilyMasterRecord, + index: int, + issues: list[SourceFamilyRepositoryIssue], +) -> None: + for field_name in ( + "source_family_master_id", + "source_version", + "source_document_id", + "master_external_key", + "status", + "record_checksum_sha256", + "created_at", + "updated_at", + ): + _append_required_string_issue( + issues, + getattr(record, field_name), + f"master_records[{index}].{field_name}", + ) + if not isinstance(record.source_year, int) or isinstance(record.source_year, bool): + issues.append( + SourceFamilyRepositoryIssue( + code="SOURCE_FAMILY_REPOSITORY_INVALID_SOURCE_YEAR", + message="source_year must be an integer.", + field_name=f"master_records[{index}].source_year", + ), + ) + if not isinstance(record.metadata, dict): + issues.append( + SourceFamilyRepositoryIssue( + code="SOURCE_FAMILY_REPOSITORY_INVALID_METADATA", + message="metadata must be a dictionary.", + field_name=f"master_records[{index}].metadata", + ), + ) + + +def _validate_detail_record( + record: SourceFamilyDetailRecord, + index: int, + master_keys: set[tuple[SourceFamily, str]], + issues: list[SourceFamilyRepositoryIssue], +) -> None: + source_family = _validate_record_source_family( + record.source_family, + f"detail_records[{index}].source_family", + issues, + ) + + for field_name in ( + "source_family_detail_id", + "source_family_master_id", + "detail_external_key", + "factor_value", + "factor_unit", + "status", + "record_checksum_sha256", + "created_at", + "updated_at", + ): + _append_required_string_issue( + issues, + getattr(record, field_name), + f"detail_records[{index}].{field_name}", + ) + for field_name in ("raw_fields", "normalized_fields"): + if not isinstance(getattr(record, field_name), dict): + issues.append( + SourceFamilyRepositoryIssue( + code="SOURCE_FAMILY_REPOSITORY_INVALID_DETAIL_FIELDS", + message="detail field payloads must be dictionaries.", + field_name=f"detail_records[{index}].{field_name}", + ), + ) + + if ( + source_family is not None + and _is_non_empty_string(record.source_family_master_id) + and (source_family, record.source_family_master_id) not in master_keys + ): + issues.append( + SourceFamilyRepositoryIssue( + code="SOURCE_FAMILY_REPOSITORY_DETAIL_MASTER_NOT_DECLARED", + message=( + "detail record source_family_master_id must reference a " + "declared master record for the same source family." + ), + field_name=f"detail_records[{index}].source_family_master_id", + ), + ) + + +def _validate_record_source_family( + value: object, + field_name: str, + issues: list[SourceFamilyRepositoryIssue], +) -> SourceFamily | None: + try: + return SourceFamily(value) + except (TypeError, ValueError): + issues.append( + SourceFamilyRepositoryIssue( + code="SOURCE_FAMILY_REPOSITORY_INVALID_SOURCE_FAMILY", + message="source_family must be a supported Phase 1 source family.", + field_name=field_name, + ), + ) + return None + + +def _append_required_string_issue( + issues: list[SourceFamilyRepositoryIssue], + value: object, + field_name: str, +) -> None: + if not _is_non_empty_string(value): + issues.append( + SourceFamilyRepositoryIssue( + code="SOURCE_FAMILY_REPOSITORY_MISSING_REQUIRED_FIELD", + message="required fields must be non-empty strings.", + field_name=field_name, + ), + ) + + +def _is_non_empty_string(value: object) -> bool: + return isinstance(value, str) and bool(value.strip()) diff --git a/src/carbonfactor_parser/pipeline/configured_cycle_config.py b/src/carbonfactor_parser/pipeline/configured_cycle_config.py new file mode 100644 index 00000000..a4ff754c --- /dev/null +++ b/src/carbonfactor_parser/pipeline/configured_cycle_config.py @@ -0,0 +1,402 @@ +"""Configuration loading for the configured ingestion cycle runner.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from pathlib import Path +from typing import Mapping, Sequence + +from carbonfactor_parser.persistence.postgresql_runtime_config import ( + POSTGRESQL_RUNTIME_APPLICATION_NAME_ENV_VAR, + POSTGRESQL_RUNTIME_DATABASE_ENV_VAR, + POSTGRESQL_RUNTIME_DEFAULT_INITIAL_YEAR, + POSTGRESQL_RUNTIME_DSN_ENV_VAR, + POSTGRESQL_RUNTIME_HOST_ENV_VAR, + POSTGRESQL_RUNTIME_INITIAL_YEAR_ENV_VAR, + POSTGRESQL_RUNTIME_PASSWORD_ENV_VAR, + POSTGRESQL_RUNTIME_PORT_ENV_VAR, + POSTGRESQL_RUNTIME_SSL_MODE_ENV_VAR, + POSTGRESQL_RUNTIME_USERNAME_ENV_VAR, + PostgreSQLRuntimeConfigLoadResult, + load_postgresql_runtime_config, + load_postgresql_runtime_config_from_environment, +) +from carbonfactor_parser.pipeline.defra_desnz_production_e2e import ( + DEFRA_DESNZ_SOURCE_FAMILY, +) +from carbonfactor_parser.pipeline.ghg_protocol_production_e2e import ( + GHG_PROTOCOL_SOURCE_FAMILY, +) +from carbonfactor_parser.pipeline.ipcc_efdb_production_e2e import ( + IPCC_EFDB_SOURCE_FAMILY, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + PRODUCTION_E2E_SOURCE_FAMILIES, +) + + +CONFIGURED_CYCLE_SOURCE_FAMILIES = PRODUCTION_E2E_SOURCE_FAMILIES +_LIVE_SOURCE_ACCESS_CONFIG_KEYS = ( + ("allow_live_source_access",), + ("allowLiveSourceAccess",), + ("live_source_access", "enabled"), + ("liveSourceAccess", "enabled"), + ("real_source_smoke", "allow_live_source_access"), + ("realSourceSmoke", "allowLiveSourceAccess"), +) + + +@dataclass(frozen=True) +class ConfiguredSourceYearArtifact: + """Config entry for one source-family year artifact.""" + + year: int + artifact_url: str + publication_url: str = "" + title: str = "" + version_label: str = "" + content_type: str = "text/csv" + format_hint: str = "csv" + + +@dataclass(frozen=True) +class ConfiguredCycleRunnerConfig: + """Validated runtime configuration for the cycle runner.""" + + postgresql_config_result: PostgreSQLRuntimeConfigLoadResult + archive_root: Path + enabled_source_families: tuple[str, ...] = CONFIGURED_CYCLE_SOURCE_FAMILIES + initial_year: int = POSTGRESQL_RUNTIME_DEFAULT_INITIAL_YEAR + cycle_interval_seconds: float = 0.0 + max_cycles: int | None = 1 + source_years: Mapping[str, Mapping[int, ConfiguredSourceYearArtifact]] | None = None + allow_live_source_access: bool = False + + +def load_configured_cycle_runner_config( + config_path: str | Path | None = None, + *, + environ: Mapping[str, str] | None = None, + max_cycles: int | None = None, +) -> ConfiguredCycleRunnerConfig: + """Load cycle-runner config from JSON file plus PostgreSQL env fallback.""" + + payload = _load_config_payload(config_path) + postgresql_config_result = _load_postgresql_config(payload, environ) + archive_root = _validated_archive_root( + _nested_get(payload, ("archive_root",)) + or _nested_get(payload, ("storage", "rawArchivePath")) + or _nested_get(payload, ("storage", "raw_archive_path")) + or "./data/raw", + ) + initial_year = _positive_int( + _coalesce_config_values( + _nested_get(payload, ("initial_year",)), + _nested_get(payload, ("cycle", "initial_year")), + _nested_get(payload, ("execution", "initialYear")), + POSTGRESQL_RUNTIME_DEFAULT_INITIAL_YEAR, + ), + field_name="initial_year", + ) + interval_seconds = _non_negative_float( + _coalesce_config_values( + _nested_get(payload, ("cycle_interval_seconds",)), + _nested_get(payload, ("cycle", "interval_seconds")), + 0, + ), + field_name="cycle_interval_seconds", + ) + configured_max_cycles = _optional_positive_int( + max_cycles + if max_cycles is not None + else _coalesce_config_values( + _nested_get(payload, ("max_cycles",)), + _nested_get(payload, ("cycle", "max_cycles")), + 1, + ), + field_name="max_cycles", + ) + enabled_source_families = _enabled_source_families(payload) + source_years = _source_years_from_payload(payload) + allow_live_source_access = _allow_live_source_access_value(payload) + + return ConfiguredCycleRunnerConfig( + postgresql_config_result=postgresql_config_result, + archive_root=archive_root, + enabled_source_families=enabled_source_families, + initial_year=initial_year, + cycle_interval_seconds=interval_seconds, + max_cycles=configured_max_cycles, + source_years=source_years, + allow_live_source_access=allow_live_source_access, + ) + + +def _load_config_payload(config_path: str | Path | None) -> Mapping[str, object]: + if config_path is None: + return {} + path = Path(config_path) + text = path.read_text(encoding="utf-8") + if path.suffix.lower() in {".json", ""}: + payload = json.loads(text) + if not isinstance(payload, Mapping): + raise ValueError("Cycle runner config root must be an object.") + return payload + raise ValueError("Cycle runner config currently supports JSON files.") + + +def _load_postgresql_config( + payload: Mapping[str, object], + environ: Mapping[str, str] | None, +) -> PostgreSQLRuntimeConfigLoadResult: + postgresql = _nested_get(payload, ("postgresql",)) + database = _nested_get(payload, ("database",)) + if isinstance(postgresql, Mapping) or isinstance(database, Mapping): + source = postgresql if isinstance(postgresql, Mapping) else database + values = { + POSTGRESQL_RUNTIME_DSN_ENV_VAR: source.get("dsn"), + POSTGRESQL_RUNTIME_HOST_ENV_VAR: source.get("host"), + POSTGRESQL_RUNTIME_PORT_ENV_VAR: source.get("port"), + POSTGRESQL_RUNTIME_DATABASE_ENV_VAR: source.get("database"), + POSTGRESQL_RUNTIME_USERNAME_ENV_VAR: source.get("username"), + POSTGRESQL_RUNTIME_PASSWORD_ENV_VAR: source.get("password"), + POSTGRESQL_RUNTIME_SSL_MODE_ENV_VAR: source.get("ssl_mode") + or source.get("sslMode"), + POSTGRESQL_RUNTIME_APPLICATION_NAME_ENV_VAR: source.get( + "application_name", + ) + or source.get("applicationName"), + POSTGRESQL_RUNTIME_INITIAL_YEAR_ENV_VAR: source.get("initial_year") + or source.get("initialYear") + or _nested_get(payload, ("initial_year",)), + } + return load_postgresql_runtime_config(values) + return load_postgresql_runtime_config_from_environment(environ) + + +def _source_years_from_payload( + payload: Mapping[str, object], +) -> Mapping[str, Mapping[int, ConfiguredSourceYearArtifact]]: + source_year_payload = ( + _nested_get(payload, ("source_years",)) + or _nested_get(payload, ("sourceYears",)) + or _nested_get(payload, ("sources",)) + or {} + ) + if not isinstance(source_year_payload, Mapping): + raise ValueError("source_years must be an object keyed by source family.") + + result: dict[str, dict[int, ConfiguredSourceYearArtifact]] = {} + for raw_family, raw_years in source_year_payload.items(): + family = _source_family_alias(str(raw_family)) + if family is None: + raise ValueError( + f"Unsupported source family in source_years: {raw_family}.", + ) + years_payload = _extract_years_payload(raw_years) + if not isinstance(years_payload, Mapping): + raise ValueError(f"source_years.{family} must be an object keyed by year.") + years: dict[int, ConfiguredSourceYearArtifact] = {} + for raw_year, raw_entry in years_payload.items(): + entry = raw_entry if isinstance(raw_entry, Mapping) else {} + try: + year = _positive_int(raw_year, field_name="source_year") + except ValueError as exc: + raise ValueError( + f"source_years.{family} year key must be a positive integer.", + ) from exc + artifact_url = str( + entry.get("artifact_url") + or entry.get("artifactUrl") + or entry.get("path") + or entry.get("local_path") + or entry.get("localPath") + or "" + ).strip() + if not artifact_url: + raise ValueError( + f"source_years.{family}.{year} requires artifact_url.", + ) + years[year] = ConfiguredSourceYearArtifact( + year=year, + artifact_url=artifact_url, + publication_url=str( + entry.get("publication_url") + or entry.get("publicationUrl") + or artifact_url + ), + title=str(entry.get("title") or f"{family} {year}"), + version_label=str( + entry.get("version_label") + or entry.get("versionLabel") + or f"{year}" + ), + content_type=str( + entry.get("content_type") + or entry.get("contentType") + or "text/csv", + ), + format_hint=str( + entry.get("format_hint") or entry.get("formatHint") or "csv", + ), + ) + result[family] = years + return result + + +def _extract_years_payload(raw_value: object) -> object: + if not isinstance(raw_value, Mapping): + return raw_value + return raw_value.get("years") or raw_value.get("source_years") or raw_value + + +def _enabled_source_families(payload: Mapping[str, object]) -> tuple[str, ...]: + configured = ( + _nested_get(payload, ("enabled_source_families",)) + or _nested_get(payload, ("enabledSourceFamilies",)) + or _nested_get(payload, ("sources_enabled",)) + ) + if isinstance(configured, str): + raw_values: Sequence[object] = tuple( + value.strip() for value in configured.split(",") + ) + elif isinstance(configured, Sequence): + raw_values = configured + else: + raw_values = CONFIGURED_CYCLE_SOURCE_FAMILIES + selected = [] + for raw_value in raw_values: + family = _source_family_alias(str(raw_value)) + if family is None: + raise ValueError(f"Unsupported enabled source family: {raw_value}.") + if family not in selected: + selected.append(family) + return tuple(selected) or CONFIGURED_CYCLE_SOURCE_FAMILIES + + +def _validated_archive_root(value: object) -> Path: + if not isinstance(value, (str, Path)): + raise ValueError("archive_root must be a string path.") + path = Path(str(value)) + if path.exists() and not path.is_dir(): + raise ValueError("archive_root must be a directory path.") + if not path.exists(): + try: + path.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise ValueError("archive_root could not be created.") from exc + if not path.is_dir(): + raise ValueError("archive_root must resolve to a directory.") + if not path.exists() or not path.stat(): + raise ValueError("archive_root is not accessible.") + return path + + +def _non_negative_float(value: object, *, field_name: str) -> float: + try: + parsed = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field_name} must be a non-negative number.") from exc + if parsed < 0: + raise ValueError(f"{field_name} must be a non-negative number.") + return parsed + + +def _source_family_alias(value: str) -> str | None: + normalized = value.strip().lower() + aliases = { + "ghg": GHG_PROTOCOL_SOURCE_FAMILY, + "ghg_protocol": GHG_PROTOCOL_SOURCE_FAMILY, + "ghgprotocol": GHG_PROTOCOL_SOURCE_FAMILY, + "defra": DEFRA_DESNZ_SOURCE_FAMILY, + "desnz": DEFRA_DESNZ_SOURCE_FAMILY, + "defra_desnz": DEFRA_DESNZ_SOURCE_FAMILY, + "defradesnz": DEFRA_DESNZ_SOURCE_FAMILY, + "ipcc": IPCC_EFDB_SOURCE_FAMILY, + "ipcc_efdb": IPCC_EFDB_SOURCE_FAMILY, + "ipccefdb": IPCC_EFDB_SOURCE_FAMILY, + } + return aliases.get(normalized) + + +def _nested_get(payload: Mapping[str, object], keys: tuple[str, ...]) -> object | None: + current: object = payload + for key in keys: + if not isinstance(current, Mapping): + return None + current = current.get(key) + return current + + +def _nested_lookup( + payload: Mapping[str, object], + keys: tuple[str, ...], +) -> tuple[bool, object | None]: + current: object = payload + for key in keys: + if not isinstance(current, Mapping) or key not in current: + return False, None + current = current[key] + return True, current + + +def _coalesce_config_values(*values: object) -> object: + for value in values: + if value is not None: + return value + return None + + +def _positive_int(value: object, *, field_name: str) -> int: + try: + parsed = int(str(value)) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field_name} must be a positive integer.") from exc + if parsed < 1: + raise ValueError(f"{field_name} must be a positive integer.") + return parsed + + +def _optional_positive_int(value: object, *, field_name: str) -> int | None: + if value is None or value == "": + return None + return _positive_int(value, field_name=field_name) + + +def _bool_value(value: object) -> bool: + if isinstance(value, bool): + return value + if value is None: + return False + normalized = str(value).strip().lower() + return normalized in {"1", "true", "yes", "y", "on"} + + +def _allow_live_source_access_value(payload: Mapping[str, object]) -> bool: + resolved_values: list[tuple[str, bool]] = [] + for key_path in _LIVE_SOURCE_ACCESS_CONFIG_KEYS: + found, value = _nested_lookup(payload, key_path) + if found: + resolved_values.append((".".join(key_path), _bool_value(value))) + + if not resolved_values: + return False + + unique_values = {value for _, value in resolved_values} + if len(unique_values) > 1: + configured_keys = ", ".join(key for key, _ in resolved_values) + raise ValueError( + "Conflicting live source access settings were provided: " + f"{configured_keys}.", + ) + + return resolved_values[0][1] + + +__all__ = ( + "CONFIGURED_CYCLE_SOURCE_FAMILIES", + "ConfiguredCycleRunnerConfig", + "ConfiguredSourceYearArtifact", + "load_configured_cycle_runner_config", +) diff --git a/src/carbonfactor_parser/pipeline/configured_cycle_dependencies.py b/src/carbonfactor_parser/pipeline/configured_cycle_dependencies.py new file mode 100644 index 00000000..e7c6de79 --- /dev/null +++ b/src/carbonfactor_parser/pipeline/configured_cycle_dependencies.py @@ -0,0 +1,170 @@ +"""Configured cycle dependency composition. + +This module wires configured-cycle runtime dependencies for the production +E2E year orchestrator without owning the runner loop, startup summary, cycle +summary output, or run-history persistence behavior. +""" + +from __future__ import annotations + +from typing import Mapping + +from carbonfactor_parser.persistence.postgresql_runtime import ( + PostgreSQLRuntimeStartupResult, +) +from carbonfactor_parser.persistence.postgresql_source_family_repository import ( + PostgreSQLSourceFamilyRuntimeRepository, +) +from carbonfactor_parser.pipeline.configured_cycle_config import ( + ConfiguredCycleRunnerConfig, + ConfiguredSourceYearArtifact, +) +from carbonfactor_parser.pipeline.defra_desnz_production_e2e import ( + DEFRA_DESNZ_SOURCE_FAMILY, + DefraDesnzPhase2ValidationBoundary, + DefraDesnzProductionParserBoundary, + DefraDesnzProductionSourceAdapter, + DefraDesnzSourceYear, +) +from carbonfactor_parser.pipeline.ghg_protocol_production_e2e import ( + GHG_PROTOCOL_SOURCE_FAMILY, + GHGProtocolPhase2ValidationBoundary, + GHGProtocolProductionParserBoundary, + GHGProtocolProductionSourceAdapter, + GHGProtocolSourceYear, +) +from carbonfactor_parser.pipeline.ipcc_efdb_production_e2e import ( + IPCC_EFDB_SOURCE_FAMILY, + IpccEfdbPhase2ValidationBoundary, + IpccEfdbProductionParserBoundary, + IpccEfdbProductionSourceAdapter, + IpccEfdbSourceYear, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EValidationResult, + ProductionE2EYearOrchestratorDependencies, +) +from carbonfactor_parser.pipeline.source_artifact_transport import ( + build_configured_artifact_transport, +) + + +class ConfiguredCycleValidationBoundary: + """Route validation to the source-family-specific validation boundary.""" + + def __init__(self) -> None: + self._boundaries = { + GHG_PROTOCOL_SOURCE_FAMILY: GHGProtocolPhase2ValidationBoundary(), + DEFRA_DESNZ_SOURCE_FAMILY: DefraDesnzPhase2ValidationBoundary(), + IPCC_EFDB_SOURCE_FAMILY: IpccEfdbPhase2ValidationBoundary(), + } + + def validate(self, batch: object) -> ProductionE2EValidationResult: + rows = tuple(getattr(batch, "rows", ())) + source_family = rows[0].source_family if rows else GHG_PROTOCOL_SOURCE_FAMILY + boundary = self._boundaries.get(source_family) + if boundary is None: + boundary = GHGProtocolPhase2ValidationBoundary() + return boundary.validate(batch) + + +def build_configured_cycle_dependencies( + config: ConfiguredCycleRunnerConfig, + runtime: PostgreSQLRuntimeStartupResult, +) -> ProductionE2EYearOrchestratorDependencies: + """Build production E2E dependencies for the configured cycle runner.""" + + transport = build_configured_artifact_transport( + allow_live_source_access=config.allow_live_source_access, + ) + source_years = config.source_years or {} + return ProductionE2EYearOrchestratorDependencies( + year_state_repository=runtime.year_state_repository, + source_adapters={ + GHG_PROTOCOL_SOURCE_FAMILY: GHGProtocolProductionSourceAdapter( + target_root=config.archive_root, + source_years=_ghg_source_years( + source_years.get(GHG_PROTOCOL_SOURCE_FAMILY, {}), + ), + transport=transport, + ), + DEFRA_DESNZ_SOURCE_FAMILY: DefraDesnzProductionSourceAdapter( + target_root=config.archive_root, + source_years=_defra_source_years( + source_years.get(DEFRA_DESNZ_SOURCE_FAMILY, {}), + ), + transport=transport, + ), + IPCC_EFDB_SOURCE_FAMILY: IpccEfdbProductionSourceAdapter( + target_root=config.archive_root, + source_years=_ipcc_source_years( + source_years.get(IPCC_EFDB_SOURCE_FAMILY, {}), + ), + transport=transport, + ), + }, + parser_boundaries={ + GHG_PROTOCOL_SOURCE_FAMILY: GHGProtocolProductionParserBoundary(), + DEFRA_DESNZ_SOURCE_FAMILY: DefraDesnzProductionParserBoundary(), + IPCC_EFDB_SOURCE_FAMILY: IpccEfdbProductionParserBoundary(), + }, + validation_boundary=ConfiguredCycleValidationBoundary(), + insert_repository=PostgreSQLSourceFamilyRuntimeRepository(runtime.connection), + ) + + +def _ghg_source_years( + values: Mapping[int, ConfiguredSourceYearArtifact], +) -> Mapping[int, GHGProtocolSourceYear]: + return { + year: GHGProtocolSourceYear( + year=entry.year, + publication_url=entry.publication_url, + artifact_url=entry.artifact_url, + title=entry.title, + version_label=entry.version_label, + content_type=entry.content_type, + format_hint=entry.format_hint, + ) + for year, entry in values.items() + } + + +def _defra_source_years( + values: Mapping[int, ConfiguredSourceYearArtifact], +) -> Mapping[int, DefraDesnzSourceYear]: + return { + year: DefraDesnzSourceYear( + year=entry.year, + publication_url=entry.publication_url, + artifact_url=entry.artifact_url, + title=entry.title, + version_label=entry.version_label, + content_type=entry.content_type, + format_hint=entry.format_hint, + ) + for year, entry in values.items() + } + + +def _ipcc_source_years( + values: Mapping[int, ConfiguredSourceYearArtifact], +) -> Mapping[int, IpccEfdbSourceYear]: + return { + year: IpccEfdbSourceYear( + year=entry.year, + publication_url=entry.publication_url, + artifact_url=entry.artifact_url, + title=entry.title, + version_label=entry.version_label, + content_type=entry.content_type, + format_hint=entry.format_hint, + ) + for year, entry in values.items() + } + + +__all__ = ( + "ConfiguredCycleValidationBoundary", + "build_configured_cycle_dependencies", +) diff --git a/src/carbonfactor_parser/pipeline/configured_cycle_history.py b/src/carbonfactor_parser/pipeline/configured_cycle_history.py new file mode 100644 index 00000000..6e670cd3 --- /dev/null +++ b/src/carbonfactor_parser/pipeline/configured_cycle_history.py @@ -0,0 +1,87 @@ +"""Configured ingestion cycle run-history persistence helpers.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Callable + +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text +from carbonfactor_parser.persistence.ingestion_run_history import ( + ParserIngestionRunHistoryRepository, + ParserIngestionRunHistoryStatus, +) +from carbonfactor_parser.persistence.ingestion_run_history_mapping import ( + build_ingestion_run_history_command_from_configured_cycle, +) +from carbonfactor_parser.pipeline.configured_cycle_models import ConfiguredCycleResult + + +def persist_configured_cycle_history( + cycle: ConfiguredCycleResult, + *, + history_repository: ParserIngestionRunHistoryRepository, + started_at: datetime, + finished_at: datetime, + emit: Callable[[str], None] | None, +) -> ConfiguredCycleResult: + """Persist run-history for a configured cycle without affecting ingestion result.""" + + command = build_ingestion_run_history_command_from_configured_cycle( + cycle, + started_at=started_at, + finished_at=finished_at, + ) + try: + persist_result = history_repository.persist_ingestion_run_history(command) + except Exception as exc: # pragma: no cover - defensive boundary protection + safe_message = redact_sensitive_text(str(exc)) + if emit is not None: + emit( + "history_persistence " + f"status=failed run_id={cycle.run_id} " + "issue code=INGESTION_RUN_HISTORY_PERSISTENCE_EXCEPTION " + f"message={safe_message}" + ) + return ConfiguredCycleResult( + cycle_number=cycle.cycle_number, + run_id=cycle.run_id, + result=cycle.result, + history_persistence_status="failed", + history_persistence_issue_count=1, + ) + + issue_count = len(persist_result.issues) + if persist_result.status is ParserIngestionRunHistoryStatus.DECLARED: + if emit is not None: + emit(f"history_persistence status=declared run_id={cycle.run_id}") + else: + if emit is not None: + for issue in persist_result.issues or (): + safe_message = redact_sensitive_text(str(issue.message)) + emit( + "history_persistence " + f"status=failed run_id={cycle.run_id} " + f"issue code={issue.code} message={safe_message}" + ) + if not persist_result.issues: + emit( + "history_persistence " + f"status=failed run_id={cycle.run_id} " + "issue code=INGESTION_RUN_HISTORY_PERSISTENCE_FAILED " + "message=run history persistence failed" + ) + issue_count = 1 + return ConfiguredCycleResult( + cycle_number=cycle.cycle_number, + run_id=cycle.run_id, + result=cycle.result, + history_persistence_status=( + "declared" + if persist_result.status is ParserIngestionRunHistoryStatus.DECLARED + else "failed" + ), + history_persistence_issue_count=issue_count, + ) + + +__all__ = ("persist_configured_cycle_history",) diff --git a/src/carbonfactor_parser/pipeline/configured_cycle_models.py b/src/carbonfactor_parser/pipeline/configured_cycle_models.py new file mode 100644 index 00000000..4e74d5e8 --- /dev/null +++ b/src/carbonfactor_parser/pipeline/configured_cycle_models.py @@ -0,0 +1,36 @@ +"""Configured ingestion cycle result models.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EYearOrchestratorResult, +) + +if TYPE_CHECKING: + from carbonfactor_parser.pipeline.configured_cycle_runner import ( + ConfiguredCycleRunnerStatus, + ) + + +@dataclass(frozen=True) +class ConfiguredCycleResult: + """One completed application cycle.""" + + cycle_number: int + run_id: str + result: ProductionE2EYearOrchestratorResult + history_persistence_status: str | None = None + history_persistence_issue_count: int = 0 + + +@dataclass(frozen=True) +class ConfiguredCycleRunnerResult: + """All cycles run by one application invocation.""" + + status: ConfiguredCycleRunnerStatus + cycles: tuple[ConfiguredCycleResult, ...] + schema_created_table_names: tuple[str, ...] + schema_missing_table_names: tuple[str, ...] diff --git a/src/carbonfactor_parser/pipeline/configured_cycle_runner.py b/src/carbonfactor_parser/pipeline/configured_cycle_runner.py new file mode 100644 index 00000000..648732fe --- /dev/null +++ b/src/carbonfactor_parser/pipeline/configured_cycle_runner.py @@ -0,0 +1,172 @@ +"""Configured PostgreSQL-backed ingestion cycle runner. + +The runner is the application/runtime layer over the existing year +orchestrator. It loads explicit local configuration, starts PostgreSQL, creates +missing Phase 1 tables, and repeatedly runs the configured source families. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from enum import Enum +import time +import uuid +from typing import Callable + +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text + +from carbonfactor_parser.persistence.ingestion_run_history import ( + ParserIngestionRunHistoryRepository, +) +from carbonfactor_parser.persistence.postgresql_ingestion_run_history_repository import ( + PostgreSQLIngestionRunHistoryRepository, +) +from carbonfactor_parser.persistence.postgresql_runtime import ( + PostgreSQLRuntimeStartupResult, + start_postgresql_runtime, +) +from carbonfactor_parser.pipeline.configured_cycle_config import ( + CONFIGURED_CYCLE_SOURCE_FAMILIES, + ConfiguredCycleRunnerConfig, + ConfiguredSourceYearArtifact, + load_configured_cycle_runner_config, +) +from carbonfactor_parser.pipeline.configured_cycle_dependencies import ( + ConfiguredCycleValidationBoundary, + build_configured_cycle_dependencies, +) +from carbonfactor_parser.pipeline.configured_cycle_history import ( + persist_configured_cycle_history, +) +from carbonfactor_parser.pipeline.configured_cycle_models import ( + ConfiguredCycleResult, + ConfiguredCycleRunnerResult, +) +from carbonfactor_parser.pipeline.configured_cycle_summary import ( + emit_configured_cycle_summary, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EYearOrchestratorRequest, + ProductionE2EYearRunStatus, + run_production_e2e_year_orchestrator, +) + + +class ConfiguredCycleRunnerStatus(str, Enum): + """Top-level configured cycle runner status.""" + + COMPLETED = "completed" + COMPLETED_WITH_FAILURES = "completed_with_failures" + + +def run_configured_cycle_runner( + config: ConfiguredCycleRunnerConfig, + *, + startup: PostgreSQLRuntimeStartupResult | None = None, + sleep: Callable[[float], None] = time.sleep, + emit: Callable[[str], None] | None = print, + run_history_repository: ParserIngestionRunHistoryRepository | None = None, + run_history_repository_factory: ( + Callable[[object], ParserIngestionRunHistoryRepository] | None + ) = None, +) -> ConfiguredCycleRunnerResult: + """Start PostgreSQL runtime and execute configured ingestion cycles.""" + + runtime = startup or start_postgresql_runtime(config.postgresql_config_result) + if emit is not None: + _emit_startup_summary(config, runtime, emit) + + dependencies = build_configured_cycle_dependencies(config, runtime) + history_repository = run_history_repository + if history_repository is None: + history_repository_factory = ( + run_history_repository_factory or PostgreSQLIngestionRunHistoryRepository + ) + history_repository = history_repository_factory(runtime.connection) + cycles: list[ConfiguredCycleResult] = [] + cycle_number = 1 + while config.max_cycles is None or cycle_number <= config.max_cycles: + run_id = f"configured-cycle-{cycle_number}-{uuid.uuid4().hex}" + started_at = datetime.now(timezone.utc) + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id=run_id, + enabled_source_families=config.enabled_source_families, + initial_year=config.initial_year, + ), + dependencies, + ) + finished_at = datetime.now(timezone.utc) + cycle = ConfiguredCycleResult( + cycle_number=cycle_number, + run_id=run_id, + result=result, + ) + if emit is not None: + emit_configured_cycle_summary(cycle, emit=emit) + cycle = persist_configured_cycle_history( + cycle, + history_repository=history_repository, + started_at=started_at, + finished_at=finished_at, + emit=emit, + ) + cycles.append(cycle) + + cycle_number += 1 + if config.max_cycles is not None and cycle_number > config.max_cycles: + break + if config.cycle_interval_seconds > 0: + sleep(config.cycle_interval_seconds) + + failed = any( + cycle.result.status is not ProductionE2EYearRunStatus.COMPLETED + for cycle in cycles + ) + return ConfiguredCycleRunnerResult( + status=( + ConfiguredCycleRunnerStatus.COMPLETED_WITH_FAILURES + if failed + else ConfiguredCycleRunnerStatus.COMPLETED + ), + cycles=tuple(cycles), + schema_created_table_names=runtime.schema_bootstrap.created_table_names, + schema_missing_table_names=runtime.schema_bootstrap.missing_table_names, + ) + + +def _emit_startup_summary( + config: ConfiguredCycleRunnerConfig, + runtime: PostgreSQLRuntimeStartupResult, + emit: Callable[[str], None], +) -> None: + emit("carbonops ingestion application started") + emit(f"archive_root={config.archive_root}") + emit(f"enabled_source_families={','.join(config.enabled_source_families)}") + emit(f"initial_year={config.initial_year}") + emit(f"cycle_interval_seconds={config.cycle_interval_seconds:g}") + emit(f"max_cycles={config.max_cycles}") + emit(f"allow_live_source_access={config.allow_live_source_access}") + emit( + "postgresql_schema " + f"created={','.join(runtime.schema_bootstrap.created_table_names) or 'none'} " + f"missing={','.join(runtime.schema_bootstrap.missing_table_names) or 'none'}" + ) + + +def _redact_sensitive_text(text: str) -> str: + return redact_sensitive_text(text) + + +__all__ = ( + "CONFIGURED_CYCLE_SOURCE_FAMILIES", + "ConfiguredCycleResult", + "ConfiguredCycleRunnerConfig", + "ConfiguredCycleRunnerResult", + "ConfiguredCycleRunnerStatus", + "ConfiguredCycleValidationBoundary", + "ConfiguredSourceYearArtifact", + "emit_configured_cycle_summary", + "load_configured_cycle_runner_config", + "run_configured_cycle_runner", +) diff --git a/src/carbonfactor_parser/pipeline/configured_cycle_summary.py b/src/carbonfactor_parser/pipeline/configured_cycle_summary.py new file mode 100644 index 00000000..bb893e25 --- /dev/null +++ b/src/carbonfactor_parser/pipeline/configured_cycle_summary.py @@ -0,0 +1,75 @@ +"""Configured ingestion cycle summary output helpers.""" + +from __future__ import annotations + +from typing import Callable + +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text +from carbonfactor_parser.pipeline.configured_cycle_models import ConfiguredCycleResult + + +def emit_configured_cycle_summary( + cycle: ConfiguredCycleResult, + *, + emit: Callable[[str], None] = print, +) -> None: + """Print user-readable summary output for one cycle.""" + + summary = cycle.result.summary + emit( + "cycle=" + f"{cycle.cycle_number} run_id={cycle.run_id} status={cycle.result.status.value}" + ) + emit( + "summary " + f"completed={summary.completed_family_count} " + f"no_available_source_year={summary.no_available_source_year_count} " + f"failed={summary.failed_family_count} " + f"parsed_rows={summary.parsed_row_count} " + f"inserted={summary.inserted_count} " + f"skipped_duplicates={summary.skipped_duplicate_count}" + ) + for family in cycle.result.family_results: + insert_summary = family.insert_summary + emit( + "source " + f"family={family.source_family} " + f"target_year={family.year_state.target_year} " + f"latest_year={family.year_state.latest_year} " + f"status={family.status.value} " + f"download_status={_download_status_value(family.download_result)} " + f"parse_status={_parse_status_value(family)} " + f"parsed_rows={family.parsed_row_count} " + f"master_inserted={getattr(insert_summary, 'master_inserted', 0)} " + f"master_skipped={getattr(insert_summary, 'master_skipped', 0)} " + f"detail_inserted={getattr(insert_summary, 'detail_inserted', 0)} " + f"detail_skipped={getattr(insert_summary, 'detail_skipped', 0)}" + ) + for failure in family.failures: + safe_message = redact_sensitive_text(str(failure.message)) + emit( + "issue " + f"family={failure.source_family} stage={failure.stage} " + f"code={failure.code} message={safe_message}" + ) + + +def _download_status_value(download_result: object | None) -> str: + if download_result is None: + return "not_run" + return str(getattr(getattr(download_result, "status", None), "value", "unknown")) + + +def _parse_status_value(family: object) -> str: + if getattr(family, "parsed_row_count", 0) > 0: + return "parsed" + failures = tuple(getattr(family, "failures", ())) + if any(getattr(failure, "stage", "") == "parser" for failure in failures): + return "failed" + download_result = getattr(family, "download_result", None) + if download_result is None or _download_status_value(download_result) != "downloaded": + return "not_run" + return "no_rows" + + +__all__ = ("emit_configured_cycle_summary",) diff --git a/src/carbonfactor_parser/pipeline/defra_desnz_production_e2e.py b/src/carbonfactor_parser/pipeline/defra_desnz_production_e2e.py new file mode 100644 index 00000000..8c57fffa --- /dev/null +++ b/src/carbonfactor_parser/pipeline/defra_desnz_production_e2e.py @@ -0,0 +1,762 @@ +"""DEFRA/DESNZ production E2E ingestion adapters. + +The adapters in this module are intentionally narrow: they discover the +year-scoped GOV.UK flat-file publication, download/archive the source artifact, +parse normalized factor rows, and adapt Phase 2 data-quality diagnostics to the +production year orchestrator boundary. +""" + +from __future__ import annotations + +import csv +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from hashlib import sha256 +import json +from pathlib import Path +import re +from typing import Callable, Mapping +from urllib.parse import urlparse +from urllib.request import Request, urlopen +import xml.etree.ElementTree as ET +from zipfile import ZipFile + +from carbonfactor_parser.normalization.contracts import ( + NormalizationResult, + NormalizedRecord, +) +from carbonfactor_parser.normalization.data_quality_validation import ( + DataQualityValidationSeverity, + validate_normalized_factor_output, +) +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputBatch, + ParserNormalizedOutputRow, + ParserNormalizedOutputRowStatus, + create_parser_normalized_output_batch, + validate_parser_normalized_output_batch, +) +from carbonfactor_parser.parsers.selection_registry_contract import ( + PHASE1_PARSER_KEYS_BY_SOURCE_FAMILY, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EDownloadedArtifact, + ProductionE2EFailureDetail, + ProductionE2ESourceYearDiscoveryRequest, + ProductionE2ESourceYearDiscoveryResult, + ProductionE2ESourceYearDiscoveryStatus, + ProductionE2ESourceYearDownloadResult, + ProductionE2ESourceYearDownloadStatus, + ProductionE2EValidationResult, + ProductionE2EValidationStatus, +) + + +DEFRA_DESNZ_SOURCE_FAMILY = "defra_desnz" +DEFRA_DESNZ_SOURCE_ID = "defra_desnz" +DEFRA_DESNZ_PARSER_KEY = PHASE1_PARSER_KEYS_BY_SOURCE_FAMILY[ + DEFRA_DESNZ_SOURCE_FAMILY +] + + +@dataclass(frozen=True) +class DefraDesnzSourceYear: + """Known DEFRA/DESNZ source-year publication metadata.""" + + year: int + publication_url: str + artifact_url: str + title: str + version_label: str + content_type: str = ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + format_hint: str = "xlsx" + + +DEFAULT_DEFRA_DESNZ_SOURCE_YEARS: Mapping[int, DefraDesnzSourceYear] = { + 2024: DefraDesnzSourceYear( + year=2024, + publication_url=( + "https://www.gov.uk/government/publications/" + "greenhouse-gas-reporting-conversion-factors-2024" + ), + artifact_url="", + title="Conversion factors 2024: flat file (for automatic processing only)", + version_label="2024-v1.1", + ), + 2025: DefraDesnzSourceYear( + year=2025, + publication_url=( + "https://www.gov.uk/government/publications/" + "greenhouse-gas-reporting-conversion-factors-2025" + ), + artifact_url="", + title="Conversion factors 2025: flat file (for automatic processing only)", + version_label="2025", + ), +} +DEFRA_DESNZ_DISCOVERY_STRATEGY = "govuk_publication_flat_file_link" +DEFRA_DESNZ_DISCOVERY_AUTOMATION = ( + "configured_artifact_url_or_govuk_publication_page" +) + + +DownloadTransport = Callable[[str], bytes] + + +class DefraDesnzProductionSourceAdapter: + """Discover and download known DEFRA/DESNZ year-scoped flat files.""" + + source_family = DEFRA_DESNZ_SOURCE_FAMILY + + def __init__( + self, + *, + target_root: str | Path, + source_years: Mapping[int, DefraDesnzSourceYear] | None = None, + transport: DownloadTransport | None = None, + ) -> None: + self._target_root = Path(target_root) + self._source_years = dict( + DEFAULT_DEFRA_DESNZ_SOURCE_YEARS + if source_years is None + else source_years + ) + self._transport = transport or _https_download + + def discover_target_year( + self, + request: ProductionE2ESourceYearDiscoveryRequest, + ) -> ProductionE2ESourceYearDiscoveryResult: + """Return available metadata for exactly one target year.""" + + source_year = self._source_years.get(request.target_year) + if source_year is None: + return ProductionE2ESourceYearDiscoveryResult( + status=( + ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ), + source_family=request.source_family, + target_year=request.target_year, + reason_code="defra_desnz_target_year_not_in_availability_map", + metadata={ + "availability_strategy": DEFRA_DESNZ_DISCOVERY_STRATEGY, + "discovery_automation": DEFRA_DESNZ_DISCOVERY_AUTOMATION, + "configured_years": tuple(sorted(self._source_years)), + "user_message": ( + "DEFRA/DESNZ target year is not in the configured " + "availability map. Add source_years configuration or " + "update the reviewed default availability map." + ), + }, + ) + + try: + artifact_url = source_year.artifact_url or self._discover_flat_file_url( + source_year, + ) + discovery_failure_metadata: Mapping[str, object] = {} + except Exception as exc: # noqa: BLE001 - discovery transport varies by runtime + artifact_url = None + discovery_failure_metadata = { + "discovery_error_type": exc.__class__.__name__, + "discovery_error_message": _redacted_error_message(exc), + } + if artifact_url is None: + return ProductionE2ESourceYearDiscoveryResult( + status=( + ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ), + source_family=request.source_family, + target_year=request.target_year, + reason_code="defra_desnz_flat_file_link_not_found", + metadata={ + "availability_strategy": DEFRA_DESNZ_DISCOVERY_STRATEGY, + "discovery_automation": DEFRA_DESNZ_DISCOVERY_AUTOMATION, + "publication_url": source_year.publication_url, + "title": source_year.title, + "version_label": source_year.version_label, + "user_message": ( + "DEFRA/DESNZ publication was configured for the target " + "year, but no GOV.UK flat-file artifact link was " + "resolved for download." + ), + **discovery_failure_metadata, + }, + ) + + return ProductionE2ESourceYearDiscoveryResult( + status=ProductionE2ESourceYearDiscoveryStatus.SOURCE_YEAR_AVAILABLE, + source_family=request.source_family, + target_year=request.target_year, + artifact_reference=artifact_url, + metadata={ + "availability_strategy": DEFRA_DESNZ_DISCOVERY_STRATEGY, + "discovery_automation": DEFRA_DESNZ_DISCOVERY_AUTOMATION, + "publication_url": source_year.publication_url, + "title": source_year.title, + "version_label": source_year.version_label, + "content_type": source_year.content_type, + "format_hint": source_year.format_hint, + }, + ) + + def _discover_flat_file_url( + self, + source_year: DefraDesnzSourceYear, + ) -> str | None: + page = self._transport(source_year.publication_url).decode( + "utf-8", + errors="replace", + ) + for href, label in re.findall( + r']+href="([^"]+)"[^>]*>(.*?)', + page, + flags=re.IGNORECASE | re.DOTALL, + ): + clean_label = re.sub(r"<[^>]+>", " ", label) + if "flat" not in clean_label.lower(): + continue + if not href.startswith("https://assets.publishing.service.gov.uk/"): + continue + return href.replace("&", "&") + return None + + def download_target_year( + self, + discovery_result: ProductionE2ESourceYearDiscoveryResult, + ) -> ProductionE2ESourceYearDownloadResult: + """Download and archive the discovered source artifact locally.""" + + if discovery_result.artifact_reference is None: + return ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.FAILED, + source_family=discovery_result.source_family, + target_year=discovery_result.target_year, + issues=( + _failure( + "download", + "DEFRA_DESNZ_PRODUCTION_MISSING_ARTIFACT_REFERENCE", + "Discovery did not provide a source artifact reference.", + "discovery_result.artifact_reference", + ), + ), + ) + + try: + content = self._transport(discovery_result.artifact_reference) + except Exception as exc: # noqa: BLE001 - transport implementation varies + return ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.FAILED, + source_family=discovery_result.source_family, + target_year=discovery_result.target_year, + issues=( + _failure( + "download", + "DEFRA_DESNZ_PRODUCTION_DOWNLOAD_FAILED", + _redacted_error_message(exc), + "artifact_reference", + ), + ), + ) + + checksum = sha256(content).hexdigest() + metadata = dict(discovery_result.metadata or {}) + filename = _artifact_filename( + discovery_result.artifact_reference, + discovery_result.target_year, + str(metadata.get("format_hint") or "xlsx"), + ) + target_dir = ( + self._target_root + / DEFRA_DESNZ_SOURCE_FAMILY + / str(discovery_result.target_year) + ) + target_dir.mkdir(parents=True, exist_ok=True) + target_path = target_dir / filename + metadata_path = target_dir / f"{filename}.metadata.json" + + if target_path.exists() and target_path.read_bytes() != content: + target_path.unlink() + if not target_path.exists(): + target_path.write_bytes(content) + + metadata_payload = { + "source_family": DEFRA_DESNZ_SOURCE_FAMILY, + "source_year": discovery_result.target_year, + "artifact_reference": discovery_result.artifact_reference, + "local_path": str(target_path), + "checksum_sha256": checksum, + "size_bytes": len(content), + **metadata, + } + metadata_path.write_text( + json.dumps(metadata_payload, sort_keys=True, indent=2), + encoding="utf-8", + ) + + return ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.DOWNLOADED, + source_family=discovery_result.source_family, + target_year=discovery_result.target_year, + artifact=ProductionE2EDownloadedArtifact( + source_family=discovery_result.source_family, + source_year=discovery_result.target_year, + artifact_reference=str(target_path), + checksum_sha256=checksum, + content_type=_text_or_none(metadata.get("content_type")), + format_hint=_text_or_none(metadata.get("format_hint")), + metadata={ + **metadata_payload, + "metadata_path": str(metadata_path), + "source_reference_uri": discovery_result.artifact_reference, + }, + ), + ) + + +class DefraDesnzProductionParserBoundary: + """Parse DEFRA/DESNZ CSV or XLSX flat-file artifacts into normalized rows.""" + + def parse( + self, + artifact: ProductionE2EDownloadedArtifact, + ) -> ParserNormalizedOutputBatch: + path = _artifact_path(artifact.artifact_reference) + rows = _read_xlsx_rows(path) if path.suffix.lower() == ".xlsx" else _read_csv_rows(path) + normalized_rows = tuple(_normalized_row(artifact, row) for row in rows) + return create_parser_normalized_output_batch(normalized_rows) + + +class DefraDesnzPhase2ValidationBoundary: + """Adapt Phase 2 normalized data-quality diagnostics to E2E validation.""" + + def validate( + self, + batch: ParserNormalizedOutputBatch, + ) -> ProductionE2EValidationResult: + issues: list[ProductionE2EFailureDetail] = [] + parser_validation = validate_parser_normalized_output_batch(batch) + for issue in parser_validation.issues: + issues.append( + ProductionE2EFailureDetail( + source_family=DEFRA_DESNZ_SOURCE_FAMILY, + stage="validation", + code=issue.code, + message=issue.message, + field_name=issue.field_name, + severity=issue.severity, + ) + ) + + normalization_result = NormalizationResult( + records=tuple( + NormalizedRecord( + record_id=row.row_id, + fields=row.normalized_fields, + source_reference=row.artifact_reference, + is_artificial=False, + ) + for row in batch.rows + ), + ) + quality_result = validate_normalized_factor_output(normalization_result) + for diagnostic in quality_result.diagnostics: + issues.append( + ProductionE2EFailureDetail( + source_family=diagnostic.source_family, + stage="validation", + code=diagnostic.code, + message=diagnostic.message, + field_name=diagnostic.field_name, + severity=diagnostic.severity.value, + ) + ) + + blocking_count = sum( + issue.severity + in {DataQualityValidationSeverity.BLOCKING_ERROR.value, "error"} + for issue in issues + ) + return ProductionE2EValidationResult( + status=( + ProductionE2EValidationStatus.FAILED_VALIDATION + if blocking_count + else ProductionE2EValidationStatus.VALIDATED + ), + diagnostic_count=len(issues), + blocking_error_count=blocking_count, + warning_count=sum(issue.severity == "warning" for issue in issues), + issues=tuple(issues), + ) + + +def _https_download(uri: str) -> bytes: + parsed = urlparse(uri) + if parsed.scheme != "https": + raise ValueError("DEFRA/DESNZ production downloads require HTTPS URIs.") + request = Request(uri, headers={"User-Agent": "carbonops-parser/0.1"}) + with urlopen(request, timeout=60) as response: # noqa: S310 - HTTPS only above + return bytes(response.read()) + + +def _redacted_error_message(exc: Exception) -> str: + raw = str(exc).strip() or exc.__class__.__name__ + parsed = urlparse(raw) + if parsed.scheme in {"http", "https"} and parsed.hostname: + host = parsed.hostname + try: + port = parsed.port + except ValueError: + port = None + authority = f"{host}:{port}" if port is not None else host + return f"{parsed.scheme}://{authority}/..." + return re.sub(r"(://)[^/@\s]+@([^/\s]+)", r"\1@\2", raw) + + +def _read_csv_rows(path: Path) -> tuple[dict[str, object], ...]: + with path.open(newline="", encoding="utf-8-sig") as handle: + reader = csv.DictReader(handle) + return tuple(dict(row) for row in reader if any(row.values())) + + +def _read_xlsx_rows(path: Path) -> tuple[dict[str, object], ...]: + table = _read_first_xlsx_table(path) + if not table: + return () + header_index = _find_header_row_index(table) + headers = [_clean_header(value) for value in table[header_index]] + rows: list[dict[str, object]] = [] + for raw_row in table[header_index + 1 :]: + if not any(_text_or_none(value) for value in raw_row): + continue + row = { + headers[index]: raw_row[index] + for index in range(min(len(headers), len(raw_row))) + if headers[index] + } + if row: + rows.append(row) + return tuple(rows) + + +def _read_first_xlsx_table(path: Path) -> list[list[object]]: + with ZipFile(path) as archive: + shared_strings = _read_shared_strings(archive) + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + rels = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + rel_targets = { + rel.attrib["Id"]: rel.attrib["Target"] + for rel in rels + if "Id" in rel.attrib and "Target" in rel.attrib + } + sheet = workbook.find(".//{*}sheet") + if sheet is None: + return [] + relationship_id = sheet.attrib.get( + "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id" + ) + if relationship_id is None: + return [] + sheet_path = "xl/" + rel_targets[relationship_id].lstrip("/") + root = ET.fromstring(archive.read(sheet_path)) + + table: list[list[object]] = [] + for row in root.findall(".//{*}sheetData/{*}row"): + values: list[object] = [] + for cell in row.findall("{*}c"): + column_index = _cell_column_index(cell.attrib.get("r", "")) + while len(values) < column_index: + values.append("") + values.append(_cell_value(cell, shared_strings)) + table.append(values) + return table + + +def _read_shared_strings(archive: ZipFile) -> tuple[str, ...]: + try: + root = ET.fromstring(archive.read("xl/sharedStrings.xml")) + except KeyError: + return () + values: list[str] = [] + for item in root.findall("{*}si"): + values.append("".join(text.text or "" for text in item.findall(".//{*}t"))) + return tuple(values) + + +def _cell_value(cell: ET.Element, shared_strings: tuple[str, ...]) -> object: + cell_type = cell.attrib.get("t") + if cell_type == "inlineStr": + return "".join(text.text or "" for text in cell.findall(".//{*}t")) + value_node = cell.find("{*}v") + if value_node is None or value_node.text is None: + return "" + raw_value = value_node.text + if cell_type == "s": + return shared_strings[int(raw_value)] + if cell_type == "str": + return raw_value + decimal_value = _decimal_or_none(raw_value) + return decimal_value if decimal_value is not None else raw_value + + +def _find_header_row_index(table: list[list[object]]) -> int: + for index, row in enumerate(table): + normalized = {_header_key(value) for value in row} + has_factor_value = bool(normalized & _FACTOR_VALUE_HEADER_KEYS) or any( + value.startswith("ghgconversionfactor") for value in normalized + ) + if has_factor_value and normalized & _UNIT_HEADER_KEYS: + return index + return 0 + + +def _normalized_row( + artifact: ProductionE2EDownloadedArtifact, + row: Mapping[str, object], +) -> ParserNormalizedOutputRow: + source_row_number = _positive_int(_first_value(row, "row_number", "Row")) or 0 + factor_value = _factor_value(row, artifact.source_year) + factor_unit = _text_or_none(_first_value(row, *_UNIT_HEADERS)) or "kg CO2e" + category = _text_or_none(_first_value(row, *_CATEGORY_HEADERS)) or "uncategorized" + subcategory = _text_or_none(_first_value(row, *_SUBCATEGORY_HEADERS)) + activity = _text_or_none(_first_value(row, *_ACTIVITY_HEADERS)) + greenhouse_gas = _text_or_none(_first_value(row, *_GAS_HEADERS)) + factor_name = _factor_name(row, category, subcategory, activity, greenhouse_gas) + factor_id = _text_or_none(_first_value(row, *_FACTOR_ID_HEADERS)) + if factor_id is None: + factor_id = _stable_factor_id(artifact.source_year, factor_name, factor_unit) + + fields = { + "source_family": DEFRA_DESNZ_SOURCE_FAMILY, + "source_id": DEFRA_DESNZ_SOURCE_ID, + "source_year": artifact.source_year, + "source_version": _source_version(artifact), + "source_checksum_sha256": artifact.checksum_sha256, + "source_document_id": _source_document_id(artifact), + "source_artifact_reference": artifact.artifact_reference, + "row_number": source_row_number or None, + "factor_id": factor_id, + "factor_name": factor_name, + "factor_value": str(factor_value), + "factor_unit": factor_unit, + "unit": factor_unit, + "category": category, + "subcategory": subcategory, + "activity": activity, + "greenhouse_gas": greenhouse_gas, + "provenance": _provenance(artifact, source_row_number), + } + row_id = f"defra_desnz:{artifact.source_year}:{factor_id}" + return ParserNormalizedOutputRow( + source_family=DEFRA_DESNZ_SOURCE_FAMILY, + source_key=DEFRA_DESNZ_SOURCE_ID, + parser_key=DEFRA_DESNZ_PARSER_KEY, + artifact_reference=artifact.artifact_reference, + row_id=row_id, + normalized_fields=tuple(sorted(fields.items(), key=lambda item: item[0])), + status=ParserNormalizedOutputRowStatus.VALIDATED, + source_row_number=source_row_number or None, + artifact_identifier=_source_document_id(artifact), + reporting_year=artifact.source_year, + ) + + +def _factor_value(row: Mapping[str, object], year: int) -> Decimal: + value = _first_value(row, *_FACTOR_VALUE_HEADERS, f"GHG Conversion Factor {year}") + decimal_value = _decimal_or_none(value) + if decimal_value is None: + raise ValueError("DEFRA/DESNZ factor row is missing a numeric factor value.") + return decimal_value + + +def _factor_name( + row: Mapping[str, object], + category: str, + subcategory: str | None, + activity: str | None, + greenhouse_gas: str | None, +) -> str: + explicit = _text_or_none(_first_value(row, *_FACTOR_NAME_HEADERS)) + if explicit is not None: + return explicit + parts = tuple( + part + for part in (category, subcategory, activity, greenhouse_gas) + if part is not None + ) + return " / ".join(parts) if parts else "DEFRA/DESNZ factor" + + +def _first_value(row: Mapping[str, object], *headers: str) -> object | None: + by_key = {_header_key(key): value for key, value in row.items()} + for header in headers: + value = by_key.get(_header_key(header)) + if _text_or_none(value) is not None or isinstance(value, int | float | Decimal): + return value + return None + + +def _header_key(value: object) -> str: + return re.sub(r"[^a-z0-9]+", "", str(value).strip().lower()) + + +def _clean_header(value: object) -> str: + return str(value).strip() + + +def _stable_factor_id(year: int, factor_name: str, factor_unit: str) -> str: + digest = sha256(f"{year}\x1f{factor_name}\x1f{factor_unit}".encode()).hexdigest() + return f"DEFRA-DESNZ-{year}-{digest[:16]}" + + +def _source_version(artifact: ProductionE2EDownloadedArtifact) -> str: + version = _text_or_none((artifact.metadata or {}).get("version_label")) + return version or f"conversion-factors-{artifact.source_year}" + + +def _source_document_id(artifact: ProductionE2EDownloadedArtifact) -> str: + checksum = artifact.checksum_sha256 or "checksum-unavailable" + return f"defra_desnz:{artifact.source_year}:{checksum[:16]}" + + +def _provenance( + artifact: ProductionE2EDownloadedArtifact, + source_row_number: int, +) -> str: + if source_row_number > 0: + return f"{artifact.artifact_reference}#row-{source_row_number}" + return artifact.artifact_reference + + +def _artifact_filename(uri: str, year: int, format_hint: str) -> str: + name = Path(urlparse(uri).path).name + if name: + return name + return f"defra-desnz-conversion-factors-{year}.{format_hint}" + + +def _artifact_path(reference: str) -> Path: + parsed = urlparse(reference) + if parsed.scheme == "file": + return Path(parsed.path) + return Path(reference) + + +def _positive_int(value: object | None) -> int | None: + if value is None: + return None + try: + parsed = int(str(value).strip()) + except ValueError: + return None + return parsed if parsed > 0 else None + + +def _decimal_or_none(value: object | None) -> Decimal | None: + if value is None or value == "": + return None + if isinstance(value, bool): + return None + try: + decimal_value = Decimal(str(value).replace(",", "").strip()) + except (InvalidOperation, ValueError): + return None + return decimal_value if decimal_value.is_finite() else None + + +def _text_or_none(value: object | None) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _cell_column_index(cell_reference: str) -> int: + letters = "".join(char for char in cell_reference if char.isalpha()) + if not letters: + return 1 + index = 0 + for char in letters.upper(): + index = index * 26 + ord(char) - ord("A") + 1 + return index + + +def _failure( + stage: str, + code: str, + message: str, + field_name: str, +) -> ProductionE2EFailureDetail: + return ProductionE2EFailureDetail( + source_family=DEFRA_DESNZ_SOURCE_FAMILY, + stage=stage, + code=code, + message=message, + field_name=field_name, + ) + + +_FACTOR_VALUE_HEADERS = ( + "factor_value", + "Factor Value", + "GHG Conversion Factor", + "Conversion Factor", + "CO2e", +) +_FACTOR_VALUE_HEADER_KEYS = frozenset(_header_key(value) for value in _FACTOR_VALUE_HEADERS) +_UNIT_HEADERS = ( + "unit", + "factor_unit", + "UOM", + "Unit", + "GHG/Unit", +) +_UNIT_HEADER_KEYS = frozenset(_header_key(value) for value in _UNIT_HEADERS) +_CATEGORY_HEADERS = ( + "category", + "Category", + "Level 1", + "Scope", +) +_SUBCATEGORY_HEADERS = ( + "subcategory", + "Subcategory", + "Level 2", + "Level 3", +) +_ACTIVITY_HEADERS = ( + "activity", + "Activity", + "Level 4", + "Column Text", + "Name", +) +_GAS_HEADERS = ( + "greenhouse_gas", + "GHG", + "Gas", +) +_FACTOR_ID_HEADERS = ( + "factor_id", + "Factor ID", + "ID", +) +_FACTOR_NAME_HEADERS = ( + "factor_name", + "Factor Name", + "Name", +) + + +__all__ = ( + "DEFAULT_DEFRA_DESNZ_SOURCE_YEARS", + "DEFRA_DESNZ_PARSER_KEY", + "DEFRA_DESNZ_SOURCE_FAMILY", + "DEFRA_DESNZ_SOURCE_ID", + "DefraDesnzPhase2ValidationBoundary", + "DefraDesnzProductionParserBoundary", + "DefraDesnzProductionSourceAdapter", + "DefraDesnzSourceYear", +) diff --git a/src/carbonfactor_parser/pipeline/ghg_protocol_production_e2e.py b/src/carbonfactor_parser/pipeline/ghg_protocol_production_e2e.py new file mode 100644 index 00000000..d304a720 --- /dev/null +++ b/src/carbonfactor_parser/pipeline/ghg_protocol_production_e2e.py @@ -0,0 +1,439 @@ +"""GHG Protocol production E2E ingestion adapters. + +This module wires the existing year orchestrator, GHG content parser, +Phase 2 validation, and PostgreSQL insert repository shapes together without +adding scheduler behavior, credentials, or source correctness claims. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from hashlib import sha256 +import json +from pathlib import Path +import re +from typing import Callable, Mapping +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from carbonfactor_parser.normalization.contracts import ( + NormalizationResult, + NormalizedRecord, +) +from carbonfactor_parser.normalization.data_quality_validation import ( + DataQualityValidationSeverity, + validate_normalized_factor_output, +) +from carbonfactor_parser.parsers import create_parser_file_content_input +from carbonfactor_parser.parsers.execution_result import ParserExecutionResultStatus +from carbonfactor_parser.parsers.ghg_protocol_content_parser import ( + parse_ghg_protocol_file_content, +) +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputBatch, + ParserNormalizedOutputRow, + ParserNormalizedOutputRowStatus, + create_parser_normalized_output_batch, + validate_parser_normalized_output_batch, +) +from carbonfactor_parser.parsers.selection_registry_contract import ( + PHASE1_PARSER_KEYS_BY_SOURCE_FAMILY, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EDownloadedArtifact, + ProductionE2EFailureDetail, + ProductionE2ESourceYearDiscoveryRequest, + ProductionE2ESourceYearDiscoveryResult, + ProductionE2ESourceYearDiscoveryStatus, + ProductionE2ESourceYearDownloadResult, + ProductionE2ESourceYearDownloadStatus, + ProductionE2EValidationResult, + ProductionE2EValidationStatus, +) + + +GHG_PROTOCOL_SOURCE_FAMILY = "ghg_protocol" +GHG_PROTOCOL_SOURCE_ID = "ghg_protocol" +GHG_PROTOCOL_PARSER_KEY = PHASE1_PARSER_KEYS_BY_SOURCE_FAMILY[ + GHG_PROTOCOL_SOURCE_FAMILY +] + + +@dataclass(frozen=True) +class GHGProtocolSourceYear: + """Configured GHG Protocol source-year artifact metadata.""" + + year: int + publication_url: str + artifact_url: str + title: str + version_label: str + content_type: str = "text/csv" + format_hint: str = "csv" + + +DEFAULT_GHG_PROTOCOL_SOURCE_YEARS: Mapping[int, GHGProtocolSourceYear] = {} +GHG_PROTOCOL_DISCOVERY_STRATEGY = "configured_artifact_required" + +DownloadTransport = Callable[[str], bytes] + + +class GHGProtocolProductionSourceAdapter: + """Discover and download configured GHG Protocol year-scoped artifacts.""" + + source_family = GHG_PROTOCOL_SOURCE_FAMILY + + def __init__( + self, + *, + target_root: str | Path, + source_years: Mapping[int, GHGProtocolSourceYear] | None = None, + transport: DownloadTransport | None = None, + ) -> None: + self._target_root = Path(target_root) + self._source_years = dict( + DEFAULT_GHG_PROTOCOL_SOURCE_YEARS + if source_years is None + else source_years + ) + self._transport = transport or _https_download + + def discover_target_year( + self, + request: ProductionE2ESourceYearDiscoveryRequest, + ) -> ProductionE2ESourceYearDiscoveryResult: + """Return configured metadata for exactly one target year.""" + + source_year = self._source_years.get(request.target_year) + if source_year is None or not source_year.artifact_url.strip(): + return ProductionE2ESourceYearDiscoveryResult( + status=( + ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ), + source_family=request.source_family, + target_year=request.target_year, + reason_code="ghg_protocol_target_year_not_configured", + metadata={ + "availability_strategy": GHG_PROTOCOL_DISCOVERY_STRATEGY, + "configured_years": tuple(sorted(self._source_years)), + "requires_configured_artifact_url": True, + "user_message": ( + "GHG Protocol has no stable public year-index discovery " + "contract in this ingestion boundary; configure an " + "artifact_url for the target source year." + ), + }, + ) + + return ProductionE2ESourceYearDiscoveryResult( + status=ProductionE2ESourceYearDiscoveryStatus.SOURCE_YEAR_AVAILABLE, + source_family=request.source_family, + target_year=request.target_year, + artifact_reference=source_year.artifact_url, + metadata={ + "availability_strategy": GHG_PROTOCOL_DISCOVERY_STRATEGY, + "publication_url": source_year.publication_url, + "title": source_year.title, + "version_label": source_year.version_label, + "content_type": source_year.content_type, + "format_hint": source_year.format_hint, + "requires_configured_artifact_url": True, + }, + ) + + def download_target_year( + self, + discovery_result: ProductionE2ESourceYearDiscoveryResult, + ) -> ProductionE2ESourceYearDownloadResult: + """Download and archive the discovered GHG Protocol source artifact.""" + + if discovery_result.artifact_reference is None: + return ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.FAILED, + source_family=discovery_result.source_family, + target_year=discovery_result.target_year, + issues=( + _failure( + "download", + "GHG_PROTOCOL_PRODUCTION_MISSING_ARTIFACT_REFERENCE", + "Discovery did not provide a source artifact reference.", + "discovery_result.artifact_reference", + ), + ), + ) + + try: + content = self._transport(discovery_result.artifact_reference) + except Exception as exc: # noqa: BLE001 - transport varies by runtime + return ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.FAILED, + source_family=discovery_result.source_family, + target_year=discovery_result.target_year, + issues=( + _failure( + "download", + "GHG_PROTOCOL_PRODUCTION_DOWNLOAD_FAILED", + _redacted_error_message(exc), + "artifact_reference", + ), + ), + ) + + metadata = dict(discovery_result.metadata or {}) + checksum = sha256(content).hexdigest() + filename = _artifact_filename( + discovery_result.artifact_reference, + discovery_result.target_year, + str(metadata.get("format_hint") or "csv"), + ) + target_dir = ( + self._target_root + / GHG_PROTOCOL_SOURCE_FAMILY + / str(discovery_result.target_year) + ) + target_dir.mkdir(parents=True, exist_ok=True) + target_path = target_dir / filename + metadata_path = target_dir / f"{filename}.metadata.json" + + if target_path.exists() and target_path.read_bytes() != content: + target_path.unlink() + if not target_path.exists(): + target_path.write_bytes(content) + + metadata_payload = { + "source_family": GHG_PROTOCOL_SOURCE_FAMILY, + "source_year": discovery_result.target_year, + "artifact_reference": discovery_result.artifact_reference, + "local_path": str(target_path), + "checksum_sha256": checksum, + "size_bytes": len(content), + **metadata, + } + metadata_path.write_text( + json.dumps(metadata_payload, sort_keys=True, indent=2), + encoding="utf-8", + ) + + return ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.DOWNLOADED, + source_family=discovery_result.source_family, + target_year=discovery_result.target_year, + artifact=ProductionE2EDownloadedArtifact( + source_family=discovery_result.source_family, + source_year=discovery_result.target_year, + artifact_reference=str(target_path), + checksum_sha256=checksum, + content_type=_text_or_none(metadata.get("content_type")), + format_hint=_text_or_none(metadata.get("format_hint")), + metadata={ + **metadata_payload, + "metadata_path": str(metadata_path), + "source_reference_uri": discovery_result.artifact_reference, + }, + ), + ) + + +class GHGProtocolProductionParserBoundary: + """Parse downloaded GHG Protocol normalized CSV artifacts into rows.""" + + def parse( + self, + artifact: ProductionE2EDownloadedArtifact, + ) -> ParserNormalizedOutputBatch: + path = _artifact_path(artifact.artifact_reference) + content = path.read_bytes() + result = parse_ghg_protocol_file_content( + create_parser_file_content_input( + source_family=GHG_PROTOCOL_SOURCE_FAMILY, + source_id=GHG_PROTOCOL_SOURCE_ID, + content=content, + content_type=artifact.content_type, + format_hint=artifact.format_hint, + artifact_reference=artifact.artifact_reference, + checksum_sha256=artifact.checksum_sha256, + ) + ) + if result.status is not ParserExecutionResultStatus.SUCCESS: + codes = ", ".join(issue.code for issue in result.issues) + raise ValueError( + "GHG Protocol production parser failed" + + (f": {codes}" if codes else ".") + ) + if result.raw_record_payload is None: + return create_parser_normalized_output_batch(()) + + rows = tuple( + _normalized_row(artifact, record.raw_fields, record.row_number) + for record in result.raw_record_payload.records + ) + return create_parser_normalized_output_batch(rows) + + +class GHGProtocolPhase2ValidationBoundary: + """Adapt Phase 2 normalized data-quality diagnostics to E2E validation.""" + + def validate( + self, + batch: ParserNormalizedOutputBatch, + ) -> ProductionE2EValidationResult: + issues: list[ProductionE2EFailureDetail] = [] + parser_validation = validate_parser_normalized_output_batch(batch) + for issue in parser_validation.issues: + issues.append( + ProductionE2EFailureDetail( + source_family=GHG_PROTOCOL_SOURCE_FAMILY, + stage="validation", + code=issue.code, + message=issue.message, + field_name=issue.field_name, + severity=issue.severity, + ) + ) + + normalization_result = NormalizationResult( + records=tuple( + NormalizedRecord( + record_id=row.row_id, + fields=row.normalized_fields, + source_reference=row.artifact_reference, + is_artificial=False, + ) + for row in batch.rows + ), + ) + quality_result = validate_normalized_factor_output(normalization_result) + for diagnostic in quality_result.diagnostics: + issues.append( + ProductionE2EFailureDetail( + source_family=diagnostic.source_family, + stage="validation", + code=diagnostic.code, + message=diagnostic.message, + field_name=diagnostic.field_name, + severity=diagnostic.severity.value, + ) + ) + + blocking_count = sum( + issue.severity + in {DataQualityValidationSeverity.BLOCKING_ERROR.value, "error"} + for issue in issues + ) + return ProductionE2EValidationResult( + status=( + ProductionE2EValidationStatus.FAILED_VALIDATION + if blocking_count + else ProductionE2EValidationStatus.VALIDATED + ), + diagnostic_count=len(issues), + blocking_error_count=blocking_count, + warning_count=sum(issue.severity == "warning" for issue in issues), + issues=tuple(issues), + ) + + +def _normalized_row( + artifact: ProductionE2EDownloadedArtifact, + fields: Mapping[str, object], + row_number: int | None, +) -> ParserNormalizedOutputRow: + factor_id = str(fields["factor_id"]) + source_year = int(fields["source_year"]) + source_version = str(fields["source_version"]) + source_document_id = _source_document_id(artifact) + provenance = f"{artifact.artifact_reference}#row-{row_number or 0}" + normalized_fields = { + **dict(fields), + "source_family": GHG_PROTOCOL_SOURCE_FAMILY, + "source_id": GHG_PROTOCOL_SOURCE_ID, + "source_document_id": source_document_id, + "source_artifact_reference": artifact.artifact_reference, + "source_checksum_sha256": artifact.checksum_sha256, + "factor_value": str(fields["factor_value"]), + "factor_unit": fields["unit"], + "provenance": provenance, + "row_number": row_number, + } + row_id = f"ghg_protocol:{source_year}:{source_version}:{factor_id}" + return ParserNormalizedOutputRow( + source_family=GHG_PROTOCOL_SOURCE_FAMILY, + source_key=GHG_PROTOCOL_SOURCE_ID, + parser_key=GHG_PROTOCOL_PARSER_KEY, + artifact_reference=artifact.artifact_reference, + row_id=row_id, + normalized_fields=tuple(sorted(normalized_fields.items())), + status=ParserNormalizedOutputRowStatus.VALIDATED, + source_row_number=row_number, + artifact_identifier=source_document_id, + reporting_year=source_year, + ) + + +def _https_download(uri: str) -> bytes: + parsed = urlparse(uri) + if parsed.scheme != "https": + raise ValueError("GHG Protocol production downloads require HTTPS URIs.") + request = Request(uri, headers={"User-Agent": "carbonops-parser/0.1"}) + with urlopen(request, timeout=60) as response: # noqa: S310 - HTTPS only above + return bytes(response.read()) + + +def _redacted_error_message(exc: Exception) -> str: + raw = str(exc).strip() or exc.__class__.__name__ + parsed = urlparse(raw) + if parsed.scheme in {"http", "https"} and parsed.hostname: + host = parsed.hostname + try: + port = parsed.port + except ValueError: + port = None + authority = f"{host}:{port}" if port is not None else host + return f"{parsed.scheme}://{authority}/..." + redacted = re.sub(r"(://)[^/@\s]+@([^/\s]+)", r"\1***@\2", raw) + return re.sub( + r"(?i)(password|passwd|pwd|token|secret|key)=([^&\s]+)", + r"\1=***", + redacted, + ) + + +def _artifact_filename(uri: str, year: int, format_hint: str) -> str: + name = Path(urlparse(uri).path).name + if name: + return name + return f"ghg-protocol-factors-{year}.{format_hint}" + + +def _artifact_path(reference: str) -> Path: + parsed = urlparse(reference) + if parsed.scheme == "file": + return Path(parsed.path) + return Path(reference) + + +def _source_document_id(artifact: ProductionE2EDownloadedArtifact) -> str: + checksum = artifact.checksum_sha256 or "checksum-unavailable" + return f"ghg_protocol:{artifact.source_year}:{checksum[:16]}" + + +def _text_or_none(value: object | None) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _failure( + stage: str, + code: str, + message: str, + field_name: str | None = None, +) -> ProductionE2EFailureDetail: + return ProductionE2EFailureDetail( + source_family=GHG_PROTOCOL_SOURCE_FAMILY, + stage=stage, + code=code, + message=message, + field_name=field_name, + ) diff --git a/src/carbonfactor_parser/pipeline/ipcc_efdb_production_e2e.py b/src/carbonfactor_parser/pipeline/ipcc_efdb_production_e2e.py new file mode 100644 index 00000000..c691f8f0 --- /dev/null +++ b/src/carbonfactor_parser/pipeline/ipcc_efdb_production_e2e.py @@ -0,0 +1,451 @@ +"""IPCC EFDB production E2E ingestion adapters. + +This module wires configured year-scoped IPCC EFDB source artifacts into the +shared production year orchestrator. It downloads only explicitly configured +artifacts, parses the existing normalized EFDB CSV extraction format, validates +normalized rows, and leaves persistence to the injected PostgreSQL repository. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from hashlib import sha256 +import json +from pathlib import Path +import re +from typing import Callable, Mapping +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from carbonfactor_parser.normalization.contracts import ( + NormalizationResult, + NormalizedRecord, +) +from carbonfactor_parser.normalization.data_quality_validation import ( + DataQualityValidationSeverity, + validate_normalized_factor_output, +) +from carbonfactor_parser.parsers import create_parser_file_content_input +from carbonfactor_parser.parsers.execution_result import ParserExecutionResultStatus +from carbonfactor_parser.parsers.ipcc_efdb_content_parser import ( + parse_ipcc_efdb_file_content, +) +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputBatch, + ParserNormalizedOutputRow, + ParserNormalizedOutputRowStatus, + create_parser_normalized_output_batch, + validate_parser_normalized_output_batch, +) +from carbonfactor_parser.parsers.selection_registry_contract import ( + PHASE1_PARSER_KEYS_BY_SOURCE_FAMILY, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EDownloadedArtifact, + ProductionE2EFailureDetail, + ProductionE2ESourceYearDiscoveryRequest, + ProductionE2ESourceYearDiscoveryResult, + ProductionE2ESourceYearDiscoveryStatus, + ProductionE2ESourceYearDownloadResult, + ProductionE2ESourceYearDownloadStatus, + ProductionE2EValidationResult, + ProductionE2EValidationStatus, +) + + +IPCC_EFDB_SOURCE_FAMILY = "ipcc_efdb" +IPCC_EFDB_SOURCE_ID = "ipcc_efdb" +IPCC_EFDB_PARSER_KEY = PHASE1_PARSER_KEYS_BY_SOURCE_FAMILY[ + IPCC_EFDB_SOURCE_FAMILY +] + + +@dataclass(frozen=True) +class IpccEfdbSourceYear: + """Configured IPCC EFDB source-year artifact metadata.""" + + year: int + publication_url: str + artifact_url: str + title: str + version_label: str + content_type: str = "text/csv" + format_hint: str = "csv" + + +DEFAULT_IPCC_EFDB_SOURCE_YEARS: Mapping[int, IpccEfdbSourceYear] = {} +IPCC_EFDB_DISCOVERY_STRATEGY = "configured_artifact_required" + +DownloadTransport = Callable[[str], bytes] + + +class IpccEfdbProductionSourceAdapter: + """Discover and download configured IPCC EFDB year-scoped artifacts.""" + + source_family = IPCC_EFDB_SOURCE_FAMILY + + def __init__( + self, + *, + target_root: str | Path, + source_years: Mapping[int, IpccEfdbSourceYear] | None = None, + transport: DownloadTransport | None = None, + ) -> None: + self._target_root = Path(target_root) + self._source_years = dict( + DEFAULT_IPCC_EFDB_SOURCE_YEARS + if source_years is None + else source_years + ) + self._transport = transport or _https_download + + def discover_target_year( + self, + request: ProductionE2ESourceYearDiscoveryRequest, + ) -> ProductionE2ESourceYearDiscoveryResult: + """Return configured metadata for exactly one target year.""" + + source_year = self._source_years.get(request.target_year) + if source_year is None or not source_year.artifact_url.strip(): + return ProductionE2ESourceYearDiscoveryResult( + status=( + ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ), + source_family=request.source_family, + target_year=request.target_year, + reason_code="ipcc_efdb_target_year_not_configured", + metadata={ + "availability_strategy": IPCC_EFDB_DISCOVERY_STRATEGY, + "configured_years": tuple(sorted(self._source_years)), + "requires_configured_artifact_url": True, + "user_message": ( + "IPCC EFDB has no stable public year-index artifact " + "discovery contract in this ingestion boundary; " + "configure an artifact_url for the target source year." + ), + }, + ) + + return ProductionE2ESourceYearDiscoveryResult( + status=ProductionE2ESourceYearDiscoveryStatus.SOURCE_YEAR_AVAILABLE, + source_family=request.source_family, + target_year=request.target_year, + artifact_reference=source_year.artifact_url, + metadata={ + "availability_strategy": IPCC_EFDB_DISCOVERY_STRATEGY, + "publication_url": source_year.publication_url, + "title": source_year.title, + "version_label": source_year.version_label, + "content_type": source_year.content_type, + "format_hint": source_year.format_hint, + "requires_configured_artifact_url": True, + }, + ) + + def download_target_year( + self, + discovery_result: ProductionE2ESourceYearDiscoveryResult, + ) -> ProductionE2ESourceYearDownloadResult: + """Download and archive the discovered IPCC EFDB source artifact.""" + + if discovery_result.artifact_reference is None: + return ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.FAILED, + source_family=discovery_result.source_family, + target_year=discovery_result.target_year, + issues=( + _failure( + "download", + "IPCC_EFDB_PRODUCTION_MISSING_ARTIFACT_REFERENCE", + "Discovery did not provide a source artifact reference.", + "discovery_result.artifact_reference", + ), + ), + ) + + try: + content = self._transport(discovery_result.artifact_reference) + except Exception as exc: # noqa: BLE001 - transport varies by runtime + return ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.FAILED, + source_family=discovery_result.source_family, + target_year=discovery_result.target_year, + issues=( + _failure( + "download", + "IPCC_EFDB_PRODUCTION_DOWNLOAD_FAILED", + _redacted_error_message(exc), + "artifact_reference", + ), + ), + ) + + metadata = dict(discovery_result.metadata or {}) + checksum = sha256(content).hexdigest() + filename = _artifact_filename( + discovery_result.artifact_reference, + discovery_result.target_year, + str(metadata.get("format_hint") or "csv"), + ) + target_dir = ( + self._target_root + / IPCC_EFDB_SOURCE_FAMILY + / str(discovery_result.target_year) + ) + target_dir.mkdir(parents=True, exist_ok=True) + target_path = target_dir / filename + metadata_path = target_dir / f"{filename}.metadata.json" + + if target_path.exists() and target_path.read_bytes() != content: + target_path.unlink() + if not target_path.exists(): + target_path.write_bytes(content) + + metadata_payload = { + "source_family": IPCC_EFDB_SOURCE_FAMILY, + "source_year": discovery_result.target_year, + "artifact_reference": discovery_result.artifact_reference, + "local_path": str(target_path), + "checksum_sha256": checksum, + "size_bytes": len(content), + **metadata, + } + metadata_path.write_text( + json.dumps(metadata_payload, sort_keys=True, indent=2), + encoding="utf-8", + ) + + return ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.DOWNLOADED, + source_family=discovery_result.source_family, + target_year=discovery_result.target_year, + artifact=ProductionE2EDownloadedArtifact( + source_family=discovery_result.source_family, + source_year=discovery_result.target_year, + artifact_reference=str(target_path), + checksum_sha256=checksum, + content_type=_text_or_none(metadata.get("content_type")), + format_hint=_text_or_none(metadata.get("format_hint")), + metadata={ + **metadata_payload, + "metadata_path": str(metadata_path), + "source_reference_uri": discovery_result.artifact_reference, + }, + ), + ) + + +class IpccEfdbProductionParserBoundary: + """Parse downloaded IPCC EFDB normalized CSV artifacts into rows.""" + + def parse( + self, + artifact: ProductionE2EDownloadedArtifact, + ) -> ParserNormalizedOutputBatch: + path = _artifact_path(artifact.artifact_reference) + content = path.read_bytes() + result = parse_ipcc_efdb_file_content( + create_parser_file_content_input( + source_family=IPCC_EFDB_SOURCE_FAMILY, + source_id=IPCC_EFDB_SOURCE_ID, + content=content, + content_type=artifact.content_type, + format_hint=artifact.format_hint, + artifact_reference=artifact.artifact_reference, + checksum_sha256=artifact.checksum_sha256, + ) + ) + if result.status is not ParserExecutionResultStatus.SUCCESS: + codes = ", ".join(issue.code for issue in result.issues) + raise ValueError( + "IPCC EFDB production parser failed" + + (f": {codes}" if codes else ".") + ) + if result.raw_record_payload is None: + return create_parser_normalized_output_batch(()) + + rows = tuple( + _normalized_row(artifact, record.raw_fields, record.row_number) + for record in result.raw_record_payload.records + ) + return create_parser_normalized_output_batch(rows) + + +class IpccEfdbPhase2ValidationBoundary: + """Adapt Phase 2 normalized data-quality diagnostics to E2E validation.""" + + def validate( + self, + batch: ParserNormalizedOutputBatch, + ) -> ProductionE2EValidationResult: + issues: list[ProductionE2EFailureDetail] = [] + parser_validation = validate_parser_normalized_output_batch(batch) + for issue in parser_validation.issues: + issues.append( + ProductionE2EFailureDetail( + source_family=IPCC_EFDB_SOURCE_FAMILY, + stage="validation", + code=issue.code, + message=issue.message, + field_name=issue.field_name, + severity=issue.severity, + ) + ) + + normalization_result = NormalizationResult( + records=tuple( + NormalizedRecord( + record_id=row.row_id, + fields=row.normalized_fields, + source_reference=row.artifact_reference, + is_artificial=False, + ) + for row in batch.rows + ), + ) + quality_result = validate_normalized_factor_output(normalization_result) + for diagnostic in quality_result.diagnostics: + issues.append( + ProductionE2EFailureDetail( + source_family=diagnostic.source_family, + stage="validation", + code=diagnostic.code, + message=diagnostic.message, + field_name=diagnostic.field_name, + severity=diagnostic.severity.value, + ) + ) + + blocking_count = sum( + issue.severity + in {DataQualityValidationSeverity.BLOCKING_ERROR.value, "error"} + for issue in issues + ) + return ProductionE2EValidationResult( + status=( + ProductionE2EValidationStatus.FAILED_VALIDATION + if blocking_count + else ProductionE2EValidationStatus.VALIDATED + ), + diagnostic_count=len(issues), + blocking_error_count=blocking_count, + warning_count=sum(issue.severity == "warning" for issue in issues), + issues=tuple(issues), + ) + + +def _normalized_row( + artifact: ProductionE2EDownloadedArtifact, + fields: Mapping[str, object], + row_number: int | None, +) -> ParserNormalizedOutputRow: + factor_id = str(fields["factor_id"]) + source_year = int(fields["source_year"]) + source_version = str(fields["source_version"]) + source_document_id = _source_document_id(artifact) + normalized_fields = { + **dict(fields), + "source_family": IPCC_EFDB_SOURCE_FAMILY, + "source_id": IPCC_EFDB_SOURCE_ID, + "source_document_id": source_document_id, + "source_artifact_reference": artifact.artifact_reference, + "source_checksum_sha256": artifact.checksum_sha256, + "factor_value": str(fields["factor_value"]), + "factor_unit": fields["unit"], + "greenhouse_gas": fields["gas"], + "row_number": row_number, + } + row_id = f"ipcc_efdb:{source_year}:{source_version}:{factor_id}" + return ParserNormalizedOutputRow( + source_family=IPCC_EFDB_SOURCE_FAMILY, + source_key=IPCC_EFDB_SOURCE_ID, + parser_key=IPCC_EFDB_PARSER_KEY, + artifact_reference=artifact.artifact_reference, + row_id=row_id, + normalized_fields=tuple(sorted(normalized_fields.items())), + status=ParserNormalizedOutputRowStatus.VALIDATED, + source_row_number=row_number, + artifact_identifier=source_document_id, + reporting_year=source_year, + ) + + +def _https_download(uri: str) -> bytes: + parsed = urlparse(uri) + if parsed.scheme != "https": + raise ValueError("IPCC EFDB production downloads require HTTPS URIs.") + request = Request(uri, headers={"User-Agent": "carbonops-parser/0.1"}) + with urlopen(request, timeout=60) as response: # noqa: S310 - HTTPS only above + return bytes(response.read()) + + +def _redacted_error_message(exc: Exception) -> str: + raw = str(exc).strip() or exc.__class__.__name__ + parsed = urlparse(raw) + if parsed.scheme in {"http", "https"} and parsed.hostname: + host = parsed.hostname + try: + port = parsed.port + except ValueError: + port = None + authority = f"{host}:{port}" if port is not None else host + return f"{parsed.scheme}://{authority}/..." + redacted = re.sub(r"(://)[^/@\s]+@([^/\s]+)", r"\1***@\2", raw) + return re.sub( + r"(?i)(password|passwd|pwd|token|secret|key)=([^&\s]+)", + r"\1=***", + redacted, + ) + + +def _artifact_filename(uri: str, year: int, format_hint: str) -> str: + name = Path(urlparse(uri).path).name + if name: + return name + return f"ipcc-efdb-factors-{year}.{format_hint}" + + +def _artifact_path(reference: str) -> Path: + parsed = urlparse(reference) + if parsed.scheme == "file": + return Path(parsed.path) + return Path(reference) + + +def _source_document_id(artifact: ProductionE2EDownloadedArtifact) -> str: + checksum = artifact.checksum_sha256 or "checksum-unavailable" + return f"ipcc_efdb:{artifact.source_year}:{checksum[:16]}" + + +def _text_or_none(value: object | None) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _failure( + stage: str, + code: str, + message: str, + field_name: str | None = None, +) -> ProductionE2EFailureDetail: + return ProductionE2EFailureDetail( + source_family=IPCC_EFDB_SOURCE_FAMILY, + stage=stage, + code=code, + message=message, + field_name=field_name, + ) + + +__all__ = ( + "DEFAULT_IPCC_EFDB_SOURCE_YEARS", + "IPCC_EFDB_PARSER_KEY", + "IPCC_EFDB_SOURCE_FAMILY", + "IPCC_EFDB_SOURCE_ID", + "IpccEfdbPhase2ValidationBoundary", + "IpccEfdbProductionParserBoundary", + "IpccEfdbProductionSourceAdapter", + "IpccEfdbSourceYear", +) diff --git a/src/carbonfactor_parser/pipeline/production_e2e_year_orchestrator.py b/src/carbonfactor_parser/pipeline/production_e2e_year_orchestrator.py new file mode 100644 index 00000000..30ffb0ef --- /dev/null +++ b/src/carbonfactor_parser/pipeline/production_e2e_year_orchestrator.py @@ -0,0 +1,744 @@ +"""Production E2E year-based orchestration boundary. + +This module coordinates injected runtime boundaries only. It does not implement +live source integrations, credentials, scheduling, or source-specific parsers. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from enum import Enum +from typing import Mapping, Protocol, Sequence, runtime_checkable + +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text + +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputBatch, +) +from carbonfactor_parser.persistence.postgresql_runtime_config import ( + POSTGRESQL_RUNTIME_DEFAULT_INITIAL_YEAR, +) + + +PRODUCTION_E2E_SOURCE_FAMILIES = ( + "ghg_protocol", + "defra_desnz", + "ipcc_efdb", +) + +_SOURCE_FAMILY_ALIASES: Mapping[str, str] = { + "ghg": "ghg_protocol", + "ghg_protocol": "ghg_protocol", + "defra": "defra_desnz", + "desnz": "defra_desnz", + "defra_desnz": "defra_desnz", + "ipcc": "ipcc_efdb", + "ipcc_efdb": "ipcc_efdb", +} + +_YEAR_STATE_KEYS: Mapping[str, str] = { + "ghg_protocol": "ghg", + "defra_desnz": "defra", + "ipcc_efdb": "ipcc", +} + + +class ProductionE2EYearRunStatus(str, Enum): + """Top-level production E2E year orchestrator statuses.""" + + COMPLETED = "completed" + COMPLETED_WITH_FAILURES = "completed_with_failures" + FAILED = "failed" + + +class ProductionE2EYearFamilyStatus(str, Enum): + """Per-source-family production E2E year statuses.""" + + COMPLETED = "completed" + NO_AVAILABLE_SOURCE_YEAR = "no_available_source_year" + FAILED = "failed" + + +class ProductionE2EYearSelectionStatus(str, Enum): + """Year-state selection statuses.""" + + INITIAL_YEAR_SELECTED = "initial_year_selected" + NEXT_YEAR_SELECTED = "next_year_selected" + + +class ProductionE2ESourceYearDiscoveryStatus(str, Enum): + """Target-year source discovery statuses.""" + + SOURCE_YEAR_AVAILABLE = "source_year_available" + NO_AVAILABLE_SOURCE_YEAR = "no_available_source_year" + + +class ProductionE2ESourceYearDownloadStatus(str, Enum): + """Target-year download statuses.""" + + DOWNLOADED = "downloaded" + FAILED = "failed" + + +class ProductionE2EValidationStatus(str, Enum): + """Validation statuses for parsed normalized output.""" + + VALIDATED = "validated" + FAILED_VALIDATION = "failed_validation" + + +@dataclass(frozen=True) +class ProductionE2EFailureDetail: + """Structured failure detail emitted by the orchestrator.""" + + source_family: str | None + stage: str + code: str + message: str + field_name: str | None = None + severity: str = "error" + + +@dataclass(frozen=True) +class ProductionE2EYearState: + """Selected target year for one source family.""" + + source_family: str + year_state_key: str + latest_year: int | None + target_year: int + initial_year: int + selection_status: ProductionE2EYearSelectionStatus + + +@dataclass(frozen=True) +class ProductionE2ESourceYearDiscoveryRequest: + """Request sent to a source-family adapter for one target year.""" + + source_family: str + target_year: int + run_id: str + correlation_id: str | None = None + + +@dataclass(frozen=True) +class ProductionE2ESourceYearDiscoveryResult: + """Source-family target-year discovery result.""" + + status: ProductionE2ESourceYearDiscoveryStatus + source_family: str + target_year: int + artifact_reference: str | None = None + reason_code: str | None = None + metadata: Mapping[str, object] | None = None + + +@dataclass(frozen=True) +class ProductionE2EDownloadedArtifact: + """Downloaded or local source artifact returned by a source adapter.""" + + source_family: str + source_year: int + artifact_reference: str + checksum_sha256: str | None = None + content_type: str | None = None + format_hint: str | None = None + metadata: Mapping[str, object] | None = None + + +@dataclass(frozen=True) +class ProductionE2ESourceYearDownloadResult: + """Source-family target-year download result.""" + + status: ProductionE2ESourceYearDownloadStatus + source_family: str + target_year: int + artifact: ProductionE2EDownloadedArtifact | None = None + issues: tuple[ProductionE2EFailureDetail, ...] = () + + +@dataclass(frozen=True) +class ProductionE2EValidationResult: + """Validation result for parsed normalized output before insert.""" + + status: ProductionE2EValidationStatus + diagnostic_count: int = 0 + blocking_error_count: int = 0 + warning_count: int = 0 + issues: tuple[ProductionE2EFailureDetail, ...] = () + + @property + def is_valid(self) -> bool: + """Return whether the validation result permits persistence.""" + + return ( + self.status is ProductionE2EValidationStatus.VALIDATED + and self.blocking_error_count == 0 + ) + + +@dataclass(frozen=True) +class ProductionE2EInsertSummary: + """Repository insert summary consumed by the orchestrator.""" + + status: str + attempted: int + inserted: int + skipped_duplicate: int = 0 + failed: int = 0 + validation_error_count: int = 0 + master_inserted: int = 0 + master_skipped: int = 0 + detail_inserted: int = 0 + detail_skipped: int = 0 + + @property + def is_success(self) -> bool: + """Return whether the insert summary has no failed rows.""" + + return self.failed == 0 and not self.status.startswith("failed") + + +@dataclass(frozen=True) +class ProductionE2EYearFamilyResult: + """Structured run summary for one source family.""" + + source_family: str + status: ProductionE2EYearFamilyStatus + year_state: ProductionE2EYearState + discovery_result: ProductionE2ESourceYearDiscoveryResult | None = None + download_result: ProductionE2ESourceYearDownloadResult | None = None + parsed_row_count: int = 0 + validation_result: ProductionE2EValidationResult | None = None + insert_summary: ProductionE2EInsertSummary | None = None + recorded_ingested_year: int | None = None + failures: tuple[ProductionE2EFailureDetail, ...] = () + + +@dataclass(frozen=True) +class ProductionE2EYearRunSummary: + """Aggregated production E2E year run summary.""" + + requested_family_count: int + completed_family_count: int + no_available_source_year_count: int + failed_family_count: int + parsed_row_count: int + attempted_insert_count: int + inserted_count: int + skipped_duplicate_count: int + failed_insert_count: int + failure_count: int + + +@dataclass(frozen=True) +class ProductionE2EYearOrchestratorRequest: + """Request for a single production E2E year-based run.""" + + run_id: str + enabled_source_families: tuple[str, ...] = PRODUCTION_E2E_SOURCE_FAMILIES + initial_year: int = POSTGRESQL_RUNTIME_DEFAULT_INITIAL_YEAR + correlation_id: str | None = None + + +@dataclass(frozen=True) +class ProductionE2EYearOrchestratorResult: + """Top-level production E2E year orchestration result.""" + + status: ProductionE2EYearRunStatus + request: ProductionE2EYearOrchestratorRequest + selected_source_families: tuple[str, ...] + family_results: tuple[ProductionE2EYearFamilyResult, ...] + summary: ProductionE2EYearRunSummary + failures: tuple[ProductionE2EFailureDetail, ...] = () + + +@runtime_checkable +class ProductionE2EYearStateRepository(Protocol): + """Repository boundary for PostgreSQL source-family year state.""" + + def latest_ingested_year(self, source_family: str) -> int | None: + """Return the latest ingested year for a source-family year-state key.""" + + def record_ingested_year(self, source_family: str, ingested_year: int) -> None: + """Record a successfully ingested source-family year.""" + + +@runtime_checkable +class ProductionE2ESourceFamilyAdapter(Protocol): + """Source-family adapter boundary for target-year discovery and download.""" + + @property + def source_family(self) -> str: + """Return the canonical source family handled by this adapter.""" + + def discover_target_year( + self, + request: ProductionE2ESourceYearDiscoveryRequest, + ) -> ProductionE2ESourceYearDiscoveryResult: + """Discover availability for exactly one target year.""" + + def download_target_year( + self, + discovery_result: ProductionE2ESourceYearDiscoveryResult, + ) -> ProductionE2ESourceYearDownloadResult: + """Download or locate the artifact for a discovered target year.""" + + +@runtime_checkable +class ProductionE2EParserBoundary(Protocol): + """Parser boundary for a downloaded/local source artifact.""" + + def parse( + self, + artifact: ProductionE2EDownloadedArtifact, + ) -> ParserNormalizedOutputBatch: + """Parse the downloaded/local artifact into normalized output rows.""" + + +@runtime_checkable +class ProductionE2EValidationBoundary(Protocol): + """Validation boundary for parsed normalized output.""" + + def validate( + self, + batch: ParserNormalizedOutputBatch, + ) -> ProductionE2EValidationResult: + """Validate normalized output before persistence.""" + + +@runtime_checkable +class ProductionE2EInsertRepository(Protocol): + """PostgreSQL insert repository boundary for normalized output.""" + + def insert_normalized_factor_records( + self, + batch: ParserNormalizedOutputBatch, + ) -> object: + """Insert normalized output into PostgreSQL.""" + + +@dataclass(frozen=True) +class ProductionE2EYearOrchestratorDependencies: + """Injected dependencies for the production E2E year orchestrator.""" + + year_state_repository: ProductionE2EYearStateRepository + source_adapters: Mapping[str, ProductionE2ESourceFamilyAdapter] + parser_boundaries: Mapping[str, ProductionE2EParserBoundary] + validation_boundary: ProductionE2EValidationBoundary + insert_repository: ProductionE2EInsertRepository + + +def run_production_e2e_year_orchestrator( + request: ProductionE2EYearOrchestratorRequest, + dependencies: ProductionE2EYearOrchestratorDependencies, +) -> ProductionE2EYearOrchestratorResult: + """Run the production E2E year-based boundary with injected adapters.""" + + if request.initial_year < 1: + raise ValueError("initial_year must be positive.") + + selected_families, selection_failures = _normalize_source_families( + request.enabled_source_families, + ) + family_results = [ + _failed_family_without_runtime( + source_family=source_family, + request=request, + failure=failure, + latest_year=None, + ) + for source_family, failure in selection_failures + ] + + for source_family in selected_families: + family_results.append( + _run_source_family( + source_family=source_family, + request=request, + dependencies=dependencies, + ), + ) + + summary = _summarize(request.enabled_source_families, family_results) + failures = tuple( + failure + for family_result in family_results + for failure in family_result.failures + ) + if summary.failed_family_count: + status = ( + ProductionE2EYearRunStatus.FAILED + if summary.completed_family_count == 0 + and summary.no_available_source_year_count == 0 + else ProductionE2EYearRunStatus.COMPLETED_WITH_FAILURES + ) + else: + status = ProductionE2EYearRunStatus.COMPLETED + + return ProductionE2EYearOrchestratorResult( + status=status, + request=request, + selected_source_families=selected_families, + family_results=tuple(family_results), + summary=summary, + failures=failures, + ) + + +def _run_source_family( + *, + source_family: str, + request: ProductionE2EYearOrchestratorRequest, + dependencies: ProductionE2EYearOrchestratorDependencies, +) -> ProductionE2EYearFamilyResult: + latest_year = dependencies.year_state_repository.latest_ingested_year( + _YEAR_STATE_KEYS[source_family], + ) + year_state = _select_year_state(source_family, latest_year, request.initial_year) + + adapter = dependencies.source_adapters.get(source_family) + if adapter is None: + return _failed_family( + year_state, + _failure( + source_family, + "source_adapter", + "PRODUCTION_E2E_MISSING_SOURCE_ADAPTER", + "No source-family adapter is configured.", + "dependencies.source_adapters", + ), + ) + + parser = dependencies.parser_boundaries.get(source_family) + if parser is None: + return _failed_family( + year_state, + _failure( + source_family, + "parser", + "PRODUCTION_E2E_MISSING_PARSER_BOUNDARY", + "No parser boundary is configured.", + "dependencies.parser_boundaries", + ), + ) + + discovery_result = adapter.discover_target_year( + ProductionE2ESourceYearDiscoveryRequest( + source_family=source_family, + target_year=year_state.target_year, + run_id=request.run_id, + correlation_id=request.correlation_id, + ), + ) + if ( + discovery_result.status + is ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ): + return ProductionE2EYearFamilyResult( + source_family=source_family, + status=ProductionE2EYearFamilyStatus.NO_AVAILABLE_SOURCE_YEAR, + year_state=year_state, + discovery_result=discovery_result, + ) + if ( + discovery_result.status + is not ProductionE2ESourceYearDiscoveryStatus.SOURCE_YEAR_AVAILABLE + ): + return _failed_family( + year_state, + _failure( + source_family, + "discovery", + "PRODUCTION_E2E_SOURCE_DISCOVERY_FAILED", + "Source-family discovery did not return an available target year.", + "discovery_result.status", + ), + discovery_result=discovery_result, + ) + + download_result = adapter.download_target_year(discovery_result) + if ( + download_result.status is not ProductionE2ESourceYearDownloadStatus.DOWNLOADED + or download_result.artifact is None + ): + failures = download_result.issues or ( + _failure( + source_family, + "download", + "PRODUCTION_E2E_SOURCE_DOWNLOAD_FAILED", + "Source-family download did not return an artifact.", + "download_result.artifact", + ), + ) + return _failed_family( + year_state, + *failures, + discovery_result=discovery_result, + download_result=download_result, + ) + + try: + batch = parser.parse(download_result.artifact) + except Exception as exc: # noqa: BLE001 - parser boundaries vary by source + return _failed_family( + year_state, + _failure( + source_family, + "parser", + "PRODUCTION_E2E_PARSER_FAILED", + _redact_sensitive_text(str(exc) or exc.__class__.__name__), + "parser", + ), + discovery_result=discovery_result, + download_result=download_result, + ) + validation_result = dependencies.validation_boundary.validate(batch) + if not validation_result.is_valid: + failures = validation_result.issues or ( + _failure( + source_family, + "validation", + "PRODUCTION_E2E_VALIDATION_FAILED", + "Parsed normalized output failed validation.", + "validation_result.status", + ), + ) + return _failed_family( + year_state, + *failures, + discovery_result=discovery_result, + download_result=download_result, + parsed_row_count=batch.row_count, + validation_result=validation_result, + ) + + insert_batch = _batch_with_run_id(batch, request.run_id) + insert_summary = _coerce_insert_summary( + dependencies.insert_repository.insert_normalized_factor_records(insert_batch), + ) + if not insert_summary.is_success: + return _failed_family( + year_state, + _failure( + source_family, + "insert", + "PRODUCTION_E2E_POSTGRESQL_INSERT_FAILED", + "PostgreSQL insert repository returned a failed summary.", + "insert_summary.status", + ), + discovery_result=discovery_result, + download_result=download_result, + parsed_row_count=batch.row_count, + validation_result=validation_result, + insert_summary=insert_summary, + ) + + dependencies.year_state_repository.record_ingested_year( + _YEAR_STATE_KEYS[source_family], + year_state.target_year, + ) + return ProductionE2EYearFamilyResult( + source_family=source_family, + status=ProductionE2EYearFamilyStatus.COMPLETED, + year_state=year_state, + discovery_result=discovery_result, + download_result=download_result, + parsed_row_count=batch.row_count, + validation_result=validation_result, + insert_summary=insert_summary, + recorded_ingested_year=year_state.target_year, + ) + + +def _normalize_source_families( + source_families: Sequence[str], +) -> tuple[tuple[str, ...], tuple[tuple[str, ProductionE2EFailureDetail], ...]]: + selected: list[str] = [] + failures: list[tuple[str, ProductionE2EFailureDetail]] = [] + for source_family in source_families: + normalized = _SOURCE_FAMILY_ALIASES.get(source_family) + if normalized is None: + failures.append( + ( + source_family, + _failure( + source_family, + "selection", + "PRODUCTION_E2E_UNKNOWN_SOURCE_FAMILY", + "Enabled source family is not supported by this boundary.", + "request.enabled_source_families", + ), + ) + ) + continue + if normalized not in selected: + selected.append(normalized) + return tuple(selected), tuple(failures) + + +def _select_year_state( + source_family: str, + latest_year: int | None, + initial_year: int, +) -> ProductionE2EYearState: + return ProductionE2EYearState( + source_family=source_family, + year_state_key=_YEAR_STATE_KEYS[source_family], + latest_year=latest_year, + target_year=initial_year if latest_year is None else latest_year + 1, + initial_year=initial_year, + selection_status=( + ProductionE2EYearSelectionStatus.INITIAL_YEAR_SELECTED + if latest_year is None + else ProductionE2EYearSelectionStatus.NEXT_YEAR_SELECTED + ), + ) + + +def _coerce_insert_summary(result: object) -> ProductionE2EInsertSummary: + return ProductionE2EInsertSummary( + status=_status_value(getattr(result, "status")), + attempted=int(getattr(result, "attempted")), + inserted=int(getattr(result, "inserted")), + skipped_duplicate=int(getattr(result, "skipped_duplicate", 0)), + failed=int(getattr(result, "failed", 0)), + validation_error_count=int(getattr(result, "validation_error_count", 0)), + master_inserted=int(getattr(result, "master_inserted", 0)), + master_skipped=int(getattr(result, "master_skipped", 0)), + detail_inserted=int(getattr(result, "detail_inserted", 0)), + detail_skipped=int(getattr(result, "detail_skipped", 0)), + ) + + +def _batch_with_run_id( + batch: ParserNormalizedOutputBatch, + run_id: str, +) -> ParserNormalizedOutputBatch: + rows = [] + for row in batch.rows: + fields = dict(row.normalized_fields) + fields.setdefault("run_id", run_id) + rows.append( + replace( + row, + normalized_fields=tuple( + sorted(fields.items(), key=lambda item: item[0]) + ), + ) + ) + return ParserNormalizedOutputBatch(rows=tuple(rows)) + + +def _status_value(status: object) -> str: + value = getattr(status, "value", status) + return str(value) + + +def _redact_sensitive_text(value: str) -> str: + return redact_sensitive_text(value) + + +def _summarize( + requested_source_families: Sequence[str], + family_results: Sequence[ProductionE2EYearFamilyResult], +) -> ProductionE2EYearRunSummary: + return ProductionE2EYearRunSummary( + requested_family_count=len(requested_source_families), + completed_family_count=sum( + result.status is ProductionE2EYearFamilyStatus.COMPLETED + for result in family_results + ), + no_available_source_year_count=sum( + result.status is ProductionE2EYearFamilyStatus.NO_AVAILABLE_SOURCE_YEAR + for result in family_results + ), + failed_family_count=sum( + result.status is ProductionE2EYearFamilyStatus.FAILED + for result in family_results + ), + parsed_row_count=sum(result.parsed_row_count for result in family_results), + attempted_insert_count=sum( + result.insert_summary.attempted + for result in family_results + if result.insert_summary is not None + ), + inserted_count=sum( + result.insert_summary.inserted + for result in family_results + if result.insert_summary is not None + ), + skipped_duplicate_count=sum( + result.insert_summary.skipped_duplicate + for result in family_results + if result.insert_summary is not None + ), + failed_insert_count=sum( + result.insert_summary.failed + for result in family_results + if result.insert_summary is not None + ), + failure_count=sum(len(result.failures) for result in family_results), + ) + + +def _failed_family_without_runtime( + *, + source_family: str, + request: ProductionE2EYearOrchestratorRequest, + failure: ProductionE2EFailureDetail, + latest_year: int | None, +) -> ProductionE2EYearFamilyResult: + canonical = _SOURCE_FAMILY_ALIASES.get(source_family, source_family) + year_state_key = _YEAR_STATE_KEYS.get(canonical, source_family) + year_state = ProductionE2EYearState( + source_family=canonical, + year_state_key=year_state_key, + latest_year=latest_year, + target_year=request.initial_year if latest_year is None else latest_year + 1, + initial_year=request.initial_year, + selection_status=( + ProductionE2EYearSelectionStatus.INITIAL_YEAR_SELECTED + if latest_year is None + else ProductionE2EYearSelectionStatus.NEXT_YEAR_SELECTED + ), + ) + return _failed_family(year_state, failure) + + +def _failed_family( + year_state: ProductionE2EYearState, + *failures: ProductionE2EFailureDetail, + discovery_result: ProductionE2ESourceYearDiscoveryResult | None = None, + download_result: ProductionE2ESourceYearDownloadResult | None = None, + parsed_row_count: int = 0, + validation_result: ProductionE2EValidationResult | None = None, + insert_summary: ProductionE2EInsertSummary | None = None, +) -> ProductionE2EYearFamilyResult: + return ProductionE2EYearFamilyResult( + source_family=year_state.source_family, + status=ProductionE2EYearFamilyStatus.FAILED, + year_state=year_state, + discovery_result=discovery_result, + download_result=download_result, + parsed_row_count=parsed_row_count, + validation_result=validation_result, + insert_summary=insert_summary, + failures=tuple(failures), + ) + + +def _failure( + source_family: str | None, + stage: str, + code: str, + message: str, + field_name: str | None = None, +) -> ProductionE2EFailureDetail: + return ProductionE2EFailureDetail( + source_family=source_family, + stage=stage, + code=code, + message=message, + field_name=field_name, + ) diff --git a/src/carbonfactor_parser/pipeline/source_artifact_transport.py b/src/carbonfactor_parser/pipeline/source_artifact_transport.py new file mode 100644 index 00000000..5d9e5b80 --- /dev/null +++ b/src/carbonfactor_parser/pipeline/source_artifact_transport.py @@ -0,0 +1,46 @@ +"""Artifact transport helpers for configured source-year artifacts.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Callable +from urllib.parse import urlparse +from urllib.request import Request, urlopen + + +def build_configured_artifact_transport( + allow_live_source_access: bool, +) -> Callable[[str], bytes]: + """Build a source artifact transport with the configured live-access policy.""" + + def transport(uri: str) -> bytes: + return _configured_artifact_transport( + uri, + allow_live_source_access=allow_live_source_access, + ) + + return transport + + +def _configured_artifact_transport( + uri: str, + *, + allow_live_source_access: bool = False, +) -> bytes: + parsed = urlparse(uri) + if parsed.scheme == "file": + return Path(parsed.path).read_bytes() + if parsed.scheme in {"", "local"}: + return Path(parsed.path if parsed.scheme == "local" else uri).read_bytes() + if parsed.scheme == "https": + if not allow_live_source_access: + raise ValueError( + "Live HTTPS source access requires explicit real-source smoke opt-in.", + ) + request = Request(uri, headers={"User-Agent": "carbonops-parser/0.1"}) + with urlopen(request, timeout=60) as response: # noqa: S310 + return bytes(response.read()) + raise ValueError("Configured artifacts must use file, local path, or HTTPS URI.") + + +__all__ = ("build_configured_artifact_transport",) diff --git a/src/carbonfactor_parser/source_acquisition/contract_api.py b/src/carbonfactor_parser/source_acquisition/contract_api.py index b300300f..c5842120 100644 --- a/src/carbonfactor_parser/source_acquisition/contract_api.py +++ b/src/carbonfactor_parser/source_acquisition/contract_api.py @@ -102,6 +102,20 @@ validate_ipcc_source_discovery_result, validate_ipcc_source_document_candidate, ) +from carbonfactor_parser.source_acquisition.ipcc_source_download_execution_boundary import ( + IPCCSourceDownloadExecutionIssue, + IPCCSourceDownloadExecutionRequest, + IPCCSourceDownloadExecutionResult, + IPCCSourceDownloadExecutionStatus, + IPCCSourceDownloadExecutionValidationResult, + IPCCSourceDownloadTransport, + IPCCSourceDownloadTransportResponse, + IPCCSourceDownloadedArtifact, + create_ipcc_source_download_execution_request, + execute_ipcc_source_download, + validate_ipcc_source_download_execution_request, + validate_ipcc_source_download_execution_result, +) from carbonfactor_parser.source_acquisition.phase1_orchestration_plan_contract import ( Phase1OrchestrationPlan, Phase1OrchestrationPlanIssue, @@ -142,6 +156,15 @@ validate_source_acquisition_run_request, validate_source_acquisition_run_result, ) +from carbonfactor_parser.source_acquisition.run_repository_contract import ( + SourceAcquisitionRunRepository, + SourceAcquisitionRunRepositoryIssue, + SourceAcquisitionRunRepositoryPersistResult, + SourceAcquisitionRunRepositoryPersistStatus, + SourceAcquisitionRunRepositoryValidationResult, + create_source_acquisition_run_repository_persist_result, + validate_source_acquisition_run_repository_inputs, +) from carbonfactor_parser.source_acquisition.source_artifact_parser_input_bridge_contract import ( SourceArtifactParserInputBridgeEntry, SourceArtifactParserInputBridgeResult, @@ -209,7 +232,20 @@ "IPCCSourceDiscoveryStatus", "IPCCSourceDiscoveryValidationResult", "IPCCSourceDocumentCandidate", + "IPCCSourceDownloadExecutionIssue", + "IPCCSourceDownloadExecutionRequest", + "IPCCSourceDownloadExecutionResult", + "IPCCSourceDownloadExecutionStatus", + "IPCCSourceDownloadExecutionValidationResult", + "IPCCSourceDownloadTransport", + "IPCCSourceDownloadTransportResponse", + "IPCCSourceDownloadedArtifact", "SourceAcquisitionRunIssue", + "SourceAcquisitionRunRepository", + "SourceAcquisitionRunRepositoryIssue", + "SourceAcquisitionRunRepositoryPersistResult", + "SourceAcquisitionRunRepositoryPersistStatus", + "SourceAcquisitionRunRepositoryValidationResult", "SourceAcquisitionRunRequest", "SourceAcquisitionRunResult", "SourceAcquisitionRunStatus", @@ -246,9 +282,11 @@ "create_ghg_source_download_execution_request", "create_ipcc_source_discovery_request", "create_ipcc_source_discovery_result", + "create_ipcc_source_download_execution_request", "create_source_artifact_parser_input_bridge_entry", "create_source_acquisition_run_request", "create_source_acquisition_run_result", + "create_source_acquisition_run_repository_persist_result", "create_source_download_artifact_from_candidate", "validate_acquisition_to_parser_plan", "validate_acquisition_to_parser_plans", @@ -273,10 +311,14 @@ "validate_ipcc_source_discovery_request", "validate_ipcc_source_discovery_result", "validate_ipcc_source_document_candidate", + "execute_ipcc_source_download", + "validate_ipcc_source_download_execution_request", + "validate_ipcc_source_download_execution_result", "validate_source_artifact_parser_input_bridge_entry", "validate_source_artifact_parser_input_bridge_result", "validate_source_acquisition_run_request", "validate_source_acquisition_run_result", + "validate_source_acquisition_run_repository_inputs", "validate_source_discovery_candidate", "validate_source_discovery_candidate_result", "validate_source_download_artifact", diff --git a/src/carbonfactor_parser/source_acquisition/defra_source_discovery_boundary.py b/src/carbonfactor_parser/source_acquisition/defra_source_discovery_boundary.py index 56b28132..65a56bfb 100644 --- a/src/carbonfactor_parser/source_acquisition/defra_source_discovery_boundary.py +++ b/src/carbonfactor_parser/source_acquisition/defra_source_discovery_boundary.py @@ -403,6 +403,15 @@ def validate_defra_source_discovery_result( issues: list[DEFRASourceDiscoveryIssue] = [] issues.extend(validate_defra_source_discovery_request(result.request).issues) + if not isinstance(result.status, DEFRASourceDiscoveryStatus): + issues.append( + DEFRASourceDiscoveryIssue( + code="DEFRA_SOURCE_DISCOVERY_RESULT_INVALID_STATUS", + message="status must be a defined DEFRA source discovery status.", + field_name="status", + ) + ) + for field_name, value in ( ("no_network", result.no_network), ("no_download", result.no_download), @@ -431,6 +440,17 @@ def validate_defra_source_discovery_result( ) ) + if ( + result.status is DEFRASourceDiscoveryStatus.DECLARED + and len(result.issues) > 0 + ): + issues.append( + DEFRASourceDiscoveryIssue( + code="DEFRA_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + message="declared result status must not include issue metadata.", + field_name="issues", + ) + ) if result.status is DEFRASourceDiscoveryStatus.DECLARED and issues: issues.append( DEFRASourceDiscoveryIssue( diff --git a/src/carbonfactor_parser/source_acquisition/defra_source_download_execution_boundary.py b/src/carbonfactor_parser/source_acquisition/defra_source_download_execution_boundary.py index d698706f..588b74ee 100644 --- a/src/carbonfactor_parser/source_acquisition/defra_source_download_execution_boundary.py +++ b/src/carbonfactor_parser/source_acquisition/defra_source_download_execution_boundary.py @@ -489,6 +489,18 @@ def validate_defra_source_download_execution_result( validate_defra_source_download_execution_request(result.request).issues ) + if not isinstance(result.status, DEFRASourceDownloadExecutionStatus): + issues.append( + DEFRASourceDownloadExecutionIssue( + code="DEFRA_SOURCE_DOWNLOAD_RESULT_INVALID_STATUS", + message=( + "status must be a defined DEFRA source download execution " + "status." + ), + field_name="status", + ) + ) + for field_name, value in ( ("no_parse", result.no_parse), ("no_database_writes", result.no_database_writes), @@ -544,9 +556,23 @@ def _validate_source_reference_uri( request: DEFRASourceDownloadExecutionRequest, issues: list[DEFRASourceDownloadExecutionIssue], ) -> None: - parsed = urlparse(request.source_reference_uri) + if not isinstance(request.source_reference_uri, str) or not ( + source_reference_uri := request.source_reference_uri.strip() + ): + return + + parsed = urlparse(source_reference_uri) scheme = parsed.scheme if not scheme: + if "://" in source_reference_uri: + issues.append( + DEFRASourceDownloadExecutionIssue( + code="DEFRA_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + message="source_reference_uri must be a well-formed URI.", + field_name="source_reference_uri", + ) + ) + return issues.append( DEFRASourceDownloadExecutionIssue( code="DEFRA_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME", @@ -564,6 +590,15 @@ def _validate_source_reference_uri( ) ) return + if scheme in {"http", "https"} and not parsed.netloc: + issues.append( + DEFRASourceDownloadExecutionIssue( + code="DEFRA_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + message="source_reference_uri must be a well-formed URI.", + field_name="source_reference_uri", + ) + ) + return if scheme == "http": issues.append( DEFRASourceDownloadExecutionIssue( @@ -873,11 +908,30 @@ def _is_relative_to(path: Path, parent: Path) -> bool: def _validate_transport_response( - response: DEFRASourceDownloadTransportResponse, + response: DEFRASourceDownloadTransportResponse | object, ) -> DEFRASourceDownloadExecutionValidationResult: issues: list[DEFRASourceDownloadExecutionIssue] = [] + if response is None: + return DEFRASourceDownloadExecutionValidationResult( + issues=( + DEFRASourceDownloadExecutionIssue( + code="DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING", + message="transport response is required.", + field_name="transport", + ), + ) + ) + content = getattr(response, "content", None) - if not isinstance(content, bytes): + if content is None: + issues.append( + DEFRASourceDownloadExecutionIssue( + code="DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT", + message="transport response content is required.", + field_name="transport.content", + ) + ) + elif not isinstance(content, bytes): issues.append( DEFRASourceDownloadExecutionIssue( code="DEFRA_SOURCE_DOWNLOAD_RESPONSE_CONTENT_NOT_BYTES", @@ -894,6 +948,21 @@ def _validate_transport_response( ) ) + _validate_optional_text( + getattr(response, "content_type", None), + "transport.content_type", + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "response content_type must be non-empty when provided.", + issues, + ) + _validate_optional_text( + getattr(response, "final_uri", None), + "transport.final_uri", + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + "response final_uri must be non-empty when provided.", + issues, + ) + return DEFRASourceDownloadExecutionValidationResult(issues=tuple(issues)) diff --git a/src/carbonfactor_parser/source_acquisition/ghg_source_discovery_boundary.py b/src/carbonfactor_parser/source_acquisition/ghg_source_discovery_boundary.py index 201f8b76..ff3c957a 100644 --- a/src/carbonfactor_parser/source_acquisition/ghg_source_discovery_boundary.py +++ b/src/carbonfactor_parser/source_acquisition/ghg_source_discovery_boundary.py @@ -403,6 +403,15 @@ def validate_ghg_source_discovery_result( issues: list[GHGSourceDiscoveryIssue] = [] issues.extend(validate_ghg_source_discovery_request(result.request).issues) + if not isinstance(result.status, GHGSourceDiscoveryStatus): + issues.append( + GHGSourceDiscoveryIssue( + code="GHG_SOURCE_DISCOVERY_RESULT_INVALID_STATUS", + message="status must be a defined GHG source discovery status.", + field_name="status", + ) + ) + for field_name, value in ( ("no_network", result.no_network), ("no_download", result.no_download), @@ -431,6 +440,17 @@ def validate_ghg_source_discovery_result( ) ) + if ( + result.status is GHGSourceDiscoveryStatus.DECLARED + and len(result.issues) > 0 + ): + issues.append( + GHGSourceDiscoveryIssue( + code="GHG_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + message="declared result status must not include issue metadata.", + field_name="issues", + ) + ) if result.status is GHGSourceDiscoveryStatus.DECLARED and issues: issues.append( GHGSourceDiscoveryIssue( diff --git a/src/carbonfactor_parser/source_acquisition/ghg_source_download_execution_boundary.py b/src/carbonfactor_parser/source_acquisition/ghg_source_download_execution_boundary.py index 56d7cc1c..8e6ee82d 100644 --- a/src/carbonfactor_parser/source_acquisition/ghg_source_download_execution_boundary.py +++ b/src/carbonfactor_parser/source_acquisition/ghg_source_download_execution_boundary.py @@ -542,9 +542,23 @@ def _validate_source_reference_uri( request: GHGSourceDownloadExecutionRequest, issues: list[GHGSourceDownloadExecutionIssue], ) -> None: - parsed = urlparse(request.source_reference_uri) + if not isinstance(request.source_reference_uri, str) or not ( + source_reference_uri := request.source_reference_uri.strip() + ): + return + + parsed = urlparse(source_reference_uri) scheme = parsed.scheme if not scheme: + if "://" in source_reference_uri: + issues.append( + GHGSourceDownloadExecutionIssue( + code="GHG_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + message="source_reference_uri must be a well-formed URI.", + field_name="source_reference_uri", + ) + ) + return issues.append( GHGSourceDownloadExecutionIssue( code="GHG_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME", @@ -562,6 +576,15 @@ def _validate_source_reference_uri( ) ) return + if scheme in {"http", "https"} and not parsed.netloc: + issues.append( + GHGSourceDownloadExecutionIssue( + code="GHG_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + message="source_reference_uri must be a well-formed URI.", + field_name="source_reference_uri", + ) + ) + return if scheme == "http": issues.append( GHGSourceDownloadExecutionIssue( @@ -871,11 +894,30 @@ def _is_relative_to(path: Path, parent: Path) -> bool: def _validate_transport_response( - response: GHGSourceDownloadTransportResponse, + response: GHGSourceDownloadTransportResponse | object, ) -> GHGSourceDownloadExecutionValidationResult: issues: list[GHGSourceDownloadExecutionIssue] = [] + if response is None: + return GHGSourceDownloadExecutionValidationResult( + issues=( + GHGSourceDownloadExecutionIssue( + code="GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING", + message="transport response is required.", + field_name="transport", + ), + ) + ) + content = getattr(response, "content", None) - if not isinstance(content, bytes): + if content is None: + issues.append( + GHGSourceDownloadExecutionIssue( + code="GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT", + message="transport response content is required.", + field_name="transport.content", + ) + ) + elif not isinstance(content, bytes): issues.append( GHGSourceDownloadExecutionIssue( code="GHG_SOURCE_DOWNLOAD_RESPONSE_CONTENT_NOT_BYTES", @@ -892,6 +934,21 @@ def _validate_transport_response( ) ) + _validate_optional_text( + getattr(response, "content_type", None), + "transport.content_type", + "GHG_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "response content_type must be non-empty when provided.", + issues, + ) + _validate_optional_text( + getattr(response, "final_uri", None), + "transport.final_uri", + "GHG_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + "response final_uri must be non-empty when provided.", + issues, + ) + return GHGSourceDownloadExecutionValidationResult(issues=tuple(issues)) diff --git a/src/carbonfactor_parser/source_acquisition/ipcc_source_discovery_boundary.py b/src/carbonfactor_parser/source_acquisition/ipcc_source_discovery_boundary.py index 467b0775..53753fa5 100644 --- a/src/carbonfactor_parser/source_acquisition/ipcc_source_discovery_boundary.py +++ b/src/carbonfactor_parser/source_acquisition/ipcc_source_discovery_boundary.py @@ -403,6 +403,15 @@ def validate_ipcc_source_discovery_result( issues: list[IPCCSourceDiscoveryIssue] = [] issues.extend(validate_ipcc_source_discovery_request(result.request).issues) + if not isinstance(result.status, IPCCSourceDiscoveryStatus): + issues.append( + IPCCSourceDiscoveryIssue( + code="IPCC_SOURCE_DISCOVERY_RESULT_INVALID_STATUS", + message="status must be a defined IPCC source discovery status.", + field_name="status", + ) + ) + for field_name, value in ( ("no_network", result.no_network), ("no_download", result.no_download), @@ -431,6 +440,17 @@ def validate_ipcc_source_discovery_result( ) ) + if ( + result.status is IPCCSourceDiscoveryStatus.DECLARED + and len(result.issues) > 0 + ): + issues.append( + IPCCSourceDiscoveryIssue( + code="IPCC_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + message="declared result status must not include issue metadata.", + field_name="issues", + ) + ) if result.status is IPCCSourceDiscoveryStatus.DECLARED and issues: issues.append( IPCCSourceDiscoveryIssue( diff --git a/src/carbonfactor_parser/source_acquisition/ipcc_source_download_execution_boundary.py b/src/carbonfactor_parser/source_acquisition/ipcc_source_download_execution_boundary.py new file mode 100644 index 00000000..65990950 --- /dev/null +++ b/src/carbonfactor_parser/source_acquisition/ipcc_source_download_execution_boundary.py @@ -0,0 +1,1249 @@ +"""Explicit IPCC-only source download execution boundary.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +import errno +from hashlib import sha256 +import os +from pathlib import Path +from typing import Protocol +from urllib.parse import urlparse + +from carbonfactor_parser.source_acquisition.ipcc_source_discovery_boundary import ( + IPCC_SOURCE_FAMILY, + IPCC_SOURCE_KEY, + IPCCSourceDocumentCandidate, +) + + +class IPCCSourceDownloadExecutionStatus(str, Enum): + """Status values for IPCC source download execution.""" + + BLOCKED = "blocked" + DOWNLOADED = "downloaded" + FAILED = "failed" + + +@dataclass(frozen=True) +class IPCCSourceDownloadExecutionRequest: + """Explicit opt-in request to download one IPCC source candidate.""" + + source_family: str + source_key: str + candidate_id: str + candidate_title: str + source_reference_uri: str + artifact_kind: str + target_root: str + target_relative_path: str + candidate_download_allowed: bool = False + allow_download_execution: bool = False + allow_file_write: bool = False + allow_network: bool = False + allow_overwrite: bool = False + allow_parse: bool = False + allow_database_writes: bool = False + allow_scheduler: bool = False + content_type: str | None = None + extension: str | None = None + expected_checksum_sha256: str | None = None + document_year: int | None = None + reporting_year: int | None = None + version_label: str | None = None + retrieved_at_label: str = "download_execution_retrieved_at_caller_boundary" + + +@dataclass(frozen=True) +class IPCCSourceDownloadTransportResponse: + """Downloaded payload returned by a caller-provided transport.""" + + content: bytes + content_type: str | None = None + final_uri: str | None = None + + +class IPCCSourceDownloadTransport(Protocol): + """Caller-provided transport for explicit IPCC source download execution.""" + + def __call__( + self, + source_reference_uri: str, + ) -> IPCCSourceDownloadTransportResponse: + """Return downloaded content for the provided source reference.""" + + +@dataclass(frozen=True) +class IPCCSourceDownloadedArtifact: + """Local artifact produced by explicit IPCC source download execution.""" + + source_family: str + source_key: str + candidate_id: str + artifact_id: str + artifact_kind: str + source_reference_uri: str + local_path: str + original_filename: str + checksum_sha256: str + size_bytes: int + content_type: str | None = None + extension: str | None = None + final_uri: str | None = None + storage_identity: str | None = None + document_year: int | None = None + reporting_year: int | None = None + version_label: str | None = None + retrieved_at_label: str | None = None + reused_existing: bool = False + + +@dataclass(frozen=True) +class IPCCSourceDownloadExecutionIssue: + """Validation or execution issue for IPCC source downloads.""" + + code: str + message: str + field_name: str + severity: str = "error" + + +@dataclass(frozen=True) +class IPCCSourceDownloadExecutionValidationResult: + """Structural validation result for IPCC source download requests.""" + + issues: tuple[IPCCSourceDownloadExecutionIssue, ...] = () + + @property + def is_valid(self) -> bool: + return not self.issues + + +@dataclass(frozen=True) +class IPCCSourceDownloadExecutionResult: + """Result of explicit IPCC source download execution.""" + + status: IPCCSourceDownloadExecutionStatus + request: IPCCSourceDownloadExecutionRequest + artifact: IPCCSourceDownloadedArtifact | None = None + issues: tuple[IPCCSourceDownloadExecutionIssue, ...] = () + no_parse: bool = True + no_database_writes: bool = True + no_sql: bool = True + no_scheduler: bool = True + + @property + def downloaded(self) -> bool: + return self.status is IPCCSourceDownloadExecutionStatus.DOWNLOADED + + +@dataclass(frozen=True) +class _SafeTargetPath: + target_path: Path + resolved_root: Path + resolved_parent: Path + resolved_target_path: Path + parent_fd: int + + +def create_ipcc_source_download_execution_request( + candidate: IPCCSourceDocumentCandidate, + *, + target_root: str, + target_relative_path: str, + allow_download_execution: bool = False, + allow_file_write: bool = False, + allow_network: bool = False, + allow_overwrite: bool = False, + retrieved_at_label: str = "download_execution_retrieved_at_caller_boundary", +) -> IPCCSourceDownloadExecutionRequest: + """Create an explicit IPCC download request from candidate metadata.""" + + return IPCCSourceDownloadExecutionRequest( + source_family=candidate.source_family, + source_key=candidate.source_key, + candidate_id=candidate.candidate_id, + candidate_title=candidate.title, + source_reference_uri=candidate.reference_uri, + artifact_kind=candidate.artifact_kind, + target_root=target_root, + target_relative_path=target_relative_path, + candidate_download_allowed=candidate.download_allowed, + allow_download_execution=allow_download_execution, + allow_file_write=allow_file_write, + allow_network=allow_network, + allow_overwrite=allow_overwrite, + content_type=candidate.content_type, + extension=candidate.extension, + expected_checksum_sha256=candidate.checksum_sha256, + document_year=candidate.document_year, + reporting_year=candidate.reporting_year, + version_label=candidate.version_label, + retrieved_at_label=retrieved_at_label, + ) + + +def validate_ipcc_source_download_execution_request( + request: IPCCSourceDownloadExecutionRequest, +) -> IPCCSourceDownloadExecutionValidationResult: + """Validate an IPCC source download request without executing it.""" + + issues: list[IPCCSourceDownloadExecutionIssue] = [] + + _validate_required_text( + request.source_family, + "source_family", + "IPCC_SOURCE_DOWNLOAD_MISSING_SOURCE_FAMILY", + "source_family must be a non-empty string.", + issues, + ) + _validate_required_text( + request.source_key, + "source_key", + "IPCC_SOURCE_DOWNLOAD_MISSING_SOURCE_KEY", + "source_key must be a non-empty string.", + issues, + ) + _validate_required_text( + request.candidate_id, + "candidate_id", + "IPCC_SOURCE_DOWNLOAD_MISSING_CANDIDATE_ID", + "candidate_id must be a non-empty string.", + issues, + ) + _validate_required_text( + request.candidate_title, + "candidate_title", + "IPCC_SOURCE_DOWNLOAD_MISSING_CANDIDATE_TITLE", + "candidate_title must be a non-empty string.", + issues, + ) + _validate_required_text( + request.source_reference_uri, + "source_reference_uri", + "IPCC_SOURCE_DOWNLOAD_MISSING_SOURCE_REFERENCE_URI", + "source_reference_uri must be a non-empty string.", + issues, + ) + _validate_required_text( + request.artifact_kind, + "artifact_kind", + "IPCC_SOURCE_DOWNLOAD_MISSING_ARTIFACT_KIND", + "artifact_kind must be a non-empty string.", + issues, + ) + _validate_required_text( + request.target_root, + "target_root", + "IPCC_SOURCE_DOWNLOAD_MISSING_TARGET_ROOT", + "target_root must be a non-empty string.", + issues, + ) + _validate_required_text( + request.target_relative_path, + "target_relative_path", + "IPCC_SOURCE_DOWNLOAD_MISSING_TARGET_RELATIVE_PATH", + "target_relative_path must be a non-empty string.", + issues, + ) + _validate_optional_text( + request.content_type, + "content_type", + "IPCC_SOURCE_DOWNLOAD_BLANK_CONTENT_TYPE", + "content_type must be non-empty when provided.", + issues, + ) + _validate_optional_text( + request.extension, + "extension", + "IPCC_SOURCE_DOWNLOAD_BLANK_EXTENSION", + "extension must be non-empty when provided.", + issues, + ) + _validate_optional_text( + request.expected_checksum_sha256, + "expected_checksum_sha256", + "IPCC_SOURCE_DOWNLOAD_BLANK_EXPECTED_CHECKSUM_SHA256", + "expected_checksum_sha256 must be non-empty when provided.", + issues, + ) + _validate_optional_text( + request.version_label, + "version_label", + "IPCC_SOURCE_DOWNLOAD_BLANK_VERSION_LABEL", + "version_label must be non-empty when provided.", + issues, + ) + _validate_required_text( + request.retrieved_at_label, + "retrieved_at_label", + "IPCC_SOURCE_DOWNLOAD_MISSING_RETRIEVED_AT_LABEL", + "retrieved_at_label must be a non-empty string.", + issues, + ) + _validate_optional_positive_int( + request.document_year, + "document_year", + "IPCC_SOURCE_DOWNLOAD_INVALID_DOCUMENT_YEAR", + "document_year must be a positive integer when provided.", + issues, + ) + _validate_optional_positive_int( + request.reporting_year, + "reporting_year", + "IPCC_SOURCE_DOWNLOAD_INVALID_REPORTING_YEAR", + "reporting_year must be a positive integer when provided.", + issues, + ) + + if request.source_family != IPCC_SOURCE_FAMILY: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_SOURCE_FAMILY_MISMATCH", + message="source_family must be ipcc_efdb.", + field_name="source_family", + ) + ) + if request.source_key != IPCC_SOURCE_KEY: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_SOURCE_KEY_MISMATCH", + message="source_key must be ipcc_efdb.", + field_name="source_key", + ) + ) + _validate_true( + request.candidate_download_allowed, + "candidate_download_allowed", + "IPCC_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE", + "candidate metadata must explicitly allow download execution.", + issues, + ) + _validate_true( + request.allow_download_execution, + "allow_download_execution", + "IPCC_SOURCE_DOWNLOAD_EXECUTION_NOT_ALLOWED", + "allow_download_execution must be True.", + issues, + ) + _validate_true( + request.allow_file_write, + "allow_file_write", + "IPCC_SOURCE_DOWNLOAD_FILE_WRITE_NOT_ALLOWED", + "allow_file_write must be True.", + issues, + ) + _validate_false( + request.allow_parse, + "allow_parse", + "IPCC_SOURCE_DOWNLOAD_PARSE_NOT_ALLOWED", + "allow_parse must be False for this boundary.", + issues, + ) + _validate_false( + request.allow_database_writes, + "allow_database_writes", + "IPCC_SOURCE_DOWNLOAD_DATABASE_WRITES_NOT_ALLOWED", + "allow_database_writes must be False for this boundary.", + issues, + ) + _validate_false( + request.allow_scheduler, + "allow_scheduler", + "IPCC_SOURCE_DOWNLOAD_SCHEDULER_NOT_ALLOWED", + "allow_scheduler must be False for this boundary.", + issues, + ) + + _validate_source_reference_uri(request, issues) + _validate_target_paths(request, issues) + + return IPCCSourceDownloadExecutionValidationResult(issues=tuple(issues)) + + +def execute_ipcc_source_download( + request: IPCCSourceDownloadExecutionRequest, + transport: IPCCSourceDownloadTransport, +) -> IPCCSourceDownloadExecutionResult: + """Execute one explicit IPCC source download using a provided transport.""" + + validation = validate_ipcc_source_download_execution_request(request) + if not validation.is_valid: + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.BLOCKED, + request=request, + issues=validation.issues, + ) + + safe_target, target_issues = _prepare_safe_target_path(request) + if target_issues: + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.BLOCKED, + request=request, + issues=target_issues, + ) + if safe_target is None: + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.BLOCKED, + request=request, + issues=( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_PATH_UNRESOLVED", + message="target path could not be resolved safely.", + field_name="target_relative_path", + ), + ), + ) + + try: + if safe_target.resolved_target_path.exists() and not request.allow_overwrite: + existing_result = _create_existing_artifact_result(request, safe_target) + if existing_result is not None: + return existing_result + + try: + response = transport(request.source_reference_uri) + except Exception as error: # noqa: BLE001 + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.FAILED, + request=request, + issues=( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TRANSPORT_FAILED", + message=f"transport failed: {error}", + field_name="source_reference_uri", + ), + ), + ) + + response_validation = _validate_transport_response(response) + if not response_validation.is_valid: + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.FAILED, + request=request, + issues=response_validation.issues, + ) + + content = bytes(response.content) + checksum_sha256 = sha256(content).hexdigest() + if ( + request.expected_checksum_sha256 is not None + and checksum_sha256.lower() != request.expected_checksum_sha256.lower() + ): + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.FAILED, + request=request, + issues=( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH", + message=( + "downloaded content checksum did not match expected " + "value." + ), + field_name="expected_checksum_sha256", + ), + ), + ) + + try: + _write_content_to_safe_target( + safe_target, + content, + allow_overwrite=request.allow_overwrite, + ) + except Exception as error: # noqa: BLE001 + issue_code = "IPCC_SOURCE_DOWNLOAD_WRITE_FAILED" + if isinstance(error, FileExistsError): + issue_code = "IPCC_SOURCE_DOWNLOAD_TARGET_EXISTS" + elif isinstance(error, OSError) and error.errno == errno.ELOOP: + issue_code = "IPCC_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE" + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.FAILED, + request=request, + issues=( + IPCCSourceDownloadExecutionIssue( + code=issue_code, + message=f"target write failed: {error}", + field_name="target_relative_path", + ), + ), + ) + + artifact = _create_artifact( + request, + safe_target, + checksum_sha256=checksum_sha256, + size_bytes=len(content), + content_type=response.content_type or request.content_type, + final_uri=response.final_uri, + reused_existing=False, + ) + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.DOWNLOADED, + request=request, + artifact=artifact, + ) + finally: + _close_safe_target_path(safe_target) + + +def validate_ipcc_source_download_execution_result( + result: IPCCSourceDownloadExecutionResult, +) -> IPCCSourceDownloadExecutionValidationResult: + """Validate an IPCC source download execution result.""" + + issues: list[IPCCSourceDownloadExecutionIssue] = [] + issues.extend( + validate_ipcc_source_download_execution_request(result.request).issues + ) + + if not isinstance(result.status, IPCCSourceDownloadExecutionStatus): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_RESULT_INVALID_STATUS", + message=( + "status must be a defined IPCC source download execution " + "status." + ), + field_name="status", + ) + ) + + for field_name, value in ( + ("no_parse", result.no_parse), + ("no_database_writes", result.no_database_writes), + ("no_sql", result.no_sql), + ("no_scheduler", result.no_scheduler), + ): + if value is not True: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED", + message=f"{field_name} must remain True.", + field_name=field_name, + ) + ) + + if result.status is IPCCSourceDownloadExecutionStatus.DOWNLOADED: + if result.artifact is None: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_RESULT_MISSING_ARTIFACT", + message="downloaded results require artifact metadata.", + field_name="artifact", + ) + ) + elif result.artifact is not None: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_RESULT_UNEXPECTED_ARTIFACT", + message="non-downloaded results must not include artifact metadata.", + field_name="artifact", + ) + ) + + if ( + result.status is not IPCCSourceDownloadExecutionStatus.DOWNLOADED + and not result.issues + ): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_RESULT_MISSING_ISSUES", + message="blocked or failed results require issue metadata.", + field_name="issues", + ) + ) + + if result.artifact is not None: + issues.extend(_validate_artifact(result.artifact).issues) + + return IPCCSourceDownloadExecutionValidationResult(issues=tuple(issues)) + + +def _validate_source_reference_uri( + request: IPCCSourceDownloadExecutionRequest, + issues: list[IPCCSourceDownloadExecutionIssue], +) -> None: + if not isinstance(request.source_reference_uri, str) or not ( + source_reference_uri := request.source_reference_uri.strip() + ): + return + + parsed = urlparse(source_reference_uri) + scheme = parsed.scheme + if not scheme: + if "://" in source_reference_uri: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + message="source_reference_uri must be a well-formed URI.", + field_name="source_reference_uri", + ) + ) + return + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME", + message="source_reference_uri must include a URI scheme.", + field_name="source_reference_uri", + ) + ) + return + if scheme == "discovery": + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE", + message="discovery references are not direct download references.", + field_name="source_reference_uri", + ) + ) + return + if scheme in {"http", "https"} and not parsed.netloc: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + message="source_reference_uri must be a well-formed URI.", + field_name="source_reference_uri", + ) + ) + return + if scheme == "http": + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_INSECURE_HTTP_NOT_ALLOWED", + message="source_reference_uri must not use insecure HTTP.", + field_name="source_reference_uri", + ) + ) + return + if scheme == "https" and not request.allow_network: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_NETWORK_NOT_ALLOWED", + message="allow_network must be True for HTTPS references.", + field_name="allow_network", + ) + ) + if scheme not in {"https", "memory", "mock"}: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI", + message="source_reference_uri scheme is not allowed.", + field_name="source_reference_uri", + ) + ) + + +def _validate_target_paths( + request: IPCCSourceDownloadExecutionRequest, + issues: list[IPCCSourceDownloadExecutionIssue], +) -> None: + target_root = Path(request.target_root) + target_relative_path = Path(request.target_relative_path) + + if not target_root.is_absolute(): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_ABSOLUTE", + message="target_root must be an absolute path.", + field_name="target_root", + ) + ) + if target_relative_path.is_absolute(): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_ABSOLUTE", + message="target_relative_path must be relative.", + field_name="target_relative_path", + ) + ) + if "://" in request.target_relative_path: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_URI", + message="target_relative_path must not be a URI.", + field_name="target_relative_path", + ) + ) + if any(part in {"", ".", ".."} for part in target_relative_path.parts): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE", + message=( + "target_relative_path must not contain empty, current, or " + "parent segments." + ), + field_name="target_relative_path", + ) + ) + + +def _prepare_safe_target_path( + request: IPCCSourceDownloadExecutionRequest, +) -> tuple[_SafeTargetPath | None, tuple[IPCCSourceDownloadExecutionIssue, ...]]: + issues: list[IPCCSourceDownloadExecutionIssue] = [] + target_root = Path(request.target_root) + target_relative_path = Path(request.target_relative_path) + + try: + target_root.mkdir(parents=True, exist_ok=True) + except OSError as error: + return None, ( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_ROOT_CREATE_FAILED", + message=f"target_root could not be created: {error}", + field_name="target_root", + ), + ) + + if not target_root.is_dir(): + return None, ( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_DIRECTORY", + message="target_root must resolve to a directory.", + field_name="target_root", + ), + ) + + try: + resolved_root = target_root.resolve(strict=True) + except OSError as error: + return None, ( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_ROOT_RESOLVE_FAILED", + message=f"target_root could not be resolved: {error}", + field_name="target_root", + ), + ) + + current_path = target_root + resolved_parent = resolved_root + for part in target_relative_path.parts[:-1]: + current_path = current_path / part + if current_path.is_symlink(): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE", + message="target parent path must not contain symlinks.", + field_name="target_relative_path", + ) + ) + continue + try: + if current_path.exists(): + if not current_path.is_dir(): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_PARENT_NOT_DIRECTORY", + message="target parent path must be a directory.", + field_name="target_relative_path", + ) + ) + continue + else: + current_path.mkdir() + except OSError as error: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_PARENT_CREATE_FAILED", + message=f"target parent path could not be prepared: {error}", + field_name="target_relative_path", + ) + ) + continue + + try: + resolved_parent = current_path.resolve(strict=True) + except OSError as error: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_PARENT_RESOLVE_FAILED", + message=f"target parent path could not be resolved: {error}", + field_name="target_relative_path", + ) + ) + continue + if not _is_relative_to(resolved_parent, resolved_root): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_CONTAINMENT_UNSAFE", + message="resolved target parent escapes target_root.", + field_name="target_relative_path", + ) + ) + + target_path = target_root / target_relative_path + if target_path.is_symlink(): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE", + message="target path must not be an existing symlink.", + field_name="target_relative_path", + ) + ) + elif target_path.exists(): + if target_path.is_dir(): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_NOT_FILE", + message="target path must not be a directory.", + field_name="target_relative_path", + ) + ) + + resolved_target_path = resolved_parent / target_relative_path.name + if not _is_relative_to(resolved_target_path, resolved_root): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_CONTAINMENT_UNSAFE", + message="resolved target path escapes target_root.", + field_name="target_relative_path", + ) + ) + + if issues: + return None, tuple(issues) + + parent_fd, parent_fd_issue = _open_safe_parent_directory_fd(resolved_parent) + if parent_fd_issue is not None: + return None, (parent_fd_issue,) + + return ( + _SafeTargetPath( + target_path=target_path, + resolved_root=resolved_root, + resolved_parent=resolved_parent, + resolved_target_path=resolved_target_path, + parent_fd=parent_fd, + ), + (), + ) + + +def _open_safe_parent_directory_fd( + resolved_parent: Path, +) -> tuple[int, IPCCSourceDownloadExecutionIssue | None]: + if not hasattr(os, "O_NOFOLLOW") or not hasattr(os, "O_DIRECTORY"): + return -1, IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_FD_UNSUPPORTED", + message="platform does not expose required safe directory flags.", + field_name="target_relative_path", + ) + if os.open not in getattr(os, "supports_dir_fd", set()): + return -1, IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TARGET_FD_UNSUPPORTED", + message="platform does not support directory-relative file opening.", + field_name="target_relative_path", + ) + + directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + try: + return os.open(resolved_parent, directory_flags), None + except OSError as error: + issue_code = "IPCC_SOURCE_DOWNLOAD_TARGET_PARENT_OPEN_FAILED" + if error.errno == errno.ELOOP: + issue_code = "IPCC_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE" + return -1, IPCCSourceDownloadExecutionIssue( + code=issue_code, + message=f"target parent directory could not be opened safely: {error}", + field_name="target_relative_path", + ) + + +def _write_content_to_safe_target( + safe_target: _SafeTargetPath, + content: bytes, + *, + allow_overwrite: bool, +) -> None: + _ensure_parent_path_still_matches_open_fd(safe_target) + + flags = os.O_WRONLY | os.O_CREAT + flags |= os.O_TRUNC if allow_overwrite else os.O_EXCL + flags |= os.O_NOFOLLOW + + file_fd = os.open( + safe_target.resolved_target_path.name, + flags, + 0o600, + dir_fd=safe_target.parent_fd, + ) + try: + target_file = os.fdopen(file_fd, "wb") + except Exception: + os.close(file_fd) + raise + with target_file: + target_file.write(content) + + +def _create_existing_artifact_result( + request: IPCCSourceDownloadExecutionRequest, + safe_target: _SafeTargetPath, +) -> IPCCSourceDownloadExecutionResult | None: + try: + content = _read_existing_content_from_safe_target(safe_target) + except Exception as error: # noqa: BLE001 + issue_code = "IPCC_SOURCE_DOWNLOAD_EXISTING_READ_FAILED" + if isinstance(error, OSError) and error.errno == errno.ELOOP: + issue_code = "IPCC_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE" + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.BLOCKED, + request=request, + issues=( + IPCCSourceDownloadExecutionIssue( + code=issue_code, + message=f"existing target could not be read safely: {error}", + field_name="target_relative_path", + ), + ), + ) + + if len(content) == 0: + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.FAILED, + request=request, + issues=( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_EXISTING_EMPTY_CONTENT", + message="existing target content must not be empty.", + field_name="target_relative_path", + ), + ), + ) + + checksum_sha256 = sha256(content).hexdigest() + if ( + request.expected_checksum_sha256 is not None + and checksum_sha256.lower() != request.expected_checksum_sha256.lower() + ): + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.FAILED, + request=request, + issues=( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_EXISTING_CHECKSUM_MISMATCH", + message="existing target checksum did not match expected value.", + field_name="expected_checksum_sha256", + ), + ), + ) + + return IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.DOWNLOADED, + request=request, + artifact=_create_artifact( + request, + safe_target, + checksum_sha256=checksum_sha256, + size_bytes=len(content), + content_type=request.content_type, + final_uri=request.source_reference_uri, + reused_existing=True, + ), + ) + + +def _read_existing_content_from_safe_target(safe_target: _SafeTargetPath) -> bytes: + _ensure_parent_path_still_matches_open_fd(safe_target) + + file_fd = os.open( + safe_target.resolved_target_path.name, + os.O_RDONLY | os.O_NOFOLLOW, + dir_fd=safe_target.parent_fd, + ) + try: + target_file = os.fdopen(file_fd, "rb") + except Exception: + os.close(file_fd) + raise + with target_file: + return target_file.read() + + +def _create_artifact( + request: IPCCSourceDownloadExecutionRequest, + safe_target: _SafeTargetPath, + *, + checksum_sha256: str, + size_bytes: int, + content_type: str | None, + final_uri: str | None, + reused_existing: bool, +) -> IPCCSourceDownloadedArtifact: + storage_identity = str(safe_target.resolved_target_path) + return IPCCSourceDownloadedArtifact( + source_family=request.source_family, + source_key=request.source_key, + candidate_id=request.candidate_id, + artifact_id=f"ipcc_source_download_artifact_{request.candidate_id}", + artifact_kind=request.artifact_kind, + source_reference_uri=request.source_reference_uri, + local_path=storage_identity, + original_filename=safe_target.resolved_target_path.name, + checksum_sha256=checksum_sha256, + size_bytes=size_bytes, + content_type=content_type, + extension=request.extension, + final_uri=final_uri, + storage_identity=storage_identity, + document_year=request.document_year, + reporting_year=request.reporting_year, + version_label=request.version_label, + retrieved_at_label=request.retrieved_at_label, + reused_existing=reused_existing, + ) + + +def _close_safe_target_path(safe_target: _SafeTargetPath) -> None: + os.close(safe_target.parent_fd) + + +def _ensure_parent_path_still_matches_open_fd( + safe_target: _SafeTargetPath, +) -> None: + if safe_target.resolved_parent.is_symlink(): + raise OSError(errno.ELOOP, "target parent path changed to a symlink") + + parent_fd_stat = os.fstat(safe_target.parent_fd) + parent_path_stat = safe_target.resolved_parent.stat() + if ( + parent_fd_stat.st_dev != parent_path_stat.st_dev + or parent_fd_stat.st_ino != parent_path_stat.st_ino + ): + raise OSError( + getattr(errno, "ESTALE", errno.EIO), + "target parent path changed after validation", + ) + + +def _is_relative_to(path: Path, parent: Path) -> bool: + try: + path.relative_to(parent) + except ValueError: + return False + return True + + +def _validate_transport_response( + response: IPCCSourceDownloadTransportResponse | object, +) -> IPCCSourceDownloadExecutionValidationResult: + issues: list[IPCCSourceDownloadExecutionIssue] = [] + if response is None: + return IPCCSourceDownloadExecutionValidationResult( + issues=( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_RESPONSE_MISSING", + message="transport response is required.", + field_name="transport", + ), + ) + ) + + content = getattr(response, "content", None) + if content is None: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT", + message="transport response content is required.", + field_name="transport.content", + ) + ) + elif not isinstance(content, bytes): + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_RESPONSE_CONTENT_NOT_BYTES", + message="transport response content must be bytes.", + field_name="transport.content", + ) + ) + elif len(content) == 0: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_RESPONSE_EMPTY_CONTENT", + message="transport response content must not be empty.", + field_name="transport.content", + ) + ) + + _validate_optional_text( + getattr(response, "content_type", None), + "transport.content_type", + "IPCC_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "response content_type must be non-empty when provided.", + issues, + ) + _validate_optional_text( + getattr(response, "final_uri", None), + "transport.final_uri", + "IPCC_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + "response final_uri must be non-empty when provided.", + issues, + ) + + return IPCCSourceDownloadExecutionValidationResult(issues=tuple(issues)) + + +def _validate_artifact( + artifact: IPCCSourceDownloadedArtifact, +) -> IPCCSourceDownloadExecutionValidationResult: + issues: list[IPCCSourceDownloadExecutionIssue] = [] + _validate_required_text( + artifact.local_path, + "artifact.local_path", + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_MISSING_LOCAL_PATH", + "artifact local_path must be a non-empty string.", + issues, + ) + _validate_required_text( + artifact.checksum_sha256, + "artifact.checksum_sha256", + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_MISSING_CHECKSUM_SHA256", + "artifact checksum_sha256 must be a non-empty string.", + issues, + ) + _validate_required_text( + artifact.source_reference_uri, + "artifact.source_reference_uri", + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_MISSING_SOURCE_REFERENCE_URI", + "artifact source_reference_uri must be a non-empty string.", + issues, + ) + _validate_optional_text( + artifact.storage_identity, + "artifact.storage_identity", + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_BLANK_STORAGE_IDENTITY", + "artifact storage_identity must be non-empty when provided.", + issues, + ) + _validate_optional_text( + artifact.version_label, + "artifact.version_label", + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_BLANK_VERSION_LABEL", + "artifact version_label must be non-empty when provided.", + issues, + ) + _validate_required_text( + artifact.retrieved_at_label, + "artifact.retrieved_at_label", + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_MISSING_RETRIEVED_AT_LABEL", + "artifact retrieved_at_label must be a non-empty string.", + issues, + ) + _validate_optional_positive_int( + artifact.size_bytes, + "artifact.size_bytes", + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_INVALID_SIZE_BYTES", + "artifact size_bytes must be a positive integer.", + issues, + ) + if artifact.source_family != IPCC_SOURCE_FAMILY: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_ARTIFACT_SOURCE_FAMILY_MISMATCH", + message="artifact source_family must be ipcc_efdb.", + field_name="artifact.source_family", + ) + ) + if artifact.source_key != IPCC_SOURCE_KEY: + issues.append( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_ARTIFACT_SOURCE_KEY_MISMATCH", + message="artifact source_key must be ipcc_efdb.", + field_name="artifact.source_key", + ) + ) + + return IPCCSourceDownloadExecutionValidationResult(issues=tuple(issues)) + + +def _target_path(request: IPCCSourceDownloadExecutionRequest) -> Path: + return Path(request.target_root) / request.target_relative_path + + +def _validate_required_text( + value: str | None, + field_name: str, + code: str, + message: str, + issues: list[IPCCSourceDownloadExecutionIssue], +) -> None: + if not isinstance(value, str) or not value.strip(): + issues.append( + IPCCSourceDownloadExecutionIssue( + code=code, + message=message, + field_name=field_name, + ) + ) + + +def _validate_optional_text( + value: str | None, + field_name: str, + code: str, + message: str, + issues: list[IPCCSourceDownloadExecutionIssue], +) -> None: + if value is not None and (not isinstance(value, str) or not value.strip()): + issues.append( + IPCCSourceDownloadExecutionIssue( + code=code, + message=message, + field_name=field_name, + ) + ) + + +def _validate_optional_positive_int( + value: int | None, + field_name: str, + code: str, + message: str, + issues: list[IPCCSourceDownloadExecutionIssue], +) -> None: + if value is not None and ( + not isinstance(value, int) or isinstance(value, bool) or value <= 0 + ): + issues.append( + IPCCSourceDownloadExecutionIssue( + code=code, + message=message, + field_name=field_name, + ) + ) + + +def _validate_true( + value: bool, + field_name: str, + code: str, + message: str, + issues: list[IPCCSourceDownloadExecutionIssue], +) -> None: + if value is not True: + issues.append( + IPCCSourceDownloadExecutionIssue( + code=code, + message=message, + field_name=field_name, + ) + ) + + +def _validate_false( + value: bool, + field_name: str, + code: str, + message: str, + issues: list[IPCCSourceDownloadExecutionIssue], +) -> None: + if value is not False: + issues.append( + IPCCSourceDownloadExecutionIssue( + code=code, + message=message, + field_name=field_name, + ) + ) diff --git a/src/carbonfactor_parser/source_acquisition/phase1_ingestion_orchestrator.py b/src/carbonfactor_parser/source_acquisition/phase1_ingestion_orchestrator.py new file mode 100644 index 00000000..3f971e55 --- /dev/null +++ b/src/carbonfactor_parser/source_acquisition/phase1_ingestion_orchestrator.py @@ -0,0 +1,809 @@ +"""Explicit Phase 1 ingestion orchestrator with injected runtime boundaries.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Mapping, Protocol, Sequence, runtime_checkable + +from carbonfactor_parser.parsers.parser_run_contract import ( + ParserRunResult, + ParserRunStatus, +) +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + create_parser_normalized_output_batch, +) +from carbonfactor_parser.parsers.run_repository_contract import ( + ParserRunRepository, + ParserRunRepositoryPersistStatus, +) +from carbonfactor_parser.persistence.parsed_factor_persistence_writer import ( + ParsedFactorPersistenceStatus, + ParsedFactorPersistenceWriterResult, + persist_parsed_factor_records, +) +from carbonfactor_parser.persistence.postgresql_runtime_config_gate import ( + PostgreSQLRuntimeConfigGateDecision, +) +from carbonfactor_parser.persistence.postgresql_schema_bootstrap import ( + PostgreSQLSchemaBootstrapReport, +) +from carbonfactor_parser.persistence.source_document_repository import ( + SourceDocumentRepository, + SourceDocumentRepositoryPersistStatus, +) +from carbonfactor_parser.persistence.source_family_repository import ( + SourceFamilyRepository, +) +from carbonfactor_parser.source_acquisition.discovery_candidate_contract import ( + SourceDiscoveryCandidateResult, +) +from carbonfactor_parser.source_acquisition.models import ( + SourceAcquisitionPlanMode, + SourceDocumentChecksumStatus, + SourceDocumentPersistenceMappingStatus, + SourceDocumentPersistenceRecord, +) +from carbonfactor_parser.source_acquisition.phase1_observability import ( + emit_phase1_operational_event, + summarize_phase1_family_result_for_diagnostics, + summarize_phase1_orchestrator_request, + summarize_phase1_orchestrator_result_for_diagnostics, +) +from carbonfactor_parser.source_acquisition.run_contract import ( + SourceAcquisitionRunResult, + SourceAcquisitionRunStatus, +) +from carbonfactor_parser.source_acquisition.run_repository_contract import ( + SourceAcquisitionRunRepository, + SourceAcquisitionRunRepositoryPersistStatus, +) + + +PHASE1_SOURCE_FAMILIES = ("ghg_protocol", "defra_desnz", "ipcc_efdb") +_SOURCE_FAMILY_ALIASES: Mapping[str, str] = { + "ghg": "ghg_protocol", + "ghg_protocol": "ghg_protocol", + "defra": "defra_desnz", + "desnz": "defra_desnz", + "defra_desnz": "defra_desnz", + "ipcc": "ipcc_efdb", + "ipcc_efdb": "ipcc_efdb", +} + + +class Phase1IngestionExecutionMode(str, Enum): + """Execution modes exposed by the orchestrator.""" + + SEQUENTIAL = "sequential" + BOUNDED_PARALLEL = "bounded_parallel" + + +class Phase1IngestionRunStatus(str, Enum): + """Top-level Phase 1 ingestion run status.""" + + COMPLETED = "completed" + COMPLETED_WITH_FAILURES = "completed_with_failures" + FAILED = "failed" + NOT_EXECUTABLE = "not_executable" + + +class Phase1IngestionFamilyStatus(str, Enum): + """Per-source-family deterministic status.""" + + COMPLETED = "completed" + FAILED_DISCOVERY = "failed_discovery" + FAILED_DOWNLOAD = "failed_download" + FAILED_SOURCE_RUN_PERSISTENCE = "failed_source_run_persistence" + FAILED_SOURCE_DOCUMENT_PERSISTENCE = "failed_source_document_persistence" + FAILED_PARSER = "failed_parser" + FAILED_PARSER_RUN_PERSISTENCE = "failed_parser_run_persistence" + FAILED_PARSED_FACTOR_PERSISTENCE = "failed_parsed_factor_persistence" + + +@dataclass(frozen=True) +class Phase1IngestionFailureDetail: + """Structured failure detail recorded by the orchestrator.""" + + source_family: str | None + stage: str + code: str + message: str + field_name: str | None = None + severity: str = "error" + + +@dataclass(frozen=True) +class Phase1IngestionRunSummary: + """Success/failure counts for a Phase 1 ingestion run.""" + + requested_family_count: int + completed_family_count: int + failed_family_count: int + source_candidate_count: int + source_artifact_count: int + parser_run_count: int + parsed_factor_row_count: int + persisted_source_run_count: int + persisted_source_document_count: int + persisted_parser_run_count: int + persisted_master_count: int + persisted_detail_count: int + failure_count: int + + +@dataclass(frozen=True) +class Phase1SourceFamilyIngestionResult: + """Recorded state for one source-family ingestion execution.""" + + source_family: str + status: Phase1IngestionFamilyStatus + discovery_result: SourceDiscoveryCandidateResult | None = None + acquisition_result: SourceAcquisitionRunResult | None = None + parser_run_result: ParserRunResult | None = None + parsed_factor_persistence_result: ParsedFactorPersistenceWriterResult | None = None + persisted_source_run_count: int = 0 + persisted_source_document_count: int = 0 + persisted_parser_run_count: int = 0 + persisted_master_count: int = 0 + persisted_detail_count: int = 0 + failures: tuple[Phase1IngestionFailureDetail, ...] = () + + +@dataclass(frozen=True) +class Phase1IngestionOrchestratorRequest: + """Explicit request for Phase 1 source-family ingestion.""" + + source_families: tuple[str, ...] + run_id: str + correlation_id: str | None = None + execution_mode: Phase1IngestionExecutionMode = ( + Phase1IngestionExecutionMode.SEQUENTIAL + ) + max_parallelism: int = 1 + runtime_config_decision: PostgreSQLRuntimeConfigGateDecision | None = None + schema_bootstrap_report: PostgreSQLSchemaBootstrapReport | None = None + + +@dataclass(frozen=True) +class Phase1IngestionOrchestratorResult: + """End-to-end Phase 1 ingestion result.""" + + status: Phase1IngestionRunStatus + request: Phase1IngestionOrchestratorRequest + selected_source_families: tuple[str, ...] + family_results: tuple[Phase1SourceFamilyIngestionResult, ...] + summary: Phase1IngestionRunSummary + failures: tuple[Phase1IngestionFailureDetail, ...] = () + + +@runtime_checkable +class Phase1SourceFamilyRuntime(Protocol): + """Injected source-family runtime used by tests or reviewed adapters.""" + + def discover( + self, + source_family: str, + request: Phase1IngestionOrchestratorRequest, + ) -> SourceDiscoveryCandidateResult: + """Discover candidate source documents for one family.""" + + def download( + self, + source_family: str, + discovery_result: SourceDiscoveryCandidateResult, + request: Phase1IngestionOrchestratorRequest, + ) -> SourceAcquisitionRunResult: + """Download candidate documents for one family.""" + + def parse( + self, + source_family: str, + acquisition_result: SourceAcquisitionRunResult, + request: Phase1IngestionOrchestratorRequest, + ) -> ParserRunResult: + """Normalize downloaded artifacts into parser rows.""" + + +@dataclass(frozen=True) +class Phase1IngestionOrchestratorDependencies: + """Injected runtime dependencies for safe orchestration tests/adapters.""" + + source_runtimes: Mapping[str, Phase1SourceFamilyRuntime] + source_run_repository: SourceAcquisitionRunRepository + source_document_repository: SourceDocumentRepository + parser_run_repository: ParserRunRepository + parsed_factor_repository: SourceFamilyRepository + + +def run_phase1_ingestion_orchestrator( + request: Phase1IngestionOrchestratorRequest, + dependencies: Phase1IngestionOrchestratorDependencies, +) -> Phase1IngestionOrchestratorResult: + """Run Phase 1 ingestion sequentially with all runtime work injected.""" + + emit_phase1_operational_event( + "phase1_ingestion_orchestrator_started", + summarize_phase1_orchestrator_request(request), + ) + selected_families, request_failures = _normalize_source_families( + request.source_families, + ) + readiness_failures = ( + *request_failures, + *_execution_mode_failures(request), + *_postgresql_readiness_failures(request), + ) + if readiness_failures: + result = _not_executable_result(request, selected_families, readiness_failures) + emit_phase1_operational_event( + "phase1_ingestion_orchestrator_completed", + summarize_phase1_orchestrator_result_for_diagnostics(result), + ) + return result + + family_results: list[Phase1SourceFamilyIngestionResult] = [] + for source_family in selected_families: + emit_phase1_operational_event( + "phase1_source_family_started", + { + "correlation_id": request.correlation_id, + "run_id": request.run_id, + "source_family": source_family, + }, + ) + family_result = _run_source_family( + source_family=source_family, + request=request, + dependencies=dependencies, + ) + family_results.append(family_result) + emit_phase1_operational_event( + "phase1_source_family_completed", + summarize_phase1_family_result_for_diagnostics( + family_result, + run_id=request.run_id, + correlation_id=request.correlation_id, + ), + ) + + result = _create_result(request, selected_families, tuple(family_results)) + emit_phase1_operational_event( + "phase1_ingestion_orchestrator_completed", + summarize_phase1_orchestrator_result_for_diagnostics(result), + ) + return result + + +def _run_source_family( + *, + source_family: str, + request: Phase1IngestionOrchestratorRequest, + dependencies: Phase1IngestionOrchestratorDependencies, +) -> Phase1SourceFamilyIngestionResult: + runtime = dependencies.source_runtimes.get(source_family) + if runtime is None: + return _failed_family( + source_family, + Phase1IngestionFamilyStatus.FAILED_DISCOVERY, + _failure( + source_family, + "discovery", + "PHASE1_INGESTION_SOURCE_RUNTIME_MISSING", + "No source runtime is registered for the selected source family.", + "source_runtimes", + ), + ) + + try: + discovery_result = runtime.discover(source_family, request) + except Exception as exc: # noqa: BLE001 + return _failed_family( + source_family, + Phase1IngestionFamilyStatus.FAILED_DISCOVERY, + _exception_failure(source_family, "discovery", exc), + ) + + try: + acquisition_result = runtime.download(source_family, discovery_result, request) + except Exception as exc: # noqa: BLE001 + return _failed_family( + source_family, + Phase1IngestionFamilyStatus.FAILED_DOWNLOAD, + _exception_failure(source_family, "download", exc), + discovery_result=discovery_result, + ) + + if acquisition_result.status is SourceAcquisitionRunStatus.FAILED: + return _failed_family( + source_family, + Phase1IngestionFamilyStatus.FAILED_DOWNLOAD, + _failure( + source_family, + "download", + "PHASE1_INGESTION_SOURCE_ACQUISITION_FAILED", + "Source acquisition returned failed status.", + "acquisition_result.status", + ), + discovery_result=discovery_result, + acquisition_result=acquisition_result, + ) + + source_run_persist = dependencies.source_run_repository.persist_runs( + (acquisition_result,), + ) + if ( + source_run_persist.status + is SourceAcquisitionRunRepositoryPersistStatus.FAILED_VALIDATION + ): + return _failed_family( + source_family, + Phase1IngestionFamilyStatus.FAILED_SOURCE_RUN_PERSISTENCE, + _repository_failure( + source_family, + "source_run_persistence", + source_run_persist.issues, + ), + discovery_result=discovery_result, + acquisition_result=acquisition_result, + ) + + source_document_records = _source_document_records(acquisition_result, request) + source_document_persist = ( + dependencies.source_document_repository.persist_source_documents( + source_document_records, + ) + ) + if ( + source_document_persist.status + is SourceDocumentRepositoryPersistStatus.FAILED_VALIDATION + ): + return _failed_family( + source_family, + Phase1IngestionFamilyStatus.FAILED_SOURCE_DOCUMENT_PERSISTENCE, + _repository_failure( + source_family, + "source_document_persistence", + source_document_persist.issues, + ), + discovery_result=discovery_result, + acquisition_result=acquisition_result, + persisted_source_run_count=source_run_persist.persisted_count, + ) + + try: + parser_run_result = runtime.parse(source_family, acquisition_result, request) + except Exception as exc: # noqa: BLE001 + return _failed_family( + source_family, + Phase1IngestionFamilyStatus.FAILED_PARSER, + _exception_failure(source_family, "parser", exc), + discovery_result=discovery_result, + acquisition_result=acquisition_result, + persisted_source_run_count=source_run_persist.persisted_count, + persisted_source_document_count=source_document_persist.persisted_count, + ) + + if parser_run_result.status is ParserRunStatus.FAILED: + return _failed_family( + source_family, + Phase1IngestionFamilyStatus.FAILED_PARSER, + _failure( + source_family, + "parser", + "PHASE1_INGESTION_PARSER_FAILED", + "Parser run returned failed status.", + "parser_run_result.status", + ), + discovery_result=discovery_result, + acquisition_result=acquisition_result, + parser_run_result=parser_run_result, + persisted_source_run_count=source_run_persist.persisted_count, + persisted_source_document_count=source_document_persist.persisted_count, + ) + + parser_run_persist = dependencies.parser_run_repository.persist_runs( + (parser_run_result,), + ) + if parser_run_persist.status is ParserRunRepositoryPersistStatus.FAILED_VALIDATION: + return _failed_family( + source_family, + Phase1IngestionFamilyStatus.FAILED_PARSER_RUN_PERSISTENCE, + _repository_failure( + source_family, + "parser_run_persistence", + parser_run_persist.issues, + ), + discovery_result=discovery_result, + acquisition_result=acquisition_result, + parser_run_result=parser_run_result, + persisted_source_run_count=source_run_persist.persisted_count, + persisted_source_document_count=source_document_persist.persisted_count, + ) + + parsed_factor_persist = persist_parsed_factor_records( + create_parser_normalized_output_batch(parser_run_result.rows), + dependencies.parsed_factor_repository, + source_document_id=( + source_document_records[0].source_document_id + if source_document_records + else None + ), + ) + if parsed_factor_persist.status is ParsedFactorPersistenceStatus.FAILED_VALIDATION: + return _failed_family( + source_family, + Phase1IngestionFamilyStatus.FAILED_PARSED_FACTOR_PERSISTENCE, + _parsed_factor_failure(source_family, parsed_factor_persist), + discovery_result=discovery_result, + acquisition_result=acquisition_result, + parser_run_result=parser_run_result, + parsed_factor_persistence_result=parsed_factor_persist, + persisted_source_run_count=source_run_persist.persisted_count, + persisted_source_document_count=source_document_persist.persisted_count, + persisted_parser_run_count=parser_run_persist.persisted_count, + ) + + return Phase1SourceFamilyIngestionResult( + source_family=source_family, + status=Phase1IngestionFamilyStatus.COMPLETED, + discovery_result=discovery_result, + acquisition_result=acquisition_result, + parser_run_result=parser_run_result, + parsed_factor_persistence_result=parsed_factor_persist, + persisted_source_run_count=source_run_persist.persisted_count, + persisted_source_document_count=source_document_persist.persisted_count, + persisted_parser_run_count=parser_run_persist.persisted_count, + persisted_master_count=parsed_factor_persist.persisted_master_count, + persisted_detail_count=parsed_factor_persist.persisted_detail_count, + ) + + +def _normalize_source_families( + source_families: Sequence[str], +) -> tuple[tuple[str, ...], tuple[Phase1IngestionFailureDetail, ...]]: + selected: list[str] = [] + failures: list[Phase1IngestionFailureDetail] = [] + for position, value in enumerate(source_families): + normalized = _SOURCE_FAMILY_ALIASES.get(value) + if normalized is None: + failures.append( + _failure( + None, + "selection", + "PHASE1_INGESTION_UNSUPPORTED_SOURCE_FAMILY", + "Source family selection must be one of the Phase 1 families.", + f"source_families[{position}]", + ) + ) + continue + if normalized not in selected: + selected.append(normalized) + + if not selected: + failures.append( + _failure( + None, + "selection", + "PHASE1_INGESTION_EMPTY_SOURCE_FAMILY_SELECTION", + "At least one Phase 1 source family must be selected explicitly.", + "source_families", + ) + ) + return tuple(selected), tuple(failures) + + +def _execution_mode_failures( + request: Phase1IngestionOrchestratorRequest, +) -> tuple[Phase1IngestionFailureDetail, ...]: + if request.execution_mode is Phase1IngestionExecutionMode.SEQUENTIAL: + if request.max_parallelism != 1: + return ( + _failure( + None, + "execution_mode", + "PHASE1_INGESTION_SEQUENTIAL_MAX_PARALLELISM_MUST_BE_ONE", + "Sequential execution must use max_parallelism=1.", + "max_parallelism", + ), + ) + return () + + return ( + _failure( + None, + "execution_mode", + "PHASE1_INGESTION_BOUNDED_PARALLEL_NOT_ENABLED", + "Bounded parallel execution is a declared extension point only.", + "execution_mode", + ), + ) + + +def _postgresql_readiness_failures( + request: Phase1IngestionOrchestratorRequest, +) -> tuple[Phase1IngestionFailureDetail, ...]: + failures: list[Phase1IngestionFailureDetail] = [] + if ( + request.runtime_config_decision is not None + and not request.runtime_config_decision.runtime_enabled + ): + issue = ( + request.runtime_config_decision.issues[0] + if request.runtime_config_decision.issues + else None + ) + failures.append( + _failure( + None, + "postgresql_runtime_config", + ( + issue.code + if issue is not None + else "PHASE1_INGESTION_POSTGRESQL_RUNTIME_NOT_READY" + ), + ( + issue.message + if issue is not None + else "PostgreSQL runtime configuration is not ready." + ), + issue.field_name if issue is not None else "runtime_config_decision", + ) + ) + + if ( + request.schema_bootstrap_report is not None + and request.schema_bootstrap_report.fail_on_missing + and request.schema_bootstrap_report.missing_table_names + ): + failures.append( + _failure( + None, + "postgresql_schema_bootstrap", + "PHASE1_INGESTION_POSTGRESQL_SCHEMA_NOT_READY", + "PostgreSQL schema bootstrap reported missing required tables.", + "schema_bootstrap_report.missing_table_names", + ) + ) + return tuple(failures) + + +def _source_document_records( + acquisition_result: SourceAcquisitionRunResult, + request: Phase1IngestionOrchestratorRequest, +) -> tuple[SourceDocumentPersistenceRecord, ...]: + return tuple( + SourceDocumentPersistenceRecord( + source_document_id=( + f"{request.run_id}_{artifact.source_family}_{artifact.artifact_id}" + ), + ingestion_run_id=request.run_id, + source_family=artifact.source_family, + source_document_uri=artifact.source_reference_uri, + source_checksum_sha256=artifact.checksum_sha256, + checksum_status=SourceDocumentChecksumStatus.DRY_RUN_UNAVAILABLE, + acquisition_status=SourceDocumentPersistenceMappingStatus.DRY_RUN_MAPPED, + acquired_at=None, + created_at="runtime_timestamp_unavailable", + updated_at="runtime_timestamp_unavailable", + logical_document_name=artifact.display_name or artifact.artifact_id, + target_logical_path=artifact.local_reference, + mode=SourceAcquisitionPlanMode.DRY_RUN, + ) + for artifact in acquisition_result.artifacts + ) + + +def _create_result( + request: Phase1IngestionOrchestratorRequest, + selected_source_families: tuple[str, ...], + family_results: tuple[Phase1SourceFamilyIngestionResult, ...], +) -> Phase1IngestionOrchestratorResult: + failures = tuple( + failure for result in family_results for failure in result.failures + ) + completed_count = sum( + 1 + for result in family_results + if result.status is Phase1IngestionFamilyStatus.COMPLETED + ) + status = ( + Phase1IngestionRunStatus.COMPLETED + if completed_count == len(family_results) + else Phase1IngestionRunStatus.COMPLETED_WITH_FAILURES + if completed_count > 0 + else Phase1IngestionRunStatus.FAILED + ) + return Phase1IngestionOrchestratorResult( + status=status, + request=request, + selected_source_families=selected_source_families, + family_results=family_results, + summary=_summary(request, family_results, failures), + failures=failures, + ) + + +def _not_executable_result( + request: Phase1IngestionOrchestratorRequest, + selected_source_families: tuple[str, ...], + failures: tuple[Phase1IngestionFailureDetail, ...], +) -> Phase1IngestionOrchestratorResult: + return Phase1IngestionOrchestratorResult( + status=Phase1IngestionRunStatus.NOT_EXECUTABLE, + request=request, + selected_source_families=selected_source_families, + family_results=(), + summary=Phase1IngestionRunSummary( + requested_family_count=len(selected_source_families), + completed_family_count=0, + failed_family_count=len(selected_source_families), + source_candidate_count=0, + source_artifact_count=0, + parser_run_count=0, + parsed_factor_row_count=0, + persisted_source_run_count=0, + persisted_source_document_count=0, + persisted_parser_run_count=0, + persisted_master_count=0, + persisted_detail_count=0, + failure_count=len(failures), + ), + failures=failures, + ) + + +def _summary( + request: Phase1IngestionOrchestratorRequest, + family_results: tuple[Phase1SourceFamilyIngestionResult, ...], + failures: tuple[Phase1IngestionFailureDetail, ...], +) -> Phase1IngestionRunSummary: + return Phase1IngestionRunSummary( + requested_family_count=len(request.source_families), + completed_family_count=sum( + 1 + for result in family_results + if result.status is Phase1IngestionFamilyStatus.COMPLETED + ), + failed_family_count=sum( + 1 + for result in family_results + if result.status is not Phase1IngestionFamilyStatus.COMPLETED + ), + source_candidate_count=sum( + len(result.discovery_result.candidates) + for result in family_results + if result.discovery_result is not None + ), + source_artifact_count=sum( + len(result.acquisition_result.artifacts) + for result in family_results + if result.acquisition_result is not None + ), + parser_run_count=sum( + 1 for result in family_results if result.parser_run_result is not None + ), + parsed_factor_row_count=sum( + len(result.parser_run_result.rows) + for result in family_results + if result.parser_run_result is not None + ), + persisted_source_run_count=sum( + result.persisted_source_run_count for result in family_results + ), + persisted_source_document_count=sum( + result.persisted_source_document_count for result in family_results + ), + persisted_parser_run_count=sum( + result.persisted_parser_run_count for result in family_results + ), + persisted_master_count=sum( + result.persisted_master_count for result in family_results + ), + persisted_detail_count=sum( + result.persisted_detail_count for result in family_results + ), + failure_count=len(failures), + ) + + +def _failed_family( + source_family: str, + status: Phase1IngestionFamilyStatus, + failure: Phase1IngestionFailureDetail, + *, + discovery_result: SourceDiscoveryCandidateResult | None = None, + acquisition_result: SourceAcquisitionRunResult | None = None, + parser_run_result: ParserRunResult | None = None, + parsed_factor_persistence_result: ParsedFactorPersistenceWriterResult | None = None, + persisted_source_run_count: int = 0, + persisted_source_document_count: int = 0, + persisted_parser_run_count: int = 0, +) -> Phase1SourceFamilyIngestionResult: + return Phase1SourceFamilyIngestionResult( + source_family=source_family, + status=status, + discovery_result=discovery_result, + acquisition_result=acquisition_result, + parser_run_result=parser_run_result, + parsed_factor_persistence_result=parsed_factor_persistence_result, + persisted_source_run_count=persisted_source_run_count, + persisted_source_document_count=persisted_source_document_count, + persisted_parser_run_count=persisted_parser_run_count, + failures=(failure,), + ) + + +def _failure( + source_family: str | None, + stage: str, + code: str, + message: str, + field_name: str | None = None, +) -> Phase1IngestionFailureDetail: + return Phase1IngestionFailureDetail( + source_family=source_family, + stage=stage, + code=code, + message=message, + field_name=field_name, + ) + + +def _exception_failure( + source_family: str, + stage: str, + exc: Exception, +) -> Phase1IngestionFailureDetail: + return _failure( + source_family, + stage, + f"PHASE1_INGESTION_{stage.upper()}_EXCEPTION", + str(exc) or exc.__class__.__name__, + stage, + ) + + +def _repository_failure( + source_family: str, + stage: str, + issues: Sequence[object], +) -> Phase1IngestionFailureDetail: + if issues: + issue = issues[0] + return _failure( + source_family, + stage, + str(getattr(issue, "code", "PHASE1_INGESTION_REPOSITORY_FAILED")), + str(getattr(issue, "message", "Repository persistence failed.")), + getattr(issue, "field_name", None), + ) + return _failure( + source_family, + stage, + "PHASE1_INGESTION_REPOSITORY_FAILED", + "Repository persistence failed without structured issues.", + stage, + ) + + +def _parsed_factor_failure( + source_family: str, + result: ParsedFactorPersistenceWriterResult, +) -> Phase1IngestionFailureDetail: + if result.issues: + issue = result.issues[0] + return _failure( + source_family, + "parsed_factor_persistence", + issue.code, + issue.message, + issue.field_name, + ) + return _failure( + source_family, + "parsed_factor_persistence", + "PHASE1_INGESTION_PARSED_FACTOR_PERSISTENCE_FAILED", + "Parsed factor persistence failed without structured issues.", + "parsed_factor_persistence", + ) diff --git a/src/carbonfactor_parser/source_acquisition/phase1_observability.py b/src/carbonfactor_parser/source_acquisition/phase1_observability.py new file mode 100644 index 00000000..0c058946 --- /dev/null +++ b/src/carbonfactor_parser/source_acquisition/phase1_observability.py @@ -0,0 +1,355 @@ +"""Structured Phase 1 observability helpers with safe diagnostic output.""" + +from __future__ import annotations + +from dataclasses import asdict, is_dataclass +import json +import logging +import re +from typing import Any, Mapping + +from carbonfactor_parser.persistence.postgresql_options import ( + PostgreSQLPersistenceOptions, +) + + +PHASE1_OPERATIONAL_LOGGER_NAME = "carbonfactor_parser.phase1" +REDACTED = "" + +_CHECKSUM_PATTERN = re.compile(r"^[0-9a-fA-F]{64}$") +_USERINFO_URI_PATTERN = re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://)[^\s/?#]*@") +_SENSITIVE_ASSIGNMENT_PATTERN = re.compile( + r"(?i)\b(password|passwd|pwd|secret|token|dsn|connection[_-]?string)=([^\s;,]+)", +) +_SENSITIVE_KEY_PARTS = ( + "password", + "passwd", + "pwd", + "secret", + "token", + "credential", + "dsn", + "connection_string", + "connectionstring", + "connection_uri", + "connectionuri", + "database_url", + "databaseurl", + "api_key", + "apikey", + "access_key", + "accesskey", + "private_key", + "privatekey", +) +_SENSITIVE_RUNTIME_OPTION_FIELDS = frozenset( + { + "host", + "database", + "username", + "application_name", + "dsn", + "connection_string", + "connection_uri", + "database_url", + } +) + + +def get_phase1_operational_logger() -> logging.Logger: + """Return the shared Phase 1 operational logger.""" + + return logging.getLogger(PHASE1_OPERATIONAL_LOGGER_NAME) + + +def emit_phase1_operational_event( + event_name: str, + payload: Mapping[str, Any], + *, + logger: logging.Logger | None = None, + level: int = logging.INFO, +) -> dict[str, Any]: + """Emit one deterministic JSON operational event and return its payload.""" + + safe_payload = redact_diagnostic_value("payload", payload) + event = _stable_mapping( + { + "event": event_name, + **safe_payload, + } + ) + active_logger = logger or get_phase1_operational_logger() + active_logger.log( + level, + json.dumps(event, sort_keys=True, separators=(",", ":")), + ) + return event + + +def summarize_postgresql_options_for_diagnostics( + options: PostgreSQLPersistenceOptions, +) -> dict[str, Any]: + """Return PostgreSQL option metadata without runtime-sensitive values.""" + + return { + "application_name": REDACTED if options.application_name is not None else None, + "connect_timeout_seconds": options.connect_timeout_seconds, + "database": REDACTED, + "host": REDACTED, + "password_set": options.password_set, + "port": options.port, + "ssl_mode": options.ssl_mode, + "username": REDACTED, + } + + +def summarize_phase1_orchestrator_request(request: Any) -> dict[str, Any]: + """Return correlation-safe request metadata for Phase 1 diagnostics.""" + + return { + "correlation_id": _safe_text(getattr(request, "correlation_id", None)), + "execution_mode": _enum_value(getattr(request, "execution_mode", None)), + "max_degree_of_parallelism": getattr(request, "max_parallelism", None), + "run_id": _safe_text(getattr(request, "run_id", None)), + "source_families": tuple(getattr(request, "source_families", ())), + } + + +def summarize_phase1_family_result_for_diagnostics( + family_result: Any, + *, + run_id: str, + correlation_id: str | None = None, +) -> dict[str, Any]: + """Return deterministic per-source-family operational counts and IDs.""" + + acquisition_result = getattr(family_result, "acquisition_result", None) + parser_run_result = getattr(family_result, "parser_run_result", None) + + return { + "correlation_id": _safe_text(correlation_id), + "documents": _document_summaries(acquisition_result, run_id), + "failures": _failure_summaries(getattr(family_result, "failures", ())), + "parser": { + "accepted_row_count": _nested_attr( + parser_run_result, + "summary", + "row_count", + 0, + ), + "failure_count": _nested_attr( + parser_run_result, + "summary", + "error_count", + 0, + ), + "result_status": _enum_value(getattr(parser_run_result, "status", None)), + "run_id": _safe_text(getattr(parser_run_result, "run_id", None)), + "validation_issue_count": _nested_attr( + parser_run_result, + "summary", + "issue_count", + 0, + ), + }, + "persistence": { + "parsed_factor_detail_count": getattr( + family_result, + "persisted_detail_count", + 0, + ), + "parsed_factor_master_count": getattr( + family_result, + "persisted_master_count", + 0, + ), + "parser_run_count": getattr(family_result, "persisted_parser_run_count", 0), + "source_document_count": getattr( + family_result, + "persisted_source_document_count", + 0, + ), + "source_run_count": getattr(family_result, "persisted_source_run_count", 0), + }, + "run_id": _safe_text(run_id), + "source_family": getattr(family_result, "source_family", None), + "source_key": _source_key(family_result), + "status": _enum_value(getattr(family_result, "status", None)), + } + + +def summarize_phase1_orchestrator_result_for_diagnostics(result: Any) -> dict[str, Any]: + """Return deterministic run-level operational counts and failures.""" + + request = getattr(result, "request", None) + summary = getattr(result, "summary", None) + return { + "correlation_id": _safe_text(getattr(request, "correlation_id", None)), + "failures": _failure_summaries(getattr(result, "failures", ())), + "run_id": _safe_text(getattr(request, "run_id", None)), + "selected_source_families": tuple( + getattr(result, "selected_source_families", ()), + ), + "source_family_statuses": tuple( + { + "source_family": getattr(family_result, "source_family", None), + "status": _enum_value(getattr(family_result, "status", None)), + } + for family_result in getattr(result, "family_results", ()) + ), + "status": _enum_value(getattr(result, "status", None)), + "summary": _dataclass_or_mapping(summary), + } + + +def redact_diagnostic_value(field_name: str, value: Any) -> Any: + """Redact sensitive diagnostic values while preserving deterministic shape.""" + + if _is_sensitive_field(field_name): + return REDACTED if value is not None else None + if isinstance(value, str): + return _safe_text(value) + if isinstance(value, Mapping): + return { + str(key): redact_diagnostic_value(str(key), item) + for key, item in sorted(value.items(), key=lambda item: str(item[0])) + } + if isinstance(value, tuple): + return tuple(redact_diagnostic_value(field_name, item) for item in value) + if isinstance(value, list): + return tuple(redact_diagnostic_value(field_name, item) for item in value) + return value + + +def _document_summaries( + acquisition_result: Any, + run_id: str, +) -> tuple[dict[str, Any], ...]: + if acquisition_result is None: + return () + return tuple( + { + "checksum_sha256": _safe_checksum( + getattr(artifact, "checksum_sha256", None), + ), + "document_id": _safe_text(getattr(artifact, "artifact_id", None)), + "source_family": getattr(artifact, "source_family", None), + "source_key": getattr(artifact, "source_key", None), + } + for artifact in getattr(acquisition_result, "artifacts", ()) + ) + + +def _failure_summaries(failures: Any) -> tuple[dict[str, Any], ...]: + return tuple( + { + "code": getattr(failure, "code", None), + "field_name": getattr(failure, "field_name", None), + "message": _safe_text(getattr(failure, "message", None)), + "severity": getattr(failure, "severity", None), + "source_family": getattr(failure, "source_family", None), + "source_key": _failure_source_key(failure), + "stage": getattr(failure, "stage", None), + } + for failure in failures + ) + + +def _dataclass_or_mapping(value: Any) -> dict[str, Any]: + if value is None: + return {} + if is_dataclass(value): + return _stable_mapping(asdict(value)) + if isinstance(value, Mapping): + return _stable_mapping(value) + return {} + + +def _stable_mapping(value: Mapping[str, Any]) -> dict[str, Any]: + return { + str(key): _stable_value(item) + for key, item in sorted(value.items(), key=lambda item: str(item[0])) + } + + +def _stable_value(value: Any) -> Any: + if isinstance(value, Mapping): + return _stable_mapping(value) + if is_dataclass(value): + return _stable_mapping(asdict(value)) + if isinstance(value, list): + return tuple(_stable_value(item) for item in value) + if isinstance(value, tuple): + return tuple(_stable_value(item) for item in value) + if hasattr(value, "value"): + return value.value + return value + + +def _safe_text(value: Any) -> Any: + if not isinstance(value, str): + return value + without_userinfo = _USERINFO_URI_PATTERN.sub( + lambda match: f"{match.group(1)}{REDACTED}@", + value, + ) + return _SENSITIVE_ASSIGNMENT_PATTERN.sub( + lambda match: f"{match.group(1)}={REDACTED}", + without_userinfo, + ) + + +def _safe_checksum(value: Any) -> str | None: + if not isinstance(value, str) or not _CHECKSUM_PATTERN.match(value): + return None + return value.lower() + + +def _is_sensitive_field(field_name: str) -> bool: + normalized = field_name.strip().lower() + compact = normalized.replace("_", "").replace("-", "") + if normalized == "password_set": + return False + return ( + normalized in _SENSITIVE_RUNTIME_OPTION_FIELDS + or any(part in normalized or part in compact for part in _SENSITIVE_KEY_PARTS) + ) + + +def _enum_value(value: Any) -> Any: + return value.value if hasattr(value, "value") else value + + +def _nested_attr(value: Any, first: str, second: str, default: Any) -> Any: + nested = getattr(value, first, None) + return getattr(nested, second, default) + + +def _source_key(family_result: Any) -> Any: + source_key = getattr(family_result, "source_key", None) + if source_key is not None: + return source_key + acquisition_result = getattr(family_result, "acquisition_result", None) + if acquisition_result is not None: + return getattr(acquisition_result, "source_key", None) + return getattr(family_result, "source_family", None) + + +def _failure_source_key(failure: Any) -> Any: + source_key = getattr(failure, "source_key", None) + if source_key is not None: + return source_key + return getattr(failure, "source_family", None) + + +__all__ = ( + "PHASE1_OPERATIONAL_LOGGER_NAME", + "REDACTED", + "emit_phase1_operational_event", + "get_phase1_operational_logger", + "redact_diagnostic_value", + "summarize_phase1_family_result_for_diagnostics", + "summarize_phase1_orchestrator_request", + "summarize_phase1_orchestrator_result_for_diagnostics", + "summarize_postgresql_options_for_diagnostics", +) diff --git a/src/carbonfactor_parser/source_acquisition/phase1_service_host.py b/src/carbonfactor_parser/source_acquisition/phase1_service_host.py new file mode 100644 index 00000000..a0dc298f --- /dev/null +++ b/src/carbonfactor_parser/source_acquisition/phase1_service_host.py @@ -0,0 +1,500 @@ +"""Python service host boundary for scheduled Phase 1 ingestion.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from enum import Enum +from threading import Lock + +from carbonfactor_parser.persistence.postgresql_options import ( + PostgreSQLPersistenceOptions, + validate_postgresql_persistence_options, +) +from carbonfactor_parser.persistence.postgresql_runtime_config_gate import ( + PostgreSQLRuntimeConfigGateDecision, +) +from carbonfactor_parser.persistence.postgresql_schema_bootstrap import ( + PostgreSQLSchemaBootstrapMode, + PostgreSQLSchemaBootstrapReport, + build_postgresql_phase1_schema_bootstrap_report, +) +from carbonfactor_parser.source_acquisition.phase1_ingestion_orchestrator import ( + Phase1IngestionExecutionMode, + Phase1IngestionOrchestratorRequest, + Phase1IngestionOrchestratorResult, +) +from carbonfactor_parser.source_acquisition.phase1_observability import ( + emit_phase1_operational_event, + summarize_phase1_orchestrator_result_for_diagnostics, + summarize_postgresql_options_for_diagnostics, +) + + +_SOURCE_FAMILY_ALIASES = { + "ghg", + "ghg_protocol", + "defra", + "desnz", + "defra_desnz", + "ipcc", + "ipcc_efdb", +} + + +class Phase1ServiceHostStatus(str, Enum): + """Lifecycle status values for the service host boundary.""" + + CREATED = "created" + READY = "ready" + BLOCKED = "blocked" + RUNNING = "running" + SHUTDOWN_REQUESTED = "shutdown_requested" + STOPPED = "stopped" + + +class Phase1ScheduledRunStatus(str, Enum): + """Scheduled run trigger status values.""" + + STARTED = "started" + SKIPPED_NOT_STARTED = "skipped_not_started" + SKIPPED_ALREADY_RUNNING = "skipped_already_running" + SKIPPED_SHUTTING_DOWN = "skipped_shutting_down" + + +@dataclass(frozen=True) +class Phase1ServiceHostConfig: + """Required startup configuration for scheduled Phase 1 ingestion.""" + + source_families: tuple[str, ...] + postgresql_options: PostgreSQLPersistenceOptions + run_id_prefix: str = "phase1-scheduled" + schedule_interval_seconds: int = 3600 + execution_mode: Phase1IngestionExecutionMode = ( + Phase1IngestionExecutionMode.SEQUENTIAL + ) + max_parallelism: int = 1 + schema_bootstrap_mode: PostgreSQLSchemaBootstrapMode = ( + PostgreSQLSchemaBootstrapMode.CHECK_ONLY + ) + fail_on_missing_schema: bool = True + runtime_config_decision: PostgreSQLRuntimeConfigGateDecision | None = None + + +@dataclass(frozen=True) +class Phase1ServiceHostIssue: + """Validation or lifecycle issue reported by the service host.""" + + code: str + message: str + field_name: str | None = None + severity: str = "error" + + +@dataclass(frozen=True) +class Phase1ServiceHostStartupResult: + """Startup validation and bootstrap check result.""" + + status: Phase1ServiceHostStatus + issues: tuple[Phase1ServiceHostIssue, ...] + schema_bootstrap_report: PostgreSQLSchemaBootstrapReport | None = None + + @property + def is_ready(self) -> bool: + return self.status is Phase1ServiceHostStatus.READY + + +@dataclass(frozen=True) +class Phase1ScheduledRunResult: + """Result of one scheduled trigger attempt.""" + + status: Phase1ScheduledRunStatus + run_id: str | None = None + orchestrator_result: Phase1IngestionOrchestratorResult | None = None + issues: tuple[Phase1ServiceHostIssue, ...] = () + + +Phase1SchemaBootstrapChecker = Callable[ + [PostgreSQLSchemaBootstrapMode, bool], + PostgreSQLSchemaBootstrapReport, +] +Phase1OrchestratorRunner = Callable[ + [Phase1IngestionOrchestratorRequest], + Phase1IngestionOrchestratorResult, +] + + +class Phase1ScheduledIngestionServiceHost: + """Synchronous service host boundary for scheduled Phase 1 runs.""" + + def __init__( + self, + config: Phase1ServiceHostConfig, + *, + schema_bootstrap_checker: Phase1SchemaBootstrapChecker | None = None, + orchestrator_runner: Phase1OrchestratorRunner, + ) -> None: + self.config = config + self._schema_bootstrap_checker = ( + schema_bootstrap_checker or _default_schema_bootstrap_checker + ) + self._orchestrator_runner = orchestrator_runner + self._lock = Lock() + self._status = Phase1ServiceHostStatus.CREATED + self._startup_result: Phase1ServiceHostStartupResult | None = None + self._schema_bootstrap_report: PostgreSQLSchemaBootstrapReport | None = None + self._run_in_progress = False + self._shutdown_requested = False + self._run_sequence = 0 + + @property + def status(self) -> Phase1ServiceHostStatus: + return self._status + + def start(self) -> Phase1ServiceHostStartupResult: + """Validate config and check schema readiness before scheduled runs.""" + + emit_phase1_operational_event( + "phase1_service_host_starting", + { + "postgresql_options": summarize_postgresql_options_for_diagnostics( + self.config.postgresql_options, + ), + "run_id_prefix": self.config.run_id_prefix, + "schedule_interval_seconds": self.config.schedule_interval_seconds, + "source_families": self.config.source_families, + }, + ) + issues = list(validate_phase1_service_host_config(self.config)) + schema_report = None + if not issues: + schema_report = self._schema_bootstrap_checker( + self.config.schema_bootstrap_mode, + self.config.fail_on_missing_schema, + ) + issues.extend(_schema_bootstrap_issues(schema_report)) + + status = ( + Phase1ServiceHostStatus.BLOCKED + if issues + else Phase1ServiceHostStatus.READY + ) + result = Phase1ServiceHostStartupResult( + status=status, + issues=tuple(issues), + schema_bootstrap_report=schema_report, + ) + + with self._lock: + if self._shutdown_requested: + self._status = Phase1ServiceHostStatus.STOPPED + self._startup_result = Phase1ServiceHostStartupResult( + status=Phase1ServiceHostStatus.BLOCKED, + issues=( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_SHUTDOWN_REQUESTED", + message="Service host startup is blocked after shutdown.", + field_name="status", + ), + ), + schema_bootstrap_report=schema_report, + ) + emit_phase1_operational_event( + "phase1_service_host_started", + _startup_diagnostic_payload(self._startup_result), + ) + return self._startup_result + + self._status = status + self._startup_result = result + self._schema_bootstrap_report = schema_report + + emit_phase1_operational_event( + "phase1_service_host_started", + _startup_diagnostic_payload(result), + ) + return result + + def trigger_scheduled_run(self) -> Phase1ScheduledRunResult: + """Trigger one scheduled Phase 1 orchestrator run if host is ready.""" + + with self._lock: + if self._shutdown_requested: + result = Phase1ScheduledRunResult( + status=Phase1ScheduledRunStatus.SKIPPED_SHUTTING_DOWN, + issues=(_shutdown_issue(),), + ) + emit_phase1_operational_event( + "phase1_service_host_scheduled_run_skipped", + _scheduled_run_diagnostic_payload(result), + ) + return result + if self._run_in_progress: + result = Phase1ScheduledRunResult( + status=Phase1ScheduledRunStatus.SKIPPED_ALREADY_RUNNING, + issues=( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_RUN_ALREADY_IN_PROGRESS", + message="Scheduled trigger skipped while a run is active.", + field_name="run_in_progress", + ), + ), + ) + emit_phase1_operational_event( + "phase1_service_host_scheduled_run_skipped", + _scheduled_run_diagnostic_payload(result), + ) + return result + if self._status is not Phase1ServiceHostStatus.READY: + result = Phase1ScheduledRunResult( + status=Phase1ScheduledRunStatus.SKIPPED_NOT_STARTED, + issues=( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_NOT_READY", + message="Service host must start successfully first.", + field_name="status", + ), + ), + ) + emit_phase1_operational_event( + "phase1_service_host_scheduled_run_skipped", + _scheduled_run_diagnostic_payload(result), + ) + return result + + self._run_in_progress = True + self._status = Phase1ServiceHostStatus.RUNNING + self._run_sequence += 1 + run_id = f"{self.config.run_id_prefix}-{self._run_sequence:06d}" + request = self._build_orchestrator_request(run_id) + + emit_phase1_operational_event( + "phase1_service_host_scheduled_run_started", + { + "correlation_id": request.correlation_id, + "run_id": run_id, + "source_families": request.source_families, + }, + ) + try: + orchestrator_result = self._orchestrator_runner(request) + finally: + with self._lock: + self._run_in_progress = False + self._status = ( + Phase1ServiceHostStatus.STOPPED + if self._shutdown_requested + else Phase1ServiceHostStatus.READY + ) + + result = Phase1ScheduledRunResult( + status=Phase1ScheduledRunStatus.STARTED, + run_id=run_id, + orchestrator_result=orchestrator_result, + ) + emit_phase1_operational_event( + "phase1_service_host_scheduled_run_completed", + _scheduled_run_diagnostic_payload(result), + ) + return result + + def request_shutdown(self) -> Phase1ServiceHostStatus: + """Request graceful shutdown without interrupting an active run.""" + + with self._lock: + self._shutdown_requested = True + if self._run_in_progress: + self._status = Phase1ServiceHostStatus.SHUTDOWN_REQUESTED + else: + self._status = Phase1ServiceHostStatus.STOPPED + return self._status + + def _build_orchestrator_request( + self, + run_id: str, + ) -> Phase1IngestionOrchestratorRequest: + return Phase1IngestionOrchestratorRequest( + source_families=self.config.source_families, + run_id=run_id, + execution_mode=self.config.execution_mode, + max_parallelism=self.config.max_parallelism, + runtime_config_decision=self.config.runtime_config_decision, + schema_bootstrap_report=self._schema_bootstrap_report, + ) + + +def validate_phase1_service_host_config( + config: Phase1ServiceHostConfig, +) -> tuple[Phase1ServiceHostIssue, ...]: + """Validate required service-host startup configuration.""" + + issues: list[Phase1ServiceHostIssue] = [] + if not config.source_families: + issues.append( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_MISSING_SOURCE_FAMILIES", + message="At least one Phase 1 source family must be configured.", + field_name="source_families", + ) + ) + for index, source_family in enumerate(config.source_families): + if source_family not in _SOURCE_FAMILY_ALIASES: + issues.append( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_UNSUPPORTED_SOURCE_FAMILY", + message="Configured source family must be a Phase 1 family.", + field_name=f"source_families[{index}]", + ) + ) + + if not config.run_id_prefix.strip(): + issues.append( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_MISSING_RUN_ID_PREFIX", + message="run_id_prefix must be a non-empty string.", + field_name="run_id_prefix", + ) + ) + + if ( + isinstance(config.schedule_interval_seconds, bool) + or not isinstance(config.schedule_interval_seconds, int) + or config.schedule_interval_seconds <= 0 + ): + issues.append( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_INVALID_SCHEDULE_INTERVAL", + message="schedule_interval_seconds must be a positive integer.", + field_name="schedule_interval_seconds", + ) + ) + + if config.execution_mode is Phase1IngestionExecutionMode.SEQUENTIAL: + if config.max_parallelism != 1: + issues.append( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_INVALID_SEQUENTIAL_PARALLELISM", + message="Sequential scheduled execution requires max_parallelism=1.", + field_name="max_parallelism", + ) + ) + else: + issues.append( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_UNSUPPORTED_EXECUTION_MODE", + message="Only sequential scheduled execution is enabled.", + field_name="execution_mode", + ) + ) + + options_result = validate_postgresql_persistence_options( + config.postgresql_options, + ) + for option_issue in options_result.issues: + issues.append( + Phase1ServiceHostIssue( + code=option_issue.code, + message=option_issue.message, + field_name=f"postgresql_options.{option_issue.field_name}", + severity=option_issue.severity, + ) + ) + if not config.postgresql_options.password_set: + issues.append( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_POSTGRESQL_PASSWORD_NOT_CONFIRMED", + message=( + "postgresql_options.password_set must confirm that a " + "credential is available outside this config object." + ), + field_name="postgresql_options.password_set", + ) + ) + + return tuple(issues) + + +def _schema_bootstrap_issues( + report: PostgreSQLSchemaBootstrapReport, +) -> tuple[Phase1ServiceHostIssue, ...]: + if not report.fail_on_missing or not report.missing_table_names: + return () + return ( + Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_POSTGRESQL_SCHEMA_NOT_READY", + message="Required Phase 1 PostgreSQL tables are missing.", + field_name="schema_bootstrap_report.missing_table_names", + ), + ) + + +def _default_schema_bootstrap_checker( + mode: PostgreSQLSchemaBootstrapMode, + fail_on_missing: bool, +) -> PostgreSQLSchemaBootstrapReport: + return build_postgresql_phase1_schema_bootstrap_report( + mode=mode, + fail_on_missing=fail_on_missing, + ) + + +def _shutdown_issue() -> Phase1ServiceHostIssue: + return Phase1ServiceHostIssue( + code="PHASE1_SERVICE_HOST_SHUTTING_DOWN", + message="Scheduled trigger skipped because shutdown was requested.", + field_name="status", + ) + + +def _startup_diagnostic_payload( + result: Phase1ServiceHostStartupResult, +) -> dict[str, object]: + report = result.schema_bootstrap_report + return { + "issues": tuple(_service_host_issue_payload(issue) for issue in result.issues), + "schema_bootstrap": { + "fail_on_missing": getattr(report, "fail_on_missing", None), + "missing_table_count": len(getattr(report, "missing_table_names", ())), + "mode": getattr(getattr(report, "mode", None), "value", None), + "status": getattr(getattr(report, "status", None), "value", None), + }, + "status": result.status.value, + } + + +def _scheduled_run_diagnostic_payload( + result: Phase1ScheduledRunResult, +) -> dict[str, object]: + payload: dict[str, object] = { + "issues": tuple(_service_host_issue_payload(issue) for issue in result.issues), + "run_id": result.run_id, + "status": result.status.value, + } + if result.orchestrator_result is not None: + payload["orchestrator"] = summarize_phase1_orchestrator_result_for_diagnostics( + result.orchestrator_result, + ) + return payload + + +def _service_host_issue_payload(issue: Phase1ServiceHostIssue) -> dict[str, object]: + return { + "code": issue.code, + "field_name": issue.field_name, + "message": issue.message, + "severity": issue.severity, + } + + +__all__ = ( + "Phase1OrchestratorRunner", + "Phase1ScheduledIngestionServiceHost", + "Phase1ScheduledRunResult", + "Phase1ScheduledRunStatus", + "Phase1SchemaBootstrapChecker", + "Phase1ServiceHostConfig", + "Phase1ServiceHostIssue", + "Phase1ServiceHostStartupResult", + "Phase1ServiceHostStatus", + "validate_phase1_service_host_config", +) diff --git a/src/carbonfactor_parser/source_acquisition/run_repository_contract.py b/src/carbonfactor_parser/source_acquisition/run_repository_contract.py new file mode 100644 index 00000000..1d5c3a6f --- /dev/null +++ b/src/carbonfactor_parser/source_acquisition/run_repository_contract.py @@ -0,0 +1,123 @@ +"""Runtime-passive repository contract for source acquisition run results.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Protocol, runtime_checkable + +from carbonfactor_parser.source_acquisition.run_contract import SourceAcquisitionRunResult + + +class SourceAcquisitionRunRepositoryPersistStatus(str, Enum): + """Deterministic metadata-only persist status values.""" + + DECLARED = "declared" + FAILED_VALIDATION = "failed_validation" + + +@dataclass(frozen=True) +class SourceAcquisitionRunRepositoryIssue: + """Metadata-only repository contract issue.""" + + code: str + message: str + field_name: str + severity: str = "error" + + +@dataclass(frozen=True) +class SourceAcquisitionRunRepositoryPersistResult: + """Metadata-only source acquisition run repository persist result.""" + + provider_name: str + status: SourceAcquisitionRunRepositoryPersistStatus + persisted_count: int + issues: tuple[SourceAcquisitionRunRepositoryIssue, ...] = () + + +@runtime_checkable +class SourceAcquisitionRunRepository(Protocol): + """Protocol for metadata-only source acquisition run repositories.""" + + @property + def provider_name(self) -> str: + """Human-readable provider name.""" + + def persist_runs( + self, + runs: tuple[SourceAcquisitionRunResult, ...], + ) -> SourceAcquisitionRunRepositoryPersistResult: + """Persist run metadata contractually without runtime side effects.""" + + +def create_source_acquisition_run_repository_persist_result( + *, + provider_name: str, + runs: tuple[SourceAcquisitionRunResult, ...], + issues: tuple[SourceAcquisitionRunRepositoryIssue, ...] = (), +) -> SourceAcquisitionRunRepositoryPersistResult: + """Create deterministic metadata-only repository persist result.""" + + validation_issues = list( + validate_source_acquisition_run_repository_inputs( + provider_name=provider_name, + runs=runs, + ).issues, + ) + validation_issues.extend(issues) + + status = ( + SourceAcquisitionRunRepositoryPersistStatus.FAILED_VALIDATION + if validation_issues + else SourceAcquisitionRunRepositoryPersistStatus.DECLARED + ) + + return SourceAcquisitionRunRepositoryPersistResult( + provider_name=provider_name, + status=status, + persisted_count=0 if validation_issues else len(runs), + issues=tuple(validation_issues), + ) + + +@dataclass(frozen=True) +class SourceAcquisitionRunRepositoryValidationResult: + """Validation result for source acquisition run repository metadata.""" + + issues: tuple[SourceAcquisitionRunRepositoryIssue, ...] = () + + @property + def is_valid(self) -> bool: + return not self.issues + + +def validate_source_acquisition_run_repository_inputs( + *, + provider_name: str, + runs: tuple[SourceAcquisitionRunResult, ...], +) -> SourceAcquisitionRunRepositoryValidationResult: + """Validate repository inputs without runtime side effects.""" + + issues: list[SourceAcquisitionRunRepositoryIssue] = [] + + if not isinstance(provider_name, str) or not provider_name.strip(): + issues.append( + SourceAcquisitionRunRepositoryIssue( + code="SOURCE_ACQUISITION_RUN_REPOSITORY_MISSING_PROVIDER_NAME", + message="provider_name must be a non-empty string.", + field_name="provider_name", + ), + ) + + for index, run in enumerate(runs): + if not isinstance(run, SourceAcquisitionRunResult): + issues.append( + SourceAcquisitionRunRepositoryIssue( + code="SOURCE_ACQUISITION_RUN_REPOSITORY_INVALID_RUN", + message="runs must contain SourceAcquisitionRunResult instances.", + field_name=f"runs[{index}]", + ), + ) + + return SourceAcquisitionRunRepositoryValidationResult(issues=tuple(issues)) diff --git a/src/carbonfactor_parser/source_acquisition/source_onboarding_registry_contract.py b/src/carbonfactor_parser/source_acquisition/source_onboarding_registry_contract.py new file mode 100644 index 00000000..eaab1c4a --- /dev/null +++ b/src/carbonfactor_parser/source_acquisition/source_onboarding_registry_contract.py @@ -0,0 +1,406 @@ +"""Runtime-passive Phase 2 source onboarding registry contract.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + + +PHASE2_ONBOARDING_SOURCE_FAMILIES = ( + "ghg_protocol", + "defra_desnz", + "ipcc_efdb", +) + +PHASE2_ONBOARDING_PARSER_KEYS_BY_SOURCE_FAMILY = { + "ghg_protocol": "ghg_protocol_phase1_parser", + "defra_desnz": "defra_desnz_phase1_parser", + "ipcc_efdb": "ipcc_efdb_phase1_parser", +} + + +class SourceOnboardingDiscoveryStrategy(str, Enum): + """Source discovery strategies that can be declared without execution.""" + + DECLARED_REFERENCE = "declared_reference" + SOURCE_SPECIFIC_DISCOVERY = "source_specific_discovery" + + +class SourceOnboardingUpdateCadence(str, Enum): + """Declared source review cadence values for onboarding metadata.""" + + UNKNOWN = "unknown" + ANNUAL = "annual" + PERIODIC = "periodic" + + +@dataclass(frozen=True) +class SourceOnboardingDocument: + """Metadata contract for one document expected from a source family.""" + + document_id: str + display_name: str + source_reference: str + expected_format: str + required: bool = True + + +@dataclass(frozen=True) +class SourceOnboardingParserCapability: + """Parser capability metadata for an onboarded source family.""" + + parser_key: str + parser_source_format: str + supports_parser_execution: bool + capability_notes: str + + +@dataclass(frozen=True) +class SourceOnboardingValidationExpectations: + """Declared validation expectations for future source onboarding.""" + + required_document_fields: tuple[str, ...] + checksum_required: bool + schema_validation_required: bool + validation_notes: str + + +@dataclass(frozen=True) +class SourceOnboardingRuntimeSafety: + """Runtime safety metadata for source onboarding planning.""" + + allows_network_calls: bool + allows_file_reads: bool + allows_database_writes: bool + requires_credentials: bool + safety_notes: str + + +@dataclass(frozen=True) +class SourceOnboardingRegistryEntry: + """Complete onboarding metadata for one source family.""" + + source_id: str + source_family: str + display_name: str + documents: tuple[SourceOnboardingDocument, ...] + discovery_strategy: SourceOnboardingDiscoveryStrategy + parser_capability: SourceOnboardingParserCapability + validation_expectations: SourceOnboardingValidationExpectations + update_cadence: SourceOnboardingUpdateCadence + runtime_safety: SourceOnboardingRuntimeSafety + enabled: bool = True + + +@dataclass(frozen=True) +class SourceOnboardingRegistry: + """Deterministic registry of source onboarding metadata.""" + + entries: tuple[SourceOnboardingRegistryEntry, ...] + + +def create_phase2_source_onboarding_registry() -> SourceOnboardingRegistry: + """Return Phase 2 onboarding metadata for existing Phase 1 source families.""" + + entries = tuple( + SourceOnboardingRegistryEntry( + source_id=source_family, + source_family=source_family, + display_name=display_name, + documents=( + SourceOnboardingDocument( + document_id=f"{source_family}_declared_reference", + display_name=f"{display_name} declared reference", + source_reference=f"discovery://{source_family}/onboarding", + expected_format="discovery", + required=True, + ), + ), + discovery_strategy=SourceOnboardingDiscoveryStrategy.DECLARED_REFERENCE, + parser_capability=SourceOnboardingParserCapability( + parser_key=( + PHASE2_ONBOARDING_PARSER_KEYS_BY_SOURCE_FAMILY[source_family] + ), + parser_source_format="discovery_reference", + supports_parser_execution=False, + capability_notes=( + "Registry metadata only; parser execution is outside this " + "onboarding contract." + ), + ), + validation_expectations=SourceOnboardingValidationExpectations( + required_document_fields=( + "document_id", + "display_name", + "source_reference", + "expected_format", + ), + checksum_required=False, + schema_validation_required=False, + validation_notes=( + "Declared discovery references are validated for contract " + "shape only." + ), + ), + update_cadence=SourceOnboardingUpdateCadence.UNKNOWN, + runtime_safety=SourceOnboardingRuntimeSafety( + allows_network_calls=False, + allows_file_reads=False, + allows_database_writes=False, + requires_credentials=False, + safety_notes=( + "Default onboarding registry is runtime-passive and local-only." + ), + ), + enabled=True, + ) + for source_family, display_name in ( + ("ghg_protocol", "GHG Protocol"), + ("defra_desnz", "DEFRA/DESNZ"), + ("ipcc_efdb", "IPCC EFDB"), + ) + ) + registry = SourceOnboardingRegistry(entries=entries) + validate_source_onboarding_registry(registry) + return registry + + +def validate_source_onboarding_registry( + registry: SourceOnboardingRegistry, +) -> SourceOnboardingRegistry: + """Validate onboarding registry shape and deterministic identifiers.""" + + if not isinstance(registry, SourceOnboardingRegistry): + raise TypeError("registry must be a SourceOnboardingRegistry.") + + seen_source_ids: set[str] = set() + seen_source_families: set[str] = set() + seen_document_ids: set[str] = set() + + for index, entry in enumerate(registry.entries): + if not isinstance(entry, SourceOnboardingRegistryEntry): + raise TypeError( + f"entries[{index}] must be a SourceOnboardingRegistryEntry.", + ) + + _validate_required_string(entry.source_id, "source_id", entry.source_id) + _validate_required_string( + entry.source_family, + "source_family", + entry.source_id, + ) + _validate_required_string(entry.display_name, "display_name", entry.source_id) + if not entry.documents: + raise ValueError( + f"documents must include at least one document for source_id " + f"'{entry.source_id}'." + ) + if entry.source_id in seen_source_ids: + raise ValueError(f"Duplicate source_id found: {entry.source_id}") + if entry.source_family in seen_source_families: + raise ValueError( + f"Duplicate source_family found: {entry.source_family}", + ) + seen_source_ids.add(entry.source_id) + seen_source_families.add(entry.source_family) + + _validate_document_order(entry) + for document in entry.documents: + _validate_document(entry, document, seen_document_ids) + if not isinstance(entry.discovery_strategy, SourceOnboardingDiscoveryStrategy): + raise TypeError( + f"discovery_strategy must be a SourceOnboardingDiscoveryStrategy " + f"for source_id '{entry.source_id}'." + ) + if not isinstance(entry.update_cadence, SourceOnboardingUpdateCadence): + raise TypeError( + f"update_cadence must be a SourceOnboardingUpdateCadence for " + f"source_id '{entry.source_id}'." + ) + _validate_bool(entry.enabled, "enabled", entry.source_id) + _validate_parser_capability(entry) + _validate_validation_expectations(entry) + _validate_runtime_safety(entry) + + _validate_registry_order(registry.entries) + return registry + + +def list_source_onboarding_entries( + registry: SourceOnboardingRegistry | None = None, +) -> tuple[SourceOnboardingRegistryEntry, ...]: + """List source onboarding entries without executing discovery or parsing.""" + + active_registry = ( + create_phase2_source_onboarding_registry() + if registry is None + else validate_source_onboarding_registry(registry) + ) + return active_registry.entries + + +def get_source_onboarding_entry( + source_family: str, + registry: SourceOnboardingRegistry | None = None, +) -> SourceOnboardingRegistryEntry | None: + """Return one onboarding entry by source family, if declared.""" + + for entry in list_source_onboarding_entries(registry): + if entry.source_family == source_family: + return entry + return None + + +def _validate_document_order(entry: SourceOnboardingRegistryEntry) -> None: + document_ids = tuple(document.document_id for document in entry.documents) + if document_ids != tuple(sorted(document_ids)): + raise ValueError( + f"documents must be ordered by document_id for source_id " + f"'{entry.source_id}'." + ) + + +def _validate_document( + entry: SourceOnboardingRegistryEntry, + document: SourceOnboardingDocument, + seen_document_ids: set[str], +) -> None: + if not isinstance(document, SourceOnboardingDocument): + raise TypeError( + f"documents for source_id '{entry.source_id}' must be " + "SourceOnboardingDocument values." + ) + + _validate_required_string(document.document_id, "document_id", entry.source_id) + _validate_required_string( + document.display_name, + "document.display_name", + entry.source_id, + ) + _validate_required_string( + document.source_reference, + "source_reference", + entry.source_id, + ) + _validate_required_string( + document.expected_format, + "expected_format", + entry.source_id, + ) + if document.document_id in seen_document_ids: + raise ValueError(f"Duplicate document_id found: {document.document_id}") + seen_document_ids.add(document.document_id) + _validate_bool(document.required, "required", entry.source_id) + + +def _validate_parser_capability(entry: SourceOnboardingRegistryEntry) -> None: + capability = entry.parser_capability + if not isinstance(capability, SourceOnboardingParserCapability): + raise TypeError( + f"parser_capability must be a SourceOnboardingParserCapability for " + f"source_id '{entry.source_id}'." + ) + _validate_required_string(capability.parser_key, "parser_key", entry.source_id) + _validate_required_string( + capability.parser_source_format, + "parser_source_format", + entry.source_id, + ) + _validate_required_string( + capability.capability_notes, + "capability_notes", + entry.source_id, + ) + _validate_bool( + capability.supports_parser_execution, + "supports_parser_execution", + entry.source_id, + ) + + +def _validate_validation_expectations(entry: SourceOnboardingRegistryEntry) -> None: + expectations = entry.validation_expectations + if not isinstance(expectations, SourceOnboardingValidationExpectations): + raise TypeError( + "validation_expectations must be a " + f"SourceOnboardingValidationExpectations for source_id '{entry.source_id}'." + ) + if not expectations.required_document_fields: + raise ValueError( + f"required_document_fields must not be empty for source_id " + f"'{entry.source_id}'." + ) + for field_name in expectations.required_document_fields: + _validate_required_string( + field_name, + "required_document_fields", + entry.source_id, + ) + _validate_required_string( + expectations.validation_notes, + "validation_notes", + entry.source_id, + ) + _validate_bool(expectations.checksum_required, "checksum_required", entry.source_id) + _validate_bool( + expectations.schema_validation_required, + "schema_validation_required", + entry.source_id, + ) + + +def _validate_runtime_safety(entry: SourceOnboardingRegistryEntry) -> None: + safety = entry.runtime_safety + if not isinstance(safety, SourceOnboardingRuntimeSafety): + raise TypeError( + f"runtime_safety must be a SourceOnboardingRuntimeSafety for source_id " + f"'{entry.source_id}'." + ) + _validate_required_string(safety.safety_notes, "safety_notes", entry.source_id) + _validate_bool(safety.allows_network_calls, "allows_network_calls", entry.source_id) + _validate_bool(safety.allows_file_reads, "allows_file_reads", entry.source_id) + _validate_bool( + safety.allows_database_writes, + "allows_database_writes", + entry.source_id, + ) + _validate_bool(safety.requires_credentials, "requires_credentials", entry.source_id) + + +def _validate_registry_order( + entries: tuple[SourceOnboardingRegistryEntry, ...], +) -> None: + ordered_entries = tuple( + sorted( + entries, + key=lambda entry: ( + _source_order_key(entry.source_family), + entry.source_id, + ), + ) + ) + if entries != ordered_entries: + raise ValueError( + "entries must follow Phase 1 source order, then source_id order.", + ) + + +def _source_order_key(source_family: str) -> int: + try: + return PHASE2_ONBOARDING_SOURCE_FAMILIES.index(source_family) + except ValueError: + return len(PHASE2_ONBOARDING_SOURCE_FAMILIES) + + +def _validate_required_string(value: str, field_name: str, source_id: str) -> None: + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"{field_name} must be a non-empty string for source_id '{source_id}'.", + ) + + +def _validate_bool(value: bool, field_name: str, source_id: str) -> None: + if not isinstance(value, bool): + raise TypeError( + f"{field_name} must be a bool for source_id '{source_id}'.", + ) diff --git a/src/dotnet/CarbonOps.Parser.Contracts/CarbonOps.Parser.Contracts.csproj b/src/dotnet/CarbonOps.Parser.Contracts/CarbonOps.Parser.Contracts.csproj index fa71b7ae..f8f3b830 100644 --- a/src/dotnet/CarbonOps.Parser.Contracts/CarbonOps.Parser.Contracts.csproj +++ b/src/dotnet/CarbonOps.Parser.Contracts/CarbonOps.Parser.Contracts.csproj @@ -6,4 +6,8 @@ enable + + + + diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs b/src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs index ba9db1c4..086ca4a9 100644 --- a/src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs +++ b/src/dotnet/CarbonOps.Parser.Contracts/ContractWireNames.cs @@ -49,6 +49,88 @@ public static string ToWireName(this SourceDiscoveryStatus value) => _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown source discovery status."), }; + public static string ToWireName(this GhgSourceDiscoveryMode value) => + value switch + { + GhgSourceDiscoveryMode.RuntimePassive => "runtime_passive", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown GHG source discovery mode."), + }; + + public static string ToWireName(this GhgSourceDiscoveryStatus value) => + value switch + { + GhgSourceDiscoveryStatus.Declared => "declared", + GhgSourceDiscoveryStatus.Invalid => "invalid", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown GHG source discovery status."), + }; + + public static string ToWireName(this DefraSourceDiscoveryMode value) => + value switch + { + DefraSourceDiscoveryMode.RuntimePassive => "runtime_passive", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown DEFRA source discovery mode."), + }; + + public static string ToWireName(this DefraSourceDiscoveryStatus value) => + value switch + { + DefraSourceDiscoveryStatus.Declared => "declared", + DefraSourceDiscoveryStatus.Invalid => "invalid", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown DEFRA source discovery status."), + }; + + public static string ToWireName(this IpccSourceDiscoveryMode value) => + value switch + { + IpccSourceDiscoveryMode.RuntimePassive => "runtime_passive", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown IPCC source discovery mode."), + }; + + public static string ToWireName(this IpccSourceDiscoveryStatus value) => + value switch + { + IpccSourceDiscoveryStatus.Declared => "declared", + IpccSourceDiscoveryStatus.Invalid => "invalid", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown IPCC source discovery status."), + }; + + public static string ToWireName(this GhgSourceDownloadExecutionStatus value) => + value switch + { + GhgSourceDownloadExecutionStatus.Blocked => "blocked", + GhgSourceDownloadExecutionStatus.Downloaded => "downloaded", + GhgSourceDownloadExecutionStatus.Failed => "failed", + _ => throw new ArgumentOutOfRangeException( + nameof(value), + value, + "Unknown GHG source download execution status."), + }; + + public static string ToWireName(this DefraSourceDownloadExecutionStatus value) => + value switch + { + DefraSourceDownloadExecutionStatus.Blocked => "blocked", + DefraSourceDownloadExecutionStatus.Downloaded => "downloaded", + DefraSourceDownloadExecutionStatus.Failed => "failed", + _ => throw new ArgumentOutOfRangeException( + nameof(value), + value, + "Unknown DEFRA source download execution status."), + }; + + public static string ToWireName(this IpccSourceDownloadExecutionStatus value) => + value switch + { + IpccSourceDownloadExecutionStatus.Blocked => "blocked", + IpccSourceDownloadExecutionStatus.Downloaded => "downloaded", + IpccSourceDownloadExecutionStatus.Failed => "failed", + IpccSourceDownloadExecutionStatus.AlreadyKnown => "already_known", + _ => throw new ArgumentOutOfRangeException( + nameof(value), + value, + "Unknown IPCC source download execution status."), + }; + public static string ToWireName(this ParserSourceFormat value) => value switch { @@ -72,6 +154,33 @@ public static string ToWireName(this ParserValidationIssueSeverity value) => _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown parser validation issue severity."), }; + public static string ToWireName(this DataQualityValidationSeverity value) => + value switch + { + DataQualityValidationSeverity.BlockingError => "blocking_error", + DataQualityValidationSeverity.Warning => "warning", + DataQualityValidationSeverity.Info => "info", + _ => throw new ArgumentOutOfRangeException( + nameof(value), + value, + "Unknown data quality validation severity."), + }; + + public static string ToWireName(this DataQualityValidationCheck value) => + value switch + { + DataQualityValidationCheck.RequiredField => "required_field", + DataQualityValidationCheck.NumericValue => "numeric_value", + DataQualityValidationCheck.Unit => "unit", + DataQualityValidationCheck.DuplicateFactorIdentity => "duplicate_factor_identity", + DataQualityValidationCheck.Provenance => "provenance", + DataQualityValidationCheck.Structure => "structure", + _ => throw new ArgumentOutOfRangeException( + nameof(value), + value, + "Unknown data quality validation check."), + }; + public static string ToWireName(this ParserDryRunStatus value) => value switch { @@ -81,6 +190,31 @@ public static string ToWireName(this ParserDryRunStatus value) => _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown parser dry-run status."), }; + public static string ToWireName(this SourceCycleRunStatus value) => + value switch + { + SourceCycleRunStatus.Ready => "ready", + SourceCycleRunStatus.Blocked => "blocked", + SourceCycleRunStatus.NoAvailableSourceYear => "no_available_source_year", + SourceCycleRunStatus.ParserNotAvailable => "parser_not_available", + SourceCycleRunStatus.Parsed => "parsed", + SourceCycleRunStatus.PersistenceNotImplemented => "persistence_not_implemented", + SourceCycleRunStatus.NotImplemented => "not_implemented", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown source cycle run status."), + }; + + public static string ToWireName(this PostgreSQLRuntimeConfigGateStatus value) => + value switch + { + PostgreSQLRuntimeConfigGateStatus.Disabled => "disabled", + PostgreSQLRuntimeConfigGateStatus.Blocked => "blocked", + PostgreSQLRuntimeConfigGateStatus.NotEnabled => "not_enabled", + _ => throw new ArgumentOutOfRangeException( + nameof(value), + value, + "Unknown PostgreSQL runtime config gate status."), + }; + public static bool TryParseSourceFamilyWireName(string? wireName, out SourceFamily value) { value = wireName switch @@ -148,6 +282,129 @@ public static bool TryParseSourceDiscoveryStatusWireName(string? wireName, out S return wireName is "declared"; } + public static bool TryParseGhgSourceDiscoveryModeWireName(string? wireName, out GhgSourceDiscoveryMode value) + { + value = wireName switch + { + "runtime_passive" => GhgSourceDiscoveryMode.RuntimePassive, + _ => default, + }; + + return wireName is "runtime_passive"; + } + + public static bool TryParseGhgSourceDiscoveryStatusWireName(string? wireName, out GhgSourceDiscoveryStatus value) + { + value = wireName switch + { + "declared" => GhgSourceDiscoveryStatus.Declared, + "invalid" => GhgSourceDiscoveryStatus.Invalid, + _ => default, + }; + + return wireName is "declared" or "invalid"; + } + + public static bool TryParseDefraSourceDiscoveryModeWireName( + string? wireName, + out DefraSourceDiscoveryMode value) + { + value = wireName switch + { + "runtime_passive" => DefraSourceDiscoveryMode.RuntimePassive, + _ => default, + }; + + return wireName is "runtime_passive"; + } + + public static bool TryParseDefraSourceDiscoveryStatusWireName( + string? wireName, + out DefraSourceDiscoveryStatus value) + { + value = wireName switch + { + "declared" => DefraSourceDiscoveryStatus.Declared, + "invalid" => DefraSourceDiscoveryStatus.Invalid, + _ => default, + }; + + return wireName is "declared" or "invalid"; + } + + public static bool TryParseIpccSourceDiscoveryModeWireName( + string? wireName, + out IpccSourceDiscoveryMode value) + { + value = wireName switch + { + "runtime_passive" => IpccSourceDiscoveryMode.RuntimePassive, + _ => default, + }; + + return wireName is "runtime_passive"; + } + + public static bool TryParseIpccSourceDiscoveryStatusWireName( + string? wireName, + out IpccSourceDiscoveryStatus value) + { + value = wireName switch + { + "declared" => IpccSourceDiscoveryStatus.Declared, + "invalid" => IpccSourceDiscoveryStatus.Invalid, + _ => default, + }; + + return wireName is "declared" or "invalid"; + } + + public static bool TryParseGhgSourceDownloadExecutionStatusWireName( + string? wireName, + out GhgSourceDownloadExecutionStatus value) + { + value = wireName switch + { + "blocked" => GhgSourceDownloadExecutionStatus.Blocked, + "downloaded" => GhgSourceDownloadExecutionStatus.Downloaded, + "failed" => GhgSourceDownloadExecutionStatus.Failed, + _ => default, + }; + + return wireName is "blocked" or "downloaded" or "failed"; + } + + public static bool TryParseDefraSourceDownloadExecutionStatusWireName( + string? wireName, + out DefraSourceDownloadExecutionStatus value) + { + value = wireName switch + { + "blocked" => DefraSourceDownloadExecutionStatus.Blocked, + "downloaded" => DefraSourceDownloadExecutionStatus.Downloaded, + "failed" => DefraSourceDownloadExecutionStatus.Failed, + _ => default, + }; + + return wireName is "blocked" or "downloaded" or "failed"; + } + + public static bool TryParseIpccSourceDownloadExecutionStatusWireName( + string? wireName, + out IpccSourceDownloadExecutionStatus value) + { + value = wireName switch + { + "blocked" => IpccSourceDownloadExecutionStatus.Blocked, + "downloaded" => IpccSourceDownloadExecutionStatus.Downloaded, + "failed" => IpccSourceDownloadExecutionStatus.Failed, + "already_known" => IpccSourceDownloadExecutionStatus.AlreadyKnown, + _ => default, + }; + + return wireName is "blocked" or "downloaded" or "failed" or "already_known"; + } + public static bool TryParseParserSourceFormatWireName(string? wireName, out ParserSourceFormat value) { value = wireName switch @@ -185,6 +442,44 @@ public static bool TryParseParserValidationIssueSeverityWireName( return wireName is "info" or "warning" or "error"; } + public static bool TryParseDataQualityValidationSeverityWireName( + string? wireName, + out DataQualityValidationSeverity value) + { + value = wireName switch + { + "blocking_error" => DataQualityValidationSeverity.BlockingError, + "warning" => DataQualityValidationSeverity.Warning, + "info" => DataQualityValidationSeverity.Info, + _ => default, + }; + + return wireName is "blocking_error" or "warning" or "info"; + } + + public static bool TryParseDataQualityValidationCheckWireName( + string? wireName, + out DataQualityValidationCheck value) + { + value = wireName switch + { + "required_field" => DataQualityValidationCheck.RequiredField, + "numeric_value" => DataQualityValidationCheck.NumericValue, + "unit" => DataQualityValidationCheck.Unit, + "duplicate_factor_identity" => DataQualityValidationCheck.DuplicateFactorIdentity, + "provenance" => DataQualityValidationCheck.Provenance, + "structure" => DataQualityValidationCheck.Structure, + _ => default, + }; + + return wireName is "required_field" + or "numeric_value" + or "unit" + or "duplicate_factor_identity" + or "provenance" + or "structure"; + } + public static bool TryParseParserDryRunStatusWireName(string? wireName, out ParserDryRunStatus value) { value = wireName switch @@ -197,4 +492,37 @@ public static bool TryParseParserDryRunStatusWireName(string? wireName, out Pars return wireName is "planned" or "invalid_request" or "execution_not_implemented"; } + + public static bool TryParseSourceCycleRunStatusWireName(string? wireName, out SourceCycleRunStatus value) + { + value = wireName switch + { + "ready" => SourceCycleRunStatus.Ready, + "blocked" => SourceCycleRunStatus.Blocked, + "no_available_source_year" => SourceCycleRunStatus.NoAvailableSourceYear, + "parser_not_available" => SourceCycleRunStatus.ParserNotAvailable, + "parsed" => SourceCycleRunStatus.Parsed, + "persistence_not_implemented" => SourceCycleRunStatus.PersistenceNotImplemented, + "not_implemented" => SourceCycleRunStatus.NotImplemented, + _ => default, + }; + + return wireName is "ready" or "blocked" or "no_available_source_year" or "parser_not_available" or + "parsed" or "persistence_not_implemented" or "not_implemented"; + } + + public static bool TryParsePostgreSQLRuntimeConfigGateStatusWireName( + string? wireName, + out PostgreSQLRuntimeConfigGateStatus value) + { + value = wireName switch + { + "disabled" => PostgreSQLRuntimeConfigGateStatus.Disabled, + "blocked" => PostgreSQLRuntimeConfigGateStatus.Blocked, + "not_enabled" => PostgreSQLRuntimeConfigGateStatus.NotEnabled, + _ => default, + }; + + return wireName is "disabled" or "blocked" or "not_enabled"; + } } diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DataQualityDiagnostic.cs b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityDiagnostic.cs new file mode 100644 index 00000000..0d0e2866 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityDiagnostic.cs @@ -0,0 +1,45 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DataQualityDiagnostic +{ + public string Code { get; } + + public string Message { get; } + + public DataQualityValidationSeverity Severity { get; } + + public DataQualityValidationCheck Check { get; } + + public string? FieldName { get; } + + public string? SourceFamily { get; } + + public DataQualityProvenanceContext? Provenance { get; } + + public IReadOnlyList Context { get; } + + public DataQualityDiagnostic( + string code, + string message, + DataQualityValidationSeverity severity, + DataQualityValidationCheck check, + string? fieldName = null, + string? sourceFamily = null, + DataQualityProvenanceContext? provenance = null, + IEnumerable? context = null) + { + Code = code; + Message = message; + Severity = severity; + Check = check; + FieldName = fieldName; + SourceFamily = DataQualityValidation.SafeDiagnosticText(sourceFamily); + Provenance = provenance; + Context = Array.AsReadOnly((context ?? []) + .OrderBy(item => item.Key, StringComparer.Ordinal) + .Select(item => new DataQualityDiagnosticContext( + item.Key, + DataQualityValidation.SafeDiagnosticValue(item.Key, item.Value))) + .ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DataQualityDiagnosticContext.cs b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityDiagnosticContext.cs new file mode 100644 index 00000000..29c18d7e --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityDiagnosticContext.cs @@ -0,0 +1,3 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DataQualityDiagnosticContext(string Key, string? Value); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DataQualityProvenanceContext.cs b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityProvenanceContext.cs new file mode 100644 index 00000000..28abc9e7 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityProvenanceContext.cs @@ -0,0 +1,36 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DataQualityProvenanceContext +{ + public string RowIdentifier { get; } + + public string? SourceFamily { get; } + + public string? SourceKey { get; } + + public string? SourceReference { get; } + + public string? RowNumber { get; } + + public string? Provenance { get; } + + public string? DocumentId { get; } + + public DataQualityProvenanceContext( + string rowIdentifier, + string? sourceFamily = null, + string? sourceKey = null, + string? sourceReference = null, + string? rowNumber = null, + string? provenance = null, + string? documentId = null) + { + RowIdentifier = rowIdentifier; + SourceFamily = DataQualityValidation.SafeDiagnosticText(sourceFamily); + SourceKey = DataQualityValidation.SafeDiagnosticText(sourceKey); + SourceReference = DataQualityValidation.SafeDiagnosticText(sourceReference); + RowNumber = rowNumber; + Provenance = DataQualityValidation.SafeDiagnosticText(provenance); + DocumentId = DataQualityValidation.SafeDiagnosticText(documentId); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidation.cs b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidation.cs new file mode 100644 index 00000000..39fa9d89 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidation.cs @@ -0,0 +1,320 @@ +using System.Globalization; +using System.Text.RegularExpressions; + +namespace CarbonOps.Parser.Contracts; + +public static partial class DataQualityValidation +{ + public const string RedactedDiagnosticValue = "[REDACTED]"; + + public static readonly IReadOnlyList DefaultSupportedFactorUnits = Array.AsReadOnly(new[] + { + "kg", + "kg CO2e", + "kg CO2e/kWh", + "kWh", + }); + + private static readonly string[] RequiredFactorFields = + [ + "source_family", + "source_id", + "factor_id", + "factor_name", + "factor_value", + "unit", + ]; + + private static readonly string[] ProvenanceFieldNames = + [ + "provenance", + "row_number", + "source_document_id", + "document_id", + ]; + + private static readonly string[] SensitiveFieldTokens = + [ + "api_key", + "authorization", + "credential", + "password", + "secret", + "token", + ]; + + [GeneratedRegex("//[^/\\s:@]+:[^@\\s/]+@")] + private static partial Regex UserInfoUriPattern(); + + [GeneratedRegex("\\b(api[_-]?key|authorization|credential|password|secret|token)=([^\\s&;,]+)", RegexOptions.IgnoreCase)] + private static partial Regex SensitiveAssignmentPattern(); + + public static DataQualityDiagnostic CreateDiagnostic( + string code, + string message, + DataQualityValidationSeverity severity, + DataQualityValidationCheck check, + string? fieldName = null, + string? sourceFamily = null, + DataQualityProvenanceContext? provenance = null, + IEnumerable? context = null) => + new(code, message, severity, check, fieldName, sourceFamily, provenance, context); + + public static DataQualityValidationResult ValidateNormalizedFactorOutput( + ParserNormalizedOutputBatch output, + IEnumerable? supportedUnits = null) + { + var unitSet = (supportedUnits ?? DefaultSupportedFactorUnits).ToHashSet(StringComparer.Ordinal); + var diagnostics = new List(); + var identityPositions = new Dictionary(StringComparer.Ordinal); + + for (var index = 0; index < output.Rows.Count; index++) + { + var position = index + 1; + var row = output.Rows[index]; + var fields = RowFields(row); + var sourceFamily = TextOrNull(FieldValue(fields, "source_family")) ?? row.SourceFamily.ToWireName(); + var provenance = ProvenanceContext(row, fields); + + diagnostics.AddRange(MissingRequiredFieldDiagnostics(fields, position, sourceFamily, provenance)); + diagnostics.AddRange(InvalidNumericDiagnostics(fields, position, sourceFamily, provenance)); + diagnostics.AddRange(UnsupportedUnitDiagnostics(fields, unitSet, position, sourceFamily, provenance)); + diagnostics.AddRange(ProvenanceGapDiagnostics(row, fields, position, sourceFamily, provenance)); + + var identity = FactorIdentity(fields); + if (identity is null) + { + continue; + } + + if (identityPositions.TryGetValue(identity, out var firstPosition)) + { + diagnostics.Add(CreateDiagnostic( + "NORMALIZED_FACTOR_DUPLICATE_IDENTITY", + "normalized factor identity must be unique within the validation result.", + DataQualityValidationSeverity.BlockingError, + DataQualityValidationCheck.DuplicateFactorIdentity, + fieldName: "factor_id", + sourceFamily: sourceFamily, + provenance: provenance, + context: + [ + new("first_record_position", firstPosition.ToString(CultureInfo.InvariantCulture)), + new("record_position", position.ToString(CultureInfo.InvariantCulture)), + ])); + } + else + { + identityPositions[identity] = position; + } + } + + return new DataQualityValidationResult(diagnostics + .OrderBy(diagnostic => ContextIntValue(diagnostic, "record_position")) + .ThenBy(diagnostic => diagnostic.Code, StringComparer.Ordinal) + .ThenBy(diagnostic => diagnostic.FieldName ?? string.Empty, StringComparer.Ordinal)); + } + + public static string? SafeDiagnosticText(string? value) + { + var text = TextOrNull(value); + if (text is null) + { + return null; + } + + var withoutUserInfo = UserInfoUriPattern().Replace(text, $"//{RedactedDiagnosticValue}@"); + return SensitiveAssignmentPattern().Replace( + withoutUserInfo, + match => $"{match.Groups[1].Value}={RedactedDiagnosticValue}"); + } + + public static string? SafeDiagnosticValue(string fieldName, string? value) => + IsSensitiveField(fieldName) && value is not null + ? RedactedDiagnosticValue + : SafeDiagnosticText(value); + + private static IReadOnlyDictionary RowFields(ParserNormalizedOutputRow row) => + row.Fields + .GroupBy(field => field.Key, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.First().Value, StringComparer.Ordinal); + + private static IEnumerable MissingRequiredFieldDiagnostics( + IReadOnlyDictionary fields, + int position, + string? sourceFamily, + DataQualityProvenanceContext provenance) + { + foreach (var fieldName in RequiredFactorFields) + { + if (!MissingField(fields, fieldName)) + { + continue; + } + + yield return CreateDiagnostic( + "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD", + "normalized factor output is missing a required field.", + DataQualityValidationSeverity.BlockingError, + DataQualityValidationCheck.RequiredField, + fieldName: fieldName, + sourceFamily: sourceFamily, + provenance: provenance, + context: + [ + new("field_name", fieldName), + new("record_position", position.ToString(CultureInfo.InvariantCulture)), + ]); + } + } + + private static IEnumerable InvalidNumericDiagnostics( + IReadOnlyDictionary fields, + int position, + string? sourceFamily, + DataQualityProvenanceContext provenance) + { + if (MissingField(fields, "factor_value") || IsValidNumeric(FieldValue(fields, "factor_value"))) + { + return []; + } + + return + [ + CreateDiagnostic( + "NORMALIZED_FACTOR_INVALID_NUMERIC_VALUE", + "normalized factor_value must be numeric.", + DataQualityValidationSeverity.BlockingError, + DataQualityValidationCheck.NumericValue, + fieldName: "factor_value", + sourceFamily: sourceFamily, + provenance: provenance, + context: + [ + new("field_name", "factor_value"), + new("record_position", position.ToString(CultureInfo.InvariantCulture)), + ]), + ]; + } + + private static IEnumerable UnsupportedUnitDiagnostics( + IReadOnlyDictionary fields, + HashSet supportedUnits, + int position, + string? sourceFamily, + DataQualityProvenanceContext provenance) + { + var unit = TextOrNull(FieldValue(fields, "unit")); + if (unit is null || supportedUnits.Contains(unit)) + { + return []; + } + + return + [ + CreateDiagnostic( + "NORMALIZED_FACTOR_UNSUPPORTED_UNIT", + "normalized factor unit is not in the configured supported unit set.", + DataQualityValidationSeverity.Warning, + DataQualityValidationCheck.Unit, + fieldName: "unit", + sourceFamily: sourceFamily, + provenance: provenance, + context: + [ + new("field_name", "unit"), + new("record_position", position.ToString(CultureInfo.InvariantCulture)), + new("supported_unit_count", supportedUnits.Count.ToString(CultureInfo.InvariantCulture)), + ]), + ]; + } + + private static IEnumerable ProvenanceGapDiagnostics( + ParserNormalizedOutputRow row, + IReadOnlyDictionary fields, + int position, + string? sourceFamily, + DataQualityProvenanceContext provenance) + { + var hasFieldProvenance = ProvenanceFieldNames.Any(name => !MissingField(fields, name)); + if (!string.IsNullOrWhiteSpace(row.ArtifactReference) || row.SourceRowNumber is not null || hasFieldProvenance) + { + return []; + } + + return + [ + CreateDiagnostic( + "NORMALIZED_FACTOR_PROVENANCE_GAP", + "normalized factor output should include row or document provenance before downstream use.", + DataQualityValidationSeverity.Warning, + DataQualityValidationCheck.Provenance, + sourceFamily: sourceFamily, + provenance: provenance, + context: + [ + new("record_position", position.ToString(CultureInfo.InvariantCulture)), + ]), + ]; + } + + private static string? FactorIdentity(IReadOnlyDictionary fields) + { + string[] identityFields = + [ + "source_family", + "source_id", + "source_year", + "source_version", + "factor_id", + "unit", + ]; + + if (identityFields.Any(fieldName => MissingField(fields, fieldName))) + { + return null; + } + + return string.Join("\u001f", identityFields.Select(fieldName => TextOrNull(FieldValue(fields, fieldName)))); + } + + private static DataQualityProvenanceContext ProvenanceContext( + ParserNormalizedOutputRow row, + IReadOnlyDictionary fields) => + new( + row.RowIdentifier, + TextOrNull(FieldValue(fields, "source_family")) ?? row.SourceFamily.ToWireName(), + TextOrNull(FieldValue(fields, "source_id")) ?? row.SourceKey, + row.ArtifactReference, + TextOrNull(FieldValue(fields, "row_number")) ?? + row.SourceRowNumber?.ToString(CultureInfo.InvariantCulture), + TextOrNull(FieldValue(fields, "provenance")), + TextOrNull(FieldValue(fields, "source_document_id")) ?? TextOrNull(FieldValue(fields, "document_id"))); + + private static bool MissingField(IReadOnlyDictionary fields, string fieldName) => + !fields.TryGetValue(fieldName, out var value) || string.IsNullOrWhiteSpace(value); + + private static string? FieldValue(IReadOnlyDictionary fields, string fieldName) => + fields.TryGetValue(fieldName, out var value) ? value : null; + + private static bool IsValidNumeric(string? value) => + !string.IsNullOrWhiteSpace(value) && + decimal.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out _); + + private static string? TextOrNull(string? value) => + string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private static string ContextValue(DataQualityDiagnostic diagnostic, string key) => + diagnostic.Context.FirstOrDefault(item => item.Key == key)?.Value ?? "0"; + + private static int ContextIntValue(DataQualityDiagnostic diagnostic, string key) => + int.TryParse(ContextValue(diagnostic, key), NumberStyles.Integer, CultureInfo.InvariantCulture, out var value) + ? value + : 0; + + private static bool IsSensitiveField(string fieldName) + { + var normalized = fieldName.ToLowerInvariant(); + return SensitiveFieldTokens.Any(token => normalized.Contains(token, StringComparison.Ordinal)); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidationCheck.cs b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidationCheck.cs new file mode 100644 index 00000000..b0a20e70 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidationCheck.cs @@ -0,0 +1,11 @@ +namespace CarbonOps.Parser.Contracts; + +public enum DataQualityValidationCheck +{ + RequiredField = 0, + NumericValue = 1, + Unit = 2, + DuplicateFactorIdentity = 3, + Provenance = 4, + Structure = 5, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidationResult.cs new file mode 100644 index 00000000..95bc866c --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidationResult.cs @@ -0,0 +1,25 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DataQualityValidationResult +{ + public IReadOnlyList Diagnostics { get; } + + public bool IsValid => !HasBlockingErrors; + + public bool HasBlockingErrors => Diagnostics.Any( + diagnostic => diagnostic.Severity == DataQualityValidationSeverity.BlockingError); + + public int BlockingErrorCount => CountSeverity(DataQualityValidationSeverity.BlockingError); + + public int WarningCount => CountSeverity(DataQualityValidationSeverity.Warning); + + public int InfoCount => CountSeverity(DataQualityValidationSeverity.Info); + + public DataQualityValidationResult(IEnumerable? diagnostics = null) + { + Diagnostics = Array.AsReadOnly((diagnostics ?? []).ToArray()); + } + + private int CountSeverity(DataQualityValidationSeverity severity) => + Diagnostics.Count(diagnostic => diagnostic.Severity == severity); +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidationSeverity.cs b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidationSeverity.cs new file mode 100644 index 00000000..046f699f --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DataQualityValidationSeverity.cs @@ -0,0 +1,8 @@ +namespace CarbonOps.Parser.Contracts; + +public enum DataQualityValidationSeverity +{ + BlockingError = 0, + Warning = 1, + Info = 2, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraDesnzNormalizedContentParser.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraDesnzNormalizedContentParser.cs new file mode 100644 index 00000000..b4588a05 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraDesnzNormalizedContentParser.cs @@ -0,0 +1,407 @@ +using System.Globalization; + +namespace CarbonOps.Parser.Contracts; + +public static class DefraDesnzNormalizedContentParser +{ + public static IReadOnlyList Header { get; } = Array.AsReadOnly(new[] + { + "source_year", + "source_version", + "category", + "subcategory", + "activity", + "factor_id", + "factor_name", + "factor_value", + "unit", + "greenhouse_gas", + "provenance", + }); + + private static readonly IReadOnlySet RequiredFields = new HashSet( + new[] + { + "source_year", + "source_version", + "category", + "factor_id", + "factor_name", + "factor_value", + "unit", + "provenance", + }, + StringComparer.Ordinal); + + public static ParserAdapterRunResult Parse( + ParserAdapterRunRequest request, + IReadOnlyDictionary contentByArtifactReference) + { + var requestValidation = request.Validate(); + if (!requestValidation.IsValid) + { + return FailedResult( + request, + [], + requestValidation.Errors.Select(error => Issue( + request, + ParserValidationIssueSeverity.Error, + "DEFRA_DESNZ_CONTENT_INVALID_REQUEST", + error))); + } + + if (request.SourceFamily != SourceFamily.DefraDesnz) + { + return FailedResult( + request, + [], + [ + Issue( + request, + ParserValidationIssueSeverity.Error, + "DEFRA_DESNZ_CONTENT_SOURCE_FAMILY_MISMATCH", + "DEFRA/DESNZ content parser only accepts defra_desnz source family.", + fieldKey: "source_family"), + ]); + } + + var rows = new List(); + var issues = new List(); + + foreach (var artifact in request.Artifacts) + { + if (!contentByArtifactReference.TryGetValue(artifact.ArtifactReference, out var content)) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Error, + "DEFRA_DESNZ_CONTENT_MISSING_ARTIFACT_CONTENT", + "DEFRA/DESNZ parser content was not provided for an input artifact.", + artifact.ArtifactReference, + fieldKey: "artifact_reference")); + continue; + } + + ParseArtifact(request, artifact, content, rows, issues); + } + + if (issues.Any(issue => issue.Severity == ParserValidationIssueSeverity.Error)) + { + return FailedResult(request, rows, issues); + } + + if (rows.Count == 0) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Warning, + "DEFRA_DESNZ_CONTENT_NO_RECORDS", + "DEFRA/DESNZ normalized content included no parseable emission factor rows.", + fieldKey: "content")); + } + + return new ParserAdapterRunResult( + request.SourceFamily, + request.SourceKey, + request.ParserKey, + ParserRunStatus.Completed, + request.Artifacts.Select(artifact => artifact.ArtifactReference), + rows, + issues, + request.RunId, + request.CorrelationId, + request.RequestedReportingYear); + } + + private static void ParseArtifact( + ParserAdapterRunRequest request, + ParserInputArtifact artifact, + string content, + ICollection rows, + ICollection issues) + { + var csvRows = CsvRows(content).ToArray(); + if (csvRows.Length == 0) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Warning, + "DEFRA_DESNZ_CONTENT_EMPTY", + "DEFRA/DESNZ content input did not include parseable content.", + artifact.ArtifactReference, + fieldKey: "content")); + return; + } + + if (!csvRows[0].SequenceEqual(Header, StringComparer.Ordinal)) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Error, + "DEFRA_DESNZ_CONTENT_INVALID_HEADER", + "DEFRA/DESNZ normalized content header must match the declared parser contract.", + artifact.ArtifactReference, + sourceRowNumber: 1, + fieldKey: "header")); + return; + } + + for (var rowIndex = 1; rowIndex < csvRows.Length; rowIndex++) + { + var sourceRowNumber = rowIndex + 1; + var values = csvRows[rowIndex]; + if (values.Count == 1 && string.IsNullOrWhiteSpace(values[0])) + { + continue; + } + + if (values.Count != Header.Count) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Error, + "DEFRA_DESNZ_CONTENT_INVALID_ROW", + "DEFRA/DESNZ normalized content row has an unexpected column count.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: $"row[{sourceRowNumber.ToString(CultureInfo.InvariantCulture)}]")); + continue; + } + + var row = Header + .Select((field, index) => new { field, value = values[index].Trim() }) + .ToDictionary(pair => pair.field, pair => pair.value, StringComparer.Ordinal); + + if (!row.Values.Any(value => !string.IsNullOrWhiteSpace(value))) + { + continue; + } + + var rowIssues = RowIssues(request, artifact, row, sourceRowNumber).ToArray(); + foreach (var rowIssue in rowIssues) + { + issues.Add(rowIssue); + } + + if (rowIssues.Length > 0) + { + continue; + } + + rows.Add(CreateOutputRow(request, artifact, row, sourceRowNumber)); + } + } + + private static IEnumerable RowIssues( + ParserAdapterRunRequest request, + ParserInputArtifact artifact, + IReadOnlyDictionary row, + int sourceRowNumber) + { + foreach (var field in Header.Where(field => RequiredFields.Contains(field) && string.IsNullOrWhiteSpace(row[field]))) + { + yield return Issue( + request, + ParserValidationIssueSeverity.Error, + "DEFRA_DESNZ_CONTENT_MISSING_REQUIRED_FIELD", + $"DEFRA/DESNZ normalized content row is missing required field: {field}.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: field, + context: + [ + new ParserValidationIssueContext("row_number", sourceRowNumber.ToString(CultureInfo.InvariantCulture)), + new ParserValidationIssueContext("field_name", field), + ]); + } + + if (!int.TryParse(row["source_year"], NumberStyles.None, CultureInfo.InvariantCulture, out var sourceYear) || + sourceYear < 1) + { + yield return Issue( + request, + ParserValidationIssueSeverity.Error, + "DEFRA_DESNZ_CONTENT_INVALID_SOURCE_YEAR", + "DEFRA/DESNZ source_year must be a positive integer.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: "source_year", + context: + [ + new ParserValidationIssueContext("row_number", sourceRowNumber.ToString(CultureInfo.InvariantCulture)), + new ParserValidationIssueContext("field_name", "source_year"), + new ParserValidationIssueContext("raw_value", row["source_year"]), + ]); + } + + if (!decimal.TryParse(row["factor_value"], NumberStyles.Number, CultureInfo.InvariantCulture, out _)) + { + yield return Issue( + request, + ParserValidationIssueSeverity.Error, + "DEFRA_DESNZ_CONTENT_INVALID_FACTOR_VALUE", + "DEFRA/DESNZ normalized factor_value must be numeric.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: "factor_value", + context: + [ + new ParserValidationIssueContext("row_number", sourceRowNumber.ToString(CultureInfo.InvariantCulture)), + new ParserValidationIssueContext("field_name", "factor_value"), + new ParserValidationIssueContext("raw_value", row["factor_value"]), + ]); + } + } + + private static ParserNormalizedOutputRow CreateOutputRow( + ParserAdapterRunRequest request, + ParserInputArtifact artifact, + IReadOnlyDictionary row, + int sourceRowNumber) + { + var sourceYear = int.Parse(row["source_year"], CultureInfo.InvariantCulture); + var rowIdentifier = string.Join( + "_", + new[] + { + "defra_desnz", + row["source_year"], + row["source_version"], + row["factor_id"], + $"row_{sourceRowNumber.ToString(CultureInfo.InvariantCulture)}", + }); + var masterId = $"defra_master_{row["source_year"]}_{row["source_version"]}_{row["factor_id"]}"; + var detailId = $"defra_detail_{row["source_year"]}_{row["source_version"]}_{row["factor_id"]}"; + + return new ParserNormalizedOutputRow( + SourceFamily.DefraDesnz, + request.SourceKey, + request.ParserKey, + artifact.ArtifactReference, + rowIdentifier, + sourceRowNumber, + [ + new ParserNormalizedField("source_family", SourceFamily.DefraDesnz.ToWireName()), + new ParserNormalizedField("source_year", row["source_year"]), + new ParserNormalizedField("source_version", row["source_version"]), + new ParserNormalizedField("factor_id", row["factor_id"]), + new ParserNormalizedField("factor_name", row["factor_name"]), + new ParserNormalizedField("factor_value", row["factor_value"]), + new ParserNormalizedField("unit", row["unit"]), + new ParserNormalizedField("category", row["category"]), + new ParserNormalizedField("subcategory", NullIfEmpty(row["subcategory"])), + new ParserNormalizedField("activity", NullIfEmpty(row["activity"])), + new ParserNormalizedField("greenhouse_gas", NullIfEmpty(row["greenhouse_gas"])), + new ParserNormalizedField("provenance_artifact_reference", artifact.ArtifactReference), + new ParserNormalizedField("provenance_checksum_algorithm", artifact.ChecksumAlgorithm), + new ParserNormalizedField("provenance_checksum_value", artifact.ChecksumValue), + new ParserNormalizedField("provenance_row_number", sourceRowNumber.ToString(CultureInfo.InvariantCulture)), + new ParserNormalizedField("provenance", row["provenance"]), + new ParserNormalizedField("source_family_master_id", masterId), + new ParserNormalizedField("source_family_detail_id", detailId), + new ParserNormalizedField("master_external_key", $"{row["source_year"]}:{row["source_version"]}:{row["factor_id"]}"), + new ParserNormalizedField("detail_external_key", $"{row["factor_id"]}:{row["unit"]}:{row["greenhouse_gas"]}"), + ], + reportingYear: sourceYear); + } + + private static ParserAdapterRunResult FailedResult( + ParserAdapterRunRequest request, + IEnumerable rows, + IEnumerable issues) => + new( + request.SourceFamily, + request.SourceKey, + request.ParserKey, + ParserRunStatus.Failed, + request.Artifacts.Select(artifact => artifact.ArtifactReference), + rows, + issues, + request.RunId, + request.CorrelationId, + request.RequestedReportingYear); + + private static ParserValidationIssue Issue( + ParserAdapterRunRequest request, + ParserValidationIssueSeverity severity, + string code, + string message, + string? artifactReference = null, + string? rowIdentifier = null, + int? sourceRowNumber = null, + string? fieldKey = null, + IEnumerable? context = null) => + new( + request.SourceFamily, + request.SourceKey, + request.ParserKey, + severity, + code, + message, + artifactReference, + rowIdentifier, + sourceRowNumber, + fieldKey, + context); + + private static IEnumerable> CsvRows(string content) + { + var row = new List(); + var field = new List(); + var inQuotes = false; + + for (var index = 0; index < content.Length; index++) + { + var current = content[index]; + if (inQuotes) + { + if (current == '"' && index + 1 < content.Length && content[index + 1] == '"') + { + field.Add('"'); + index++; + continue; + } + + if (current == '"') + { + inQuotes = false; + continue; + } + + field.Add(current); + continue; + } + + switch (current) + { + case '"': + inQuotes = true; + break; + case ',': + row.Add(new string(field.ToArray())); + field.Clear(); + break; + case '\r': + break; + case '\n': + row.Add(new string(field.ToArray())); + field.Clear(); + yield return row.ToArray(); + row.Clear(); + break; + default: + field.Add(current); + break; + } + } + + if (field.Count > 0 || row.Count > 0) + { + row.Add(new string(field.ToArray())); + yield return row.ToArray(); + } + } + + private static string? NullIfEmpty(string value) => + string.IsNullOrWhiteSpace(value) ? null : value; +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryBoundary.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryBoundary.cs new file mode 100644 index 00000000..464ed418 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryBoundary.cs @@ -0,0 +1,403 @@ +namespace CarbonOps.Parser.Contracts; + +public static class DefraSourceDiscoveryBoundary +{ + private const string DefraSourceKey = "defra_desnz"; + private const string DiscoveryReferenceUri = "discovery://defra_desnz/homepage"; + private const string ArtifactKind = "discovery"; + + public static DefraSourceDiscoveryRequest CreateRequest() => + new( + SourceFamily.DefraDesnz, + DefraSourceKey, + DiscoveryReferenceUri); + + public static DefraSourceDiscoveryResult CreateResult(DefraSourceDiscoveryRequest? request = null) + { + var activeRequest = request ?? CreateRequest(); + var requestValidation = Validate(activeRequest); + if (!requestValidation.IsValid) + { + return new DefraSourceDiscoveryResult( + DefraSourceDiscoveryStatus.Invalid, + activeRequest, + Array.Empty(), + requestValidation.Issues); + } + + var candidate = new DefraSourceDocumentCandidate( + SourceFamily.DefraDesnz, + DefraSourceKey, + "defra_source_discovery_candidate_001_defra_desnz", + "DEFRA/DESNZ", + activeRequest.DiscoveryReferenceUri, + ArtifactKind, + versionLabel: "dn047_defra_discovery_boundary", + discoveredAtLabel: "runtime_passive_discovery_unavailable"); + var candidateValidation = Validate(candidate); + + return new DefraSourceDiscoveryResult( + candidateValidation.IsValid ? DefraSourceDiscoveryStatus.Declared : DefraSourceDiscoveryStatus.Invalid, + activeRequest, + candidateValidation.IsValid ? new[] { candidate } : Array.Empty(), + candidateValidation.Issues); + } + + public static DefraSourceDiscoveryValidationResult Validate(DefraSourceDiscoveryRequest? request) + { + var issues = new List(); + + if (request is null) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_MISSING_REQUEST", + "request is required.", + "request")); + return new DefraSourceDiscoveryValidationResult(issues); + } + + if (!Enum.IsDefined(request.SourceFamily)) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_INVALID_SOURCE_FAMILY", + "source_family must be a defined source family.", + "source_family")); + } + + ValidateRequiredText( + request.SourceKey, + "source_key", + "DEFRA_SOURCE_DISCOVERY_MISSING_SOURCE_KEY", + "source_key must be a non-empty string.", + issues); + ValidateRequiredText( + request.DiscoveryReferenceUri, + "discovery_reference_uri", + "DEFRA_SOURCE_DISCOVERY_MISSING_REFERENCE_URI", + "discovery_reference_uri must be a non-empty string.", + issues); + + if (request.SourceFamily != SourceFamily.DefraDesnz) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_SOURCE_FAMILY_MISMATCH", + "source_family must be defra_desnz.", + "source_family")); + } + + if (request.SourceKey != DefraSourceKey) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_SOURCE_KEY_MISMATCH", + "source_key must be defra_desnz.", + "source_key")); + } + + if (request.Mode != DefraSourceDiscoveryMode.RuntimePassive) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_UNSUPPORTED_MODE", + "mode must remain runtime_passive.", + "mode")); + } + + ValidateFalse( + request.AllowNetwork, + "allow_network", + "DEFRA_SOURCE_DISCOVERY_NETWORK_NOT_ALLOWED", + "allow_network must be false for this boundary.", + issues); + ValidateFalse( + request.AllowDownload, + "allow_download", + "DEFRA_SOURCE_DISCOVERY_DOWNLOAD_NOT_ALLOWED", + "allow_download must be false for this boundary.", + issues); + ValidateFalse( + request.AllowParse, + "allow_parse", + "DEFRA_SOURCE_DISCOVERY_PARSE_NOT_ALLOWED", + "allow_parse must be false for this boundary.", + issues); + ValidateFalse( + request.AllowDatabaseWrites, + "allow_database_writes", + "DEFRA_SOURCE_DISCOVERY_DATABASE_WRITES_NOT_ALLOWED", + "allow_database_writes must be false for this boundary.", + issues); + ValidateFalse( + request.AllowScheduler, + "allow_scheduler", + "DEFRA_SOURCE_DISCOVERY_SCHEDULER_NOT_ALLOWED", + "allow_scheduler must be false for this boundary.", + issues); + + return new DefraSourceDiscoveryValidationResult(issues); + } + + public static DefraSourceDiscoveryValidationResult Validate(DefraSourceDocumentCandidate? candidate) + { + var issues = new List(); + + if (candidate is null) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING", + "candidate is required.", + "candidate")); + return new DefraSourceDiscoveryValidationResult(issues); + } + + if (!Enum.IsDefined(candidate.SourceFamily)) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_INVALID_SOURCE_FAMILY", + "source_family must be a defined source family.", + "source_family")); + } + + ValidateRequiredText( + candidate.SourceKey, + "source_key", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_SOURCE_KEY", + "source_key must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.CandidateId, + "candidate_id", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_CANDIDATE_ID", + "candidate_id must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.Title, + "title", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_TITLE", + "title must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.ReferenceUri, + "reference_uri", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_REFERENCE_URI", + "reference_uri must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.ArtifactKind, + "artifact_kind", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_ARTIFACT_KIND", + "artifact_kind must be a non-empty string.", + issues); + ValidateOptionalText( + candidate.ContentType, + "content_type", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_BLANK_CONTENT_TYPE", + "content_type must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.Extension, + "extension", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_BLANK_EXTENSION", + "extension must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.ChecksumSha256, + "checksum_sha256", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_BLANK_CHECKSUM_SHA256", + "checksum_sha256 must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.VersionLabel, + "version_label", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_BLANK_VERSION_LABEL", + "version_label must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.DiscoveredAtLabel, + "discovered_at_label", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_BLANK_DISCOVERED_AT_LABEL", + "discovered_at_label must be non-empty when provided.", + issues); + ValidateOptionalPositiveInt( + candidate.DocumentYear, + "document_year", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_INVALID_DOCUMENT_YEAR", + "document_year must be a positive integer when provided.", + issues); + ValidateOptionalPositiveInt( + candidate.ReportingYear, + "reporting_year", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_INVALID_REPORTING_YEAR", + "reporting_year must be a positive integer when provided.", + issues); + + if (candidate.SourceFamily != SourceFamily.DefraDesnz) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_SOURCE_FAMILY_MISMATCH", + "source_family must match the DEFRA source family.", + "source_family")); + } + + if (candidate.SourceKey != DefraSourceKey) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_SOURCE_KEY_MISMATCH", + "source_key must match the DEFRA source key.", + "source_key")); + } + + if (candidate.ArtifactKind != ArtifactKind) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_ARTIFACT_KIND_MISMATCH", + "artifact_kind must match the DEFRA expected format.", + "artifact_kind")); + } + + if (candidate.Status != DefraSourceDiscoveryStatus.Declared) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_UNSUPPORTED_STATUS", + "candidate status must remain declared.", + "status")); + } + + if (candidate.DownloadAllowed) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_DOWNLOAD_NOT_ALLOWED", + "download_allowed must be false for this boundary.", + "download_allowed")); + } + + return new DefraSourceDiscoveryValidationResult(issues); + } + + public static DefraSourceDiscoveryValidationResult Validate(DefraSourceDiscoveryResult? result) + { + var issues = new List(); + + if (result is null) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_RESULT_MISSING", + "result is required.", + "result")); + return new DefraSourceDiscoveryValidationResult(issues); + } + + issues.AddRange(Validate(result.Request).Issues); + + if (!Enum.IsDefined(result.Status)) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_RESULT_INVALID_STATUS", + "status must be a defined DEFRA source discovery status.", + "status")); + } + + foreach (var (fieldName, value) in new[] + { + ("no_network", result.NoNetwork), + ("no_download", result.NoDownload), + ("no_parse", result.NoParse), + ("no_database_writes", result.NoDatabaseWrites), + ("no_sql", result.NoSql), + ("no_scheduler", result.NoScheduler), + }) + { + if (!value) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED", + $"{fieldName} must remain true.", + fieldName)); + } + } + + for (var index = 0; index < result.Candidates.Count; index++) + { + foreach (var issue in Validate(result.Candidates[index]).Issues) + { + issues.Add(issue with { FieldName = $"candidates[{index + 1}].{issue.FieldName}" }); + } + } + + if (result.Status == DefraSourceDiscoveryStatus.Declared && result.Issues.Count > 0) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + "declared result status must not include issue metadata.", + "issues")); + } + + if (result.Status == DefraSourceDiscoveryStatus.Declared && issues.Count > 0) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + "declared result status requires valid metadata.", + "status")); + } + + if (result.Status == DefraSourceDiscoveryStatus.Invalid && result.Issues.Count == 0) + { + issues.Add(new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_RESULT_MISSING_INVALID_ISSUES", + "invalid result status requires issue metadata.", + "issues")); + } + + return new DefraSourceDiscoveryValidationResult(issues); + } + + private static void ValidateRequiredText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (string.IsNullOrWhiteSpace(value)) + { + issues.Add(new DefraSourceDiscoveryIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is not null && string.IsNullOrWhiteSpace(value)) + { + issues.Add(new DefraSourceDiscoveryIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalPositiveInt( + int? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is <= 0) + { + issues.Add(new DefraSourceDiscoveryIssue(code, message, fieldName)); + } + } + + private static void ValidateFalse( + bool value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value) + { + issues.Add(new DefraSourceDiscoveryIssue(code, message, fieldName)); + } + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryIssue.cs new file mode 100644 index 00000000..9eba1174 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDiscoveryIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryMode.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryMode.cs new file mode 100644 index 00000000..feb66864 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryMode.cs @@ -0,0 +1,6 @@ +namespace CarbonOps.Parser.Contracts; + +public enum DefraSourceDiscoveryMode +{ + RuntimePassive = 0, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryRequest.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryRequest.cs new file mode 100644 index 00000000..e79658ba --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryRequest.cs @@ -0,0 +1,44 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDiscoveryRequest +{ + public SourceFamily SourceFamily { get; } + + public string SourceKey { get; } + + public string DiscoveryReferenceUri { get; } + + public DefraSourceDiscoveryMode Mode { get; } + + public bool AllowNetwork { get; } + + public bool AllowDownload { get; } + + public bool AllowParse { get; } + + public bool AllowDatabaseWrites { get; } + + public bool AllowScheduler { get; } + + public DefraSourceDiscoveryRequest( + SourceFamily sourceFamily, + string sourceKey, + string discoveryReferenceUri, + DefraSourceDiscoveryMode mode = DefraSourceDiscoveryMode.RuntimePassive, + bool allowNetwork = false, + bool allowDownload = false, + bool allowParse = false, + bool allowDatabaseWrites = false, + bool allowScheduler = false) + { + SourceFamily = sourceFamily; + SourceKey = sourceKey; + DiscoveryReferenceUri = discoveryReferenceUri; + Mode = mode; + AllowNetwork = allowNetwork; + AllowDownload = allowDownload; + AllowParse = allowParse; + AllowDatabaseWrites = allowDatabaseWrites; + AllowScheduler = allowScheduler; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryResult.cs new file mode 100644 index 00000000..70f4e171 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryResult.cs @@ -0,0 +1,55 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDiscoveryResult +{ + public DefraSourceDiscoveryStatus Status { get; } + + public DefraSourceDiscoveryRequest Request { get; } + + public IReadOnlyList Candidates { get; } + + public IReadOnlyList Issues { get; } + + public bool NoNetwork { get; } + + public bool NoDownload { get; } + + public bool NoParse { get; } + + public bool NoDatabaseWrites { get; } + + public bool NoSql { get; } + + public bool NoScheduler { get; } + + public int CandidateCount => Candidates.Count; + + public IReadOnlyList CandidateIds { get; } + + public DefraSourceDiscoveryResult( + DefraSourceDiscoveryStatus status, + DefraSourceDiscoveryRequest request, + IEnumerable candidates, + IEnumerable? issues = null, + bool noNetwork = true, + bool noDownload = true, + bool noParse = true, + bool noDatabaseWrites = true, + bool noSql = true, + bool noScheduler = true) + { + var candidateSnapshot = candidates.ToArray(); + + Status = status; + Request = request; + Candidates = Array.AsReadOnly(candidateSnapshot); + Issues = Array.AsReadOnly((issues ?? Array.Empty()).ToArray()); + NoNetwork = noNetwork; + NoDownload = noDownload; + NoParse = noParse; + NoDatabaseWrites = noDatabaseWrites; + NoSql = noSql; + NoScheduler = noScheduler; + CandidateIds = Array.AsReadOnly(candidateSnapshot.Select(candidate => candidate.CandidateId).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryStatus.cs new file mode 100644 index 00000000..d08505f3 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryStatus.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public enum DefraSourceDiscoveryStatus +{ + Declared = 0, + Invalid = 1, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryValidationResult.cs new file mode 100644 index 00000000..c2b94ac0 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDiscoveryValidationResult.cs @@ -0,0 +1,13 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDiscoveryValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public DefraSourceDiscoveryValidationResult(IEnumerable? issues = null) + { + Issues = Array.AsReadOnly((issues ?? Array.Empty()).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDocumentCandidate.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDocumentCandidate.cs new file mode 100644 index 00000000..f1f0731b --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDocumentCandidate.cs @@ -0,0 +1,68 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDocumentCandidate +{ + public SourceFamily SourceFamily { get; } + + public string SourceKey { get; } + + public string CandidateId { get; } + + public string Title { get; } + + public string ReferenceUri { get; } + + public string ArtifactKind { get; } + + public DefraSourceDiscoveryStatus Status { get; } + + public int? DocumentYear { get; } + + public int? ReportingYear { get; } + + public string? ContentType { get; } + + public string? Extension { get; } + + public string? ChecksumSha256 { get; } + + public string? VersionLabel { get; } + + public string? DiscoveredAtLabel { get; } + + public bool DownloadAllowed { get; } + + public DefraSourceDocumentCandidate( + SourceFamily sourceFamily, + string sourceKey, + string candidateId, + string title, + string referenceUri, + string artifactKind, + DefraSourceDiscoveryStatus status = DefraSourceDiscoveryStatus.Declared, + int? documentYear = null, + int? reportingYear = null, + string? contentType = null, + string? extension = null, + string? checksumSha256 = null, + string? versionLabel = null, + string? discoveredAtLabel = null, + bool downloadAllowed = false) + { + SourceFamily = sourceFamily; + SourceKey = sourceKey; + CandidateId = candidateId; + Title = title; + ReferenceUri = referenceUri; + ArtifactKind = artifactKind; + Status = status; + DocumentYear = documentYear; + ReportingYear = reportingYear; + ContentType = contentType; + Extension = extension; + ChecksumSha256 = checksumSha256; + VersionLabel = versionLabel; + DiscoveredAtLabel = discoveredAtLabel; + DownloadAllowed = downloadAllowed; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionBoundary.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionBoundary.cs new file mode 100644 index 00000000..86bf178c --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionBoundary.cs @@ -0,0 +1,728 @@ +using System.Security.Cryptography; + +namespace CarbonOps.Parser.Contracts; + +public static class DefraSourceDownloadExecutionBoundary +{ + private const string DefraSourceKey = "defra_desnz"; + + public static DefraSourceDownloadExecutionRequest CreateRequest( + DefraSourceDocumentCandidate candidate, + string targetRoot, + string targetRelativePath, + bool allowDownloadExecution = false, + bool allowFileWrite = false, + bool allowNetwork = false, + bool allowOverwrite = false) => + new( + candidate.SourceFamily, + candidate.SourceKey, + candidate.CandidateId, + candidate.Title, + candidate.ReferenceUri, + candidate.ArtifactKind, + targetRoot, + targetRelativePath, + candidate.DownloadAllowed, + allowDownloadExecution, + allowFileWrite, + allowNetwork, + allowOverwrite, + contentType: candidate.ContentType, + extension: candidate.Extension, + expectedChecksumSha256: candidate.ChecksumSha256, + documentYear: candidate.DocumentYear, + reportingYear: candidate.ReportingYear, + versionLabel: candidate.VersionLabel); + + public static DefraSourceDownloadExecutionValidationResult Validate( + DefraSourceDownloadExecutionRequest? request) + { + var issues = new List(); + + if (request is null) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_MISSING_REQUEST", + "request is required.", + "request")); + return new DefraSourceDownloadExecutionValidationResult(issues); + } + + if (!Enum.IsDefined(request.SourceFamily)) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_INVALID_SOURCE_FAMILY", + "source_family must be a defined source family.", + "source_family")); + } + + ValidateRequiredText( + request.SourceKey, + "source_key", + "DEFRA_SOURCE_DOWNLOAD_MISSING_SOURCE_KEY", + "source_key must be a non-empty string.", + issues); + ValidateRequiredText( + request.CandidateId, + "candidate_id", + "DEFRA_SOURCE_DOWNLOAD_MISSING_CANDIDATE_ID", + "candidate_id must be a non-empty string.", + issues); + ValidateRequiredText( + request.CandidateTitle, + "candidate_title", + "DEFRA_SOURCE_DOWNLOAD_MISSING_CANDIDATE_TITLE", + "candidate_title must be a non-empty string.", + issues); + ValidateRequiredText( + request.SourceReferenceUri, + "source_reference_uri", + "DEFRA_SOURCE_DOWNLOAD_MISSING_SOURCE_REFERENCE_URI", + "source_reference_uri must be a non-empty string.", + issues); + ValidateRequiredText( + request.ArtifactKind, + "artifact_kind", + "DEFRA_SOURCE_DOWNLOAD_MISSING_ARTIFACT_KIND", + "artifact_kind must be a non-empty string.", + issues); + ValidateRequiredText( + request.TargetRoot, + "target_root", + "DEFRA_SOURCE_DOWNLOAD_MISSING_TARGET_ROOT", + "target_root must be a non-empty string.", + issues); + ValidateRequiredText( + request.TargetRelativePath, + "target_relative_path", + "DEFRA_SOURCE_DOWNLOAD_MISSING_TARGET_RELATIVE_PATH", + "target_relative_path must be a non-empty string.", + issues); + ValidateOptionalText( + request.ContentType, + "content_type", + "DEFRA_SOURCE_DOWNLOAD_BLANK_CONTENT_TYPE", + "content_type must be non-empty when provided.", + issues); + ValidateOptionalText( + request.Extension, + "extension", + "DEFRA_SOURCE_DOWNLOAD_BLANK_EXTENSION", + "extension must be non-empty when provided.", + issues); + ValidateOptionalText( + request.ExpectedChecksumSha256, + "expected_checksum_sha256", + "DEFRA_SOURCE_DOWNLOAD_BLANK_EXPECTED_CHECKSUM_SHA256", + "expected_checksum_sha256 must be non-empty when provided.", + issues); + ValidateOptionalText( + request.VersionLabel, + "version_label", + "DEFRA_SOURCE_DOWNLOAD_BLANK_VERSION_LABEL", + "version_label must be non-empty when provided.", + issues); + ValidateOptionalPositiveInt( + request.DocumentYear, + "document_year", + "DEFRA_SOURCE_DOWNLOAD_INVALID_DOCUMENT_YEAR", + "document_year must be a positive integer when provided.", + issues); + ValidateOptionalPositiveInt( + request.ReportingYear, + "reporting_year", + "DEFRA_SOURCE_DOWNLOAD_INVALID_REPORTING_YEAR", + "reporting_year must be a positive integer when provided.", + issues); + + if (request.SourceFamily != SourceFamily.DefraDesnz) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_SOURCE_FAMILY_MISMATCH", + "source_family must be defra_desnz.", + "source_family")); + } + + if (request.SourceKey != DefraSourceKey) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_SOURCE_KEY_MISMATCH", + "source_key must be defra_desnz.", + "source_key")); + } + + ValidateTrue( + request.CandidateDownloadAllowed, + "candidate_download_allowed", + "DEFRA_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE", + "candidate metadata must explicitly allow download execution.", + issues); + ValidateTrue( + request.AllowDownloadExecution, + "allow_download_execution", + "DEFRA_SOURCE_DOWNLOAD_EXECUTION_NOT_ALLOWED", + "allow_download_execution must be true.", + issues); + ValidateTrue( + request.AllowFileWrite, + "allow_file_write", + "DEFRA_SOURCE_DOWNLOAD_FILE_WRITE_NOT_ALLOWED", + "allow_file_write must be true.", + issues); + ValidateFalse( + request.AllowParse, + "allow_parse", + "DEFRA_SOURCE_DOWNLOAD_PARSE_NOT_ALLOWED", + "allow_parse must be false for this boundary.", + issues); + ValidateFalse( + request.AllowDatabaseWrites, + "allow_database_writes", + "DEFRA_SOURCE_DOWNLOAD_DATABASE_WRITES_NOT_ALLOWED", + "allow_database_writes must be false for this boundary.", + issues); + ValidateFalse( + request.AllowScheduler, + "allow_scheduler", + "DEFRA_SOURCE_DOWNLOAD_SCHEDULER_NOT_ALLOWED", + "allow_scheduler must be false for this boundary.", + issues); + + ValidateSourceReferenceUri(request, issues); + ValidateTargetPaths(request, issues); + + return new DefraSourceDownloadExecutionValidationResult(issues); + } + + public static DefraSourceDownloadExecutionResult Execute( + DefraSourceDownloadExecutionRequest request, + Func transport) + { + var validation = Validate(request); + if (!validation.IsValid) + { + return new DefraSourceDownloadExecutionResult( + DefraSourceDownloadExecutionStatus.Blocked, + request, + issues: validation.Issues); + } + + var safeTarget = PrepareSafeTargetPath(request); + if (!safeTarget.Validation.IsValid) + { + return new DefraSourceDownloadExecutionResult( + DefraSourceDownloadExecutionStatus.Blocked, + request, + issues: safeTarget.Validation.Issues); + } + + DefraSourceDownloadTransportResponse response; + try + { + response = transport(request.SourceReferenceUri); + } + catch (Exception error) + { + return new DefraSourceDownloadExecutionResult( + DefraSourceDownloadExecutionStatus.Failed, + request, + issues: + [ + new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_TRANSPORT_FAILED", + $"transport failed: {error.Message}", + "source_reference_uri"), + ]); + } + + var responseValidation = ValidateTransportResponse(response); + if (!responseValidation.IsValid) + { + return new DefraSourceDownloadExecutionResult( + DefraSourceDownloadExecutionStatus.Failed, + request, + issues: responseValidation.Issues); + } + + var checksum = Convert.ToHexString(SHA256.HashData(response.Content)).ToLowerInvariant(); + if (request.ExpectedChecksumSha256 is not null + && !string.Equals(checksum, request.ExpectedChecksumSha256, StringComparison.OrdinalIgnoreCase)) + { + return new DefraSourceDownloadExecutionResult( + DefraSourceDownloadExecutionStatus.Failed, + request, + issues: + [ + new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH", + "downloaded content checksum did not match expected value.", + "expected_checksum_sha256"), + ]); + } + + safeTarget = PrepareSafeTargetPath(request); + if (!safeTarget.Validation.IsValid) + { + return new DefraSourceDownloadExecutionResult( + DefraSourceDownloadExecutionStatus.Blocked, + request, + issues: safeTarget.Validation.Issues); + } + + try + { + WriteContentToSafeTarget(safeTarget.TargetPath, response.Content, request.AllowOverwrite); + } + catch (IOException error) when (File.Exists(safeTarget.TargetPath) && !request.AllowOverwrite) + { + return WriteFailed(request, "DEFRA_SOURCE_DOWNLOAD_TARGET_EXISTS", error); + } + catch (Exception error) + { + return WriteFailed(request, "DEFRA_SOURCE_DOWNLOAD_WRITE_FAILED", error); + } + + var artifact = new DefraSourceDownloadedArtifact( + request.SourceFamily, + request.SourceKey, + request.CandidateId, + $"defra_source_download_artifact_{request.CandidateId}", + request.ArtifactKind, + request.SourceReferenceUri, + safeTarget.TargetPath, + Path.GetFileName(safeTarget.TargetPath), + checksum, + response.Content.LongLength, + response.ContentType ?? request.ContentType, + request.Extension, + response.FinalUri, + request.DocumentYear, + request.ReportingYear, + request.VersionLabel); + + return new DefraSourceDownloadExecutionResult( + DefraSourceDownloadExecutionStatus.Downloaded, + request, + artifact); + } + + public static DefraSourceDownloadExecutionValidationResult Validate( + DefraSourceDownloadExecutionResult? result) + { + var issues = new List(); + + if (result is null) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_RESULT_MISSING", + "result is required.", + "result")); + return new DefraSourceDownloadExecutionValidationResult(issues); + } + + issues.AddRange(Validate(result.Request).Issues); + + if (!Enum.IsDefined(result.Status)) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_RESULT_INVALID_STATUS", + "status must be a defined DEFRA source download execution status.", + "status")); + } + + foreach (var (fieldName, value) in new[] + { + ("no_parse", result.NoParse), + ("no_database_writes", result.NoDatabaseWrites), + ("no_sql", result.NoSql), + ("no_scheduler", result.NoScheduler), + }) + { + if (!value) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED", + $"{fieldName} must remain true.", + fieldName)); + } + } + + if (result.Status == DefraSourceDownloadExecutionStatus.Downloaded && result.Artifact is null) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_RESULT_MISSING_ARTIFACT", + "downloaded results require artifact metadata.", + "artifact")); + } + else if (result.Status != DefraSourceDownloadExecutionStatus.Downloaded && result.Artifact is not null) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_RESULT_UNEXPECTED_ARTIFACT", + "non-downloaded results must not include artifact metadata.", + "artifact")); + } + + if (result.Status != DefraSourceDownloadExecutionStatus.Downloaded && result.Issues.Count == 0) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_RESULT_MISSING_ISSUES", + "blocked or failed results require issue metadata.", + "issues")); + } + + return new DefraSourceDownloadExecutionValidationResult(issues); + } + + private static DefraSourceDownloadExecutionResult WriteFailed( + DefraSourceDownloadExecutionRequest request, + string code, + Exception error) => + new( + DefraSourceDownloadExecutionStatus.Failed, + request, + issues: + [ + new DefraSourceDownloadExecutionIssue( + code, + $"target write failed: {error.Message}", + "target_relative_path"), + ]); + + private static DefraSourceDownloadExecutionValidationResult ValidateTransportResponse( + DefraSourceDownloadTransportResponse? response) + { + var issues = new List(); + + if (response is null) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING", + "transport response is required.", + "transport")); + return new DefraSourceDownloadExecutionValidationResult(issues); + } + + if (response.Content is null) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT", + "transport response content is required.", + "content")); + } + else if (response.Content.Length == 0) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_EMPTY_CONTENT", + "transport response content must not be empty.", + "content")); + } + + ValidateOptionalText( + response.ContentType, + "content_type", + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "response content_type must be non-empty when provided.", + issues); + ValidateOptionalText( + response.FinalUri, + "final_uri", + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + "response final_uri must be non-empty when provided.", + issues); + + return new DefraSourceDownloadExecutionValidationResult(issues); + } + + private static (string TargetPath, DefraSourceDownloadExecutionValidationResult Validation) PrepareSafeTargetPath( + DefraSourceDownloadExecutionRequest request) + { + var issues = new List(); + + string root; + string targetPath; + try + { + root = Path.GetFullPath(request.TargetRoot); + targetPath = Path.GetFullPath(Path.Combine(root, request.TargetRelativePath)); + } + catch (Exception error) when (error is ArgumentException or NotSupportedException or PathTooLongException) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_TARGET_PATH_UNRESOLVED", + "target path could not be resolved safely.", + "target_relative_path")); + return (string.Empty, new DefraSourceDownloadExecutionValidationResult(issues)); + } + + if (!IsPathInsideRoot(root, targetPath)) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE", + "target_relative_path must stay within target_root.", + "target_relative_path")); + } + + if (ContainsExistingSymlink(root, targetPath)) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE", + "target path must not traverse an existing symbolic link.", + "target_relative_path")); + } + + if (File.Exists(targetPath) && !request.AllowOverwrite) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_TARGET_EXISTS", + "target path already exists and allow_overwrite is false.", + "target_relative_path")); + } + + return (targetPath, new DefraSourceDownloadExecutionValidationResult(issues)); + } + + private static void WriteContentToSafeTarget(string targetPath, byte[] content, bool allowOverwrite) + { + var parent = Path.GetDirectoryName(targetPath); + if (!string.IsNullOrWhiteSpace(parent)) + { + Directory.CreateDirectory(parent); + } + + if (IsSymlink(targetPath)) + { + throw new IOException("target path is a symbolic link."); + } + + var mode = allowOverwrite ? FileMode.Create : FileMode.CreateNew; + using var stream = new FileStream(targetPath, mode, FileAccess.Write, FileShare.None); + stream.Write(content, 0, content.Length); + } + + private static bool IsPathInsideRoot(string root, string targetPath) + { + var rootWithSeparator = Path.EndsInDirectorySeparator(root) + ? root + : root + Path.DirectorySeparatorChar; + + return targetPath.StartsWith(rootWithSeparator, StringComparison.Ordinal) + || string.Equals(root, targetPath, StringComparison.Ordinal); + } + + private static bool ContainsExistingSymlink(string root, string targetPath) + { + var relative = Path.GetRelativePath(root, targetPath); + if (relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative)) + { + return false; + } + + if (IsSymlink(root)) + { + return true; + } + + var current = root; + foreach (var segment in relative.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)) + { + if (string.IsNullOrWhiteSpace(segment)) + { + continue; + } + + current = Path.Combine(current, segment); + if ((Directory.Exists(current) || File.Exists(current)) && IsSymlink(current)) + { + return true; + } + } + + return false; + } + + private static bool IsSymlink(string path) + { + try + { + return (File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0; + } + catch (FileNotFoundException) + { + return false; + } + catch (DirectoryNotFoundException) + { + return false; + } + } + + private static void ValidateSourceReferenceUri( + DefraSourceDownloadExecutionRequest request, + ICollection issues) + { + if (string.IsNullOrWhiteSpace(request.SourceReferenceUri)) + { + return; + } + + if (!Uri.TryCreate(request.SourceReferenceUri, UriKind.Absolute, out var uri)) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + request.SourceReferenceUri.Contains("://", StringComparison.Ordinal) + ? "DEFRA_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI" + : "DEFRA_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME", + request.SourceReferenceUri.Contains("://", StringComparison.Ordinal) + ? "source_reference_uri must be a well-formed URI." + : "source_reference_uri must include a URI scheme.", + "source_reference_uri")); + return; + } + + if ((uri.Scheme == Uri.UriSchemeHttps || uri.Scheme == Uri.UriSchemeHttp) + && string.IsNullOrWhiteSpace(uri.Host)) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + "source_reference_uri must be a well-formed URI.", + "source_reference_uri")); + } + else if (uri.Scheme == "discovery") + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE", + "discovery references are not direct download references.", + "source_reference_uri")); + } + else if (uri.Scheme == Uri.UriSchemeHttps && !request.AllowNetwork) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_NETWORK_NOT_ALLOWED", + "allow_network must be true for https source references.", + "source_reference_uri")); + } + else if (uri.Scheme == Uri.UriSchemeHttp) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_INSECURE_HTTP_NOT_ALLOWED", + "http source references are not allowed.", + "source_reference_uri")); + } + else if (uri.Scheme is not "mock" and not "memory") + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI", + "source_reference_uri must use an allowed execution scheme.", + "source_reference_uri")); + } + } + + private static void ValidateTargetPaths( + DefraSourceDownloadExecutionRequest request, + ICollection issues) + { + if (!string.IsNullOrWhiteSpace(request.TargetRoot) && !Path.IsPathFullyQualified(request.TargetRoot)) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_ABSOLUTE", + "target_root must be an absolute path.", + "target_root")); + } + + if (string.IsNullOrWhiteSpace(request.TargetRelativePath)) + { + return; + } + + if (Path.IsPathFullyQualified(request.TargetRelativePath)) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_ABSOLUTE", + "target_relative_path must be relative.", + "target_relative_path")); + } + + if (Uri.TryCreate(request.TargetRelativePath, UriKind.Absolute, out var targetUri) + && !string.IsNullOrWhiteSpace(targetUri.Scheme)) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_URI", + "target_relative_path must not be a URI.", + "target_relative_path")); + } + + var segments = request.TargetRelativePath.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + if (segments.Any(segment => segment == "..")) + { + issues.Add(new DefraSourceDownloadExecutionIssue( + "DEFRA_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE", + "target_relative_path must not contain parent traversal.", + "target_relative_path")); + } + } + + private static void ValidateRequiredText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (string.IsNullOrWhiteSpace(value)) + { + issues.Add(new DefraSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is not null && string.IsNullOrWhiteSpace(value)) + { + issues.Add(new DefraSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalPositiveInt( + int? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is <= 0) + { + issues.Add(new DefraSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateTrue( + bool value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (!value) + { + issues.Add(new DefraSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateFalse( + bool value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value) + { + issues.Add(new DefraSourceDownloadExecutionIssue(code, message, fieldName)); + } + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionIssue.cs new file mode 100644 index 00000000..6197868f --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDownloadExecutionIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionRequest.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionRequest.cs new file mode 100644 index 00000000..ce7ff0ca --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionRequest.cs @@ -0,0 +1,96 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDownloadExecutionRequest +{ + public SourceFamily SourceFamily { get; init; } + + public string SourceKey { get; init; } + + public string CandidateId { get; init; } + + public string CandidateTitle { get; init; } + + public string SourceReferenceUri { get; init; } + + public string ArtifactKind { get; init; } + + public string TargetRoot { get; init; } + + public string TargetRelativePath { get; init; } + + public bool CandidateDownloadAllowed { get; init; } + + public bool AllowDownloadExecution { get; init; } + + public bool AllowFileWrite { get; init; } + + public bool AllowNetwork { get; init; } + + public bool AllowOverwrite { get; init; } + + public bool AllowParse { get; init; } + + public bool AllowDatabaseWrites { get; init; } + + public bool AllowScheduler { get; init; } + + public string? ContentType { get; init; } + + public string? Extension { get; init; } + + public string? ExpectedChecksumSha256 { get; init; } + + public int? DocumentYear { get; init; } + + public int? ReportingYear { get; init; } + + public string? VersionLabel { get; init; } + + public DefraSourceDownloadExecutionRequest( + SourceFamily sourceFamily, + string sourceKey, + string candidateId, + string candidateTitle, + string sourceReferenceUri, + string artifactKind, + string targetRoot, + string targetRelativePath, + bool candidateDownloadAllowed = false, + bool allowDownloadExecution = false, + bool allowFileWrite = false, + bool allowNetwork = false, + bool allowOverwrite = false, + bool allowParse = false, + bool allowDatabaseWrites = false, + bool allowScheduler = false, + string? contentType = null, + string? extension = null, + string? expectedChecksumSha256 = null, + int? documentYear = null, + int? reportingYear = null, + string? versionLabel = null) + { + SourceFamily = sourceFamily; + SourceKey = sourceKey; + CandidateId = candidateId; + CandidateTitle = candidateTitle; + SourceReferenceUri = sourceReferenceUri; + ArtifactKind = artifactKind; + TargetRoot = targetRoot; + TargetRelativePath = targetRelativePath; + CandidateDownloadAllowed = candidateDownloadAllowed; + AllowDownloadExecution = allowDownloadExecution; + AllowFileWrite = allowFileWrite; + AllowNetwork = allowNetwork; + AllowOverwrite = allowOverwrite; + AllowParse = allowParse; + AllowDatabaseWrites = allowDatabaseWrites; + AllowScheduler = allowScheduler; + ContentType = contentType; + Extension = extension; + ExpectedChecksumSha256 = expectedChecksumSha256; + DocumentYear = documentYear; + ReportingYear = reportingYear; + VersionLabel = versionLabel; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionResult.cs new file mode 100644 index 00000000..689b61b5 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionResult.cs @@ -0,0 +1,42 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDownloadExecutionResult +{ + public DefraSourceDownloadExecutionStatus Status { get; init; } + + public DefraSourceDownloadExecutionRequest Request { get; init; } + + public DefraSourceDownloadedArtifact? Artifact { get; init; } + + public IReadOnlyList Issues { get; init; } + + public bool NoParse { get; init; } + + public bool NoDatabaseWrites { get; init; } + + public bool NoSql { get; init; } + + public bool NoScheduler { get; init; } + + public bool Downloaded => Status == DefraSourceDownloadExecutionStatus.Downloaded; + + public DefraSourceDownloadExecutionResult( + DefraSourceDownloadExecutionStatus status, + DefraSourceDownloadExecutionRequest request, + DefraSourceDownloadedArtifact? artifact = null, + IEnumerable? issues = null, + bool noParse = true, + bool noDatabaseWrites = true, + bool noSql = true, + bool noScheduler = true) + { + Status = status; + Request = request; + Artifact = artifact; + Issues = (issues ?? Array.Empty()).ToArray(); + NoParse = noParse; + NoDatabaseWrites = noDatabaseWrites; + NoSql = noSql; + NoScheduler = noScheduler; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionStatus.cs new file mode 100644 index 00000000..8182b236 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionStatus.cs @@ -0,0 +1,8 @@ +namespace CarbonOps.Parser.Contracts; + +public enum DefraSourceDownloadExecutionStatus +{ + Blocked = 0, + Downloaded = 1, + Failed = 2, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionValidationResult.cs new file mode 100644 index 00000000..d3efbab2 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadExecutionValidationResult.cs @@ -0,0 +1,14 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDownloadExecutionValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public DefraSourceDownloadExecutionValidationResult( + IEnumerable? issues = null) + { + Issues = (issues ?? Array.Empty()).ToArray(); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadTransportResponse.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadTransportResponse.cs new file mode 100644 index 00000000..c1574387 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadTransportResponse.cs @@ -0,0 +1,6 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDownloadTransportResponse( + byte[] Content, + string? ContentType = null, + string? FinalUri = null); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadedArtifact.cs b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadedArtifact.cs new file mode 100644 index 00000000..d09edec0 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/DefraSourceDownloadedArtifact.cs @@ -0,0 +1,19 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record DefraSourceDownloadedArtifact( + SourceFamily SourceFamily, + string SourceKey, + string CandidateId, + string ArtifactId, + string ArtifactKind, + string SourceReferenceUri, + string LocalPath, + string OriginalFilename, + string ChecksumSha256, + long SizeBytes, + string? ContentType = null, + string? Extension = null, + string? FinalUri = null, + int? DocumentYear = null, + int? ReportingYear = null, + string? VersionLabel = null); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgProtocolNormalizedContentParser.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgProtocolNormalizedContentParser.cs new file mode 100644 index 00000000..85292dcc --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgProtocolNormalizedContentParser.cs @@ -0,0 +1,407 @@ +using System.Globalization; + +namespace CarbonOps.Parser.Contracts; + +public static class GhgProtocolNormalizedContentParser +{ + public static IReadOnlyList Header { get; } = Array.AsReadOnly(new[] + { + "record_type", + "source_year", + "source_version", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", + "subcategory", + "scope", + "gas", + "provenance_note", + }); + + private static readonly IReadOnlySet RequiredFields = new HashSet( + new[] + { + "source_year", + "source_version", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", + }, + StringComparer.Ordinal); + + public static ParserAdapterRunResult Parse( + ParserAdapterRunRequest request, + IReadOnlyDictionary contentByArtifactReference) + { + var requestValidation = request.Validate(); + if (!requestValidation.IsValid) + { + return FailedResult( + request, + [], + requestValidation.Errors.Select(error => Issue( + request, + ParserValidationIssueSeverity.Error, + "GHG_PROTOCOL_CONTENT_INVALID_REQUEST", + error))); + } + + if (request.SourceFamily != SourceFamily.GhgProtocol) + { + return FailedResult( + request, + [], + [ + Issue( + request, + ParserValidationIssueSeverity.Error, + "GHG_PROTOCOL_CONTENT_SOURCE_FAMILY_MISMATCH", + "GHG Protocol content parser only accepts ghg_protocol source family.", + fieldKey: "source_family"), + ]); + } + + var rows = new List(); + var issues = new List(); + + foreach (var artifact in request.Artifacts) + { + if (!contentByArtifactReference.TryGetValue(artifact.ArtifactReference, out var content)) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Error, + "GHG_PROTOCOL_CONTENT_MISSING_ARTIFACT_CONTENT", + "GHG Protocol parser content was not provided for an input artifact.", + artifact.ArtifactReference, + fieldKey: "artifact_reference")); + continue; + } + + ParseArtifact(request, artifact, content, rows, issues); + } + + if (issues.Any(issue => issue.Severity == ParserValidationIssueSeverity.Error)) + { + return FailedResult(request, rows, issues); + } + + if (rows.Count == 0) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Warning, + "GHG_PROTOCOL_CONTENT_NO_RECORDS", + "GHG Protocol normalized content included no parseable emission factor rows.", + fieldKey: "content")); + } + + return new ParserAdapterRunResult( + request.SourceFamily, + request.SourceKey, + request.ParserKey, + ParserRunStatus.Completed, + request.Artifacts.Select(artifact => artifact.ArtifactReference), + rows, + issues, + request.RunId, + request.CorrelationId, + request.RequestedReportingYear); + } + + private static void ParseArtifact( + ParserAdapterRunRequest request, + ParserInputArtifact artifact, + string content, + ICollection rows, + ICollection issues) + { + var csvRows = CsvRows(content).ToArray(); + if (csvRows.Length == 0) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Warning, + "GHG_PROTOCOL_CONTENT_EMPTY", + "GHG Protocol content input did not include parseable content.", + artifact.ArtifactReference, + fieldKey: "content")); + return; + } + + if (!csvRows[0].SequenceEqual(Header, StringComparer.Ordinal)) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Error, + "GHG_PROTOCOL_CONTENT_INVALID_HEADER", + "GHG Protocol normalized content header must match the declared parser contract.", + artifact.ArtifactReference, + sourceRowNumber: 1, + fieldKey: "header")); + return; + } + + for (var rowIndex = 1; rowIndex < csvRows.Length; rowIndex++) + { + var sourceRowNumber = rowIndex + 1; + var values = csvRows[rowIndex]; + if (values.Count == 1 && string.IsNullOrWhiteSpace(values[0])) + { + continue; + } + + if (values.Count != Header.Count) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Error, + "GHG_PROTOCOL_CONTENT_INVALID_ROW", + "GHG Protocol content row has an unexpected column count.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: $"row[{sourceRowNumber}]")); + continue; + } + + var row = Header + .Select((field, index) => new { field, value = values[index].Trim() }) + .ToDictionary(pair => pair.field, pair => pair.value, StringComparer.Ordinal); + + if (!row.Values.Any(value => !string.IsNullOrWhiteSpace(value))) + { + continue; + } + + if (row["record_type"] != "emission_factor") + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Warning, + "GHG_PROTOCOL_CONTENT_UNSUPPORTED_ROW_SKIPPED", + "GHG Protocol content row was skipped because record_type is unsupported.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: "record_type", + context: + [ + new ParserValidationIssueContext("record_type", row["record_type"]), + ])); + continue; + } + + var rowIssues = RowIssues(request, artifact, row, sourceRowNumber).ToArray(); + foreach (var rowIssue in rowIssues) + { + issues.Add(rowIssue); + } + + if (rowIssues.Length > 0) + { + continue; + } + + rows.Add(CreateOutputRow(request, artifact, row, sourceRowNumber)); + } + } + + private static IEnumerable RowIssues( + ParserAdapterRunRequest request, + ParserInputArtifact artifact, + IReadOnlyDictionary row, + int sourceRowNumber) + { + foreach (var field in Header.Where(field => RequiredFields.Contains(field) && string.IsNullOrWhiteSpace(row[field]))) + { + yield return Issue( + request, + ParserValidationIssueSeverity.Error, + "GHG_PROTOCOL_CONTENT_MISSING_REQUIRED_FIELD", + $"GHG Protocol emission factor row is missing required field: {field}.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: field); + } + + if (!int.TryParse(row["source_year"], NumberStyles.None, CultureInfo.InvariantCulture, out var sourceYear) || + sourceYear < 1) + { + yield return Issue( + request, + ParserValidationIssueSeverity.Error, + "GHG_PROTOCOL_CONTENT_INVALID_SOURCE_YEAR", + "GHG Protocol source_year must be a positive integer.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: "source_year"); + } + + if (!decimal.TryParse(row["factor_value"], NumberStyles.Number, CultureInfo.InvariantCulture, out _)) + { + yield return Issue( + request, + ParserValidationIssueSeverity.Error, + "GHG_PROTOCOL_CONTENT_INVALID_FACTOR_VALUE", + "GHG Protocol factor_value must be a decimal number.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: "factor_value"); + } + } + + private static ParserNormalizedOutputRow CreateOutputRow( + ParserAdapterRunRequest request, + ParserInputArtifact artifact, + IReadOnlyDictionary row, + int sourceRowNumber) + { + var sourceYear = int.Parse(row["source_year"], CultureInfo.InvariantCulture); + var rowIdentifier = string.Join( + "_", + new[] + { + "ghg_protocol", + row["source_year"], + row["source_version"], + row["factor_id"], + $"row_{sourceRowNumber.ToString(CultureInfo.InvariantCulture)}", + }); + var masterId = $"ghg_master_{row["source_year"]}_{row["source_version"]}_{row["factor_id"]}"; + var detailId = $"ghg_detail_{row["source_year"]}_{row["source_version"]}_{row["factor_id"]}"; + + return new ParserNormalizedOutputRow( + SourceFamily.GhgProtocol, + request.SourceKey, + request.ParserKey, + artifact.ArtifactReference, + rowIdentifier, + sourceRowNumber, + [ + new ParserNormalizedField("source_family", SourceFamily.GhgProtocol.ToWireName()), + new ParserNormalizedField("source_year", row["source_year"]), + new ParserNormalizedField("source_version", row["source_version"]), + new ParserNormalizedField("factor_id", row["factor_id"]), + new ParserNormalizedField("factor_name", row["factor_name"]), + new ParserNormalizedField("factor_value", row["factor_value"]), + new ParserNormalizedField("unit", row["unit"]), + new ParserNormalizedField("category", row["category"]), + new ParserNormalizedField("subcategory", NullIfEmpty(row["subcategory"])), + new ParserNormalizedField("scope", NullIfEmpty(row["scope"])), + new ParserNormalizedField("gas", NullIfEmpty(row["gas"])), + new ParserNormalizedField("provenance_artifact_reference", artifact.ArtifactReference), + new ParserNormalizedField("provenance_checksum_algorithm", artifact.ChecksumAlgorithm), + new ParserNormalizedField("provenance_checksum_value", artifact.ChecksumValue), + new ParserNormalizedField("provenance_row_number", sourceRowNumber.ToString(CultureInfo.InvariantCulture)), + new ParserNormalizedField("provenance_note", NullIfEmpty(row["provenance_note"])), + new ParserNormalizedField("source_family_master_id", masterId), + new ParserNormalizedField("source_family_detail_id", detailId), + new ParserNormalizedField("master_external_key", $"{row["source_year"]}:{row["source_version"]}:{row["factor_id"]}"), + new ParserNormalizedField("detail_external_key", $"{row["factor_id"]}:{row["unit"]}"), + ], + reportingYear: sourceYear); + } + + private static ParserAdapterRunResult FailedResult( + ParserAdapterRunRequest request, + IEnumerable rows, + IEnumerable issues) => + new( + request.SourceFamily, + request.SourceKey, + request.ParserKey, + ParserRunStatus.Failed, + request.Artifacts.Select(artifact => artifact.ArtifactReference), + rows, + issues, + request.RunId, + request.CorrelationId, + request.RequestedReportingYear); + + private static ParserValidationIssue Issue( + ParserAdapterRunRequest request, + ParserValidationIssueSeverity severity, + string code, + string message, + string? artifactReference = null, + string? rowIdentifier = null, + int? sourceRowNumber = null, + string? fieldKey = null, + IEnumerable? context = null) => + new( + request.SourceFamily, + request.SourceKey, + request.ParserKey, + severity, + code, + message, + artifactReference, + rowIdentifier, + sourceRowNumber, + fieldKey, + context); + + private static IEnumerable> CsvRows(string content) + { + var row = new List(); + var field = new List(); + var inQuotes = false; + + for (var index = 0; index < content.Length; index++) + { + var current = content[index]; + if (inQuotes) + { + if (current == '"' && index + 1 < content.Length && content[index + 1] == '"') + { + field.Add('"'); + index++; + continue; + } + + if (current == '"') + { + inQuotes = false; + continue; + } + + field.Add(current); + continue; + } + + switch (current) + { + case '"': + inQuotes = true; + break; + case ',': + row.Add(new string(field.ToArray())); + field.Clear(); + break; + case '\r': + break; + case '\n': + row.Add(new string(field.ToArray())); + field.Clear(); + yield return row.ToArray(); + row.Clear(); + break; + default: + field.Add(current); + break; + } + } + + if (field.Count > 0 || row.Count > 0) + { + row.Add(new string(field.ToArray())); + yield return row.ToArray(); + } + } + + private static string? NullIfEmpty(string value) => + string.IsNullOrWhiteSpace(value) ? null : value; +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryBoundary.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryBoundary.cs new file mode 100644 index 00000000..2e853f6b --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryBoundary.cs @@ -0,0 +1,403 @@ +namespace CarbonOps.Parser.Contracts; + +public static class GhgSourceDiscoveryBoundary +{ + private const string GhgSourceKey = "ghg_protocol"; + private const string DiscoveryReferenceUri = "discovery://ghg_protocol/acquisition"; + private const string ArtifactKind = "discovery"; + + public static GhgSourceDiscoveryRequest CreateRequest() => + new( + SourceFamily.GhgProtocol, + GhgSourceKey, + DiscoveryReferenceUri); + + public static GhgSourceDiscoveryResult CreateResult(GhgSourceDiscoveryRequest? request = null) + { + var activeRequest = request ?? CreateRequest(); + var requestValidation = Validate(activeRequest); + if (!requestValidation.IsValid) + { + return new GhgSourceDiscoveryResult( + GhgSourceDiscoveryStatus.Invalid, + activeRequest, + Array.Empty(), + requestValidation.Issues); + } + + var candidate = new GhgSourceDocumentCandidate( + SourceFamily.GhgProtocol, + GhgSourceKey, + "ghg_source_discovery_candidate_001_ghg_protocol", + "GHG Protocol", + activeRequest.DiscoveryReferenceUri, + ArtifactKind, + versionLabel: "dn045_ghg_discovery_boundary", + discoveredAtLabel: "runtime_passive_discovery_unavailable"); + var candidateValidation = Validate(candidate); + + return new GhgSourceDiscoveryResult( + candidateValidation.IsValid ? GhgSourceDiscoveryStatus.Declared : GhgSourceDiscoveryStatus.Invalid, + activeRequest, + candidateValidation.IsValid ? new[] { candidate } : Array.Empty(), + candidateValidation.Issues); + } + + public static GhgSourceDiscoveryValidationResult Validate(GhgSourceDiscoveryRequest? request) + { + var issues = new List(); + + if (request is null) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_MISSING_REQUEST", + "request is required.", + "request")); + return new GhgSourceDiscoveryValidationResult(issues); + } + + if (!Enum.IsDefined(request.SourceFamily)) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_INVALID_SOURCE_FAMILY", + "source_family must be a defined source family.", + "source_family")); + } + + ValidateRequiredText( + request.SourceKey, + "source_key", + "GHG_SOURCE_DISCOVERY_MISSING_SOURCE_KEY", + "source_key must be a non-empty string.", + issues); + ValidateRequiredText( + request.DiscoveryReferenceUri, + "discovery_reference_uri", + "GHG_SOURCE_DISCOVERY_MISSING_REFERENCE_URI", + "discovery_reference_uri must be a non-empty string.", + issues); + + if (request.SourceFamily != SourceFamily.GhgProtocol) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_SOURCE_FAMILY_MISMATCH", + "source_family must be ghg_protocol.", + "source_family")); + } + + if (request.SourceKey != GhgSourceKey) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_SOURCE_KEY_MISMATCH", + "source_key must be ghg_protocol.", + "source_key")); + } + + if (request.Mode != GhgSourceDiscoveryMode.RuntimePassive) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_UNSUPPORTED_MODE", + "mode must remain runtime_passive.", + "mode")); + } + + ValidateFalse( + request.AllowNetwork, + "allow_network", + "GHG_SOURCE_DISCOVERY_NETWORK_NOT_ALLOWED", + "allow_network must be false for this boundary.", + issues); + ValidateFalse( + request.AllowDownload, + "allow_download", + "GHG_SOURCE_DISCOVERY_DOWNLOAD_NOT_ALLOWED", + "allow_download must be false for this boundary.", + issues); + ValidateFalse( + request.AllowParse, + "allow_parse", + "GHG_SOURCE_DISCOVERY_PARSE_NOT_ALLOWED", + "allow_parse must be false for this boundary.", + issues); + ValidateFalse( + request.AllowDatabaseWrites, + "allow_database_writes", + "GHG_SOURCE_DISCOVERY_DATABASE_WRITES_NOT_ALLOWED", + "allow_database_writes must be false for this boundary.", + issues); + ValidateFalse( + request.AllowScheduler, + "allow_scheduler", + "GHG_SOURCE_DISCOVERY_SCHEDULER_NOT_ALLOWED", + "allow_scheduler must be false for this boundary.", + issues); + + return new GhgSourceDiscoveryValidationResult(issues); + } + + public static GhgSourceDiscoveryValidationResult Validate(GhgSourceDocumentCandidate? candidate) + { + var issues = new List(); + + if (candidate is null) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING", + "candidate is required.", + "candidate")); + return new GhgSourceDiscoveryValidationResult(issues); + } + + if (!Enum.IsDefined(candidate.SourceFamily)) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_CANDIDATE_INVALID_SOURCE_FAMILY", + "source_family must be a defined source family.", + "source_family")); + } + + ValidateRequiredText( + candidate.SourceKey, + "source_key", + "GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_SOURCE_KEY", + "source_key must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.CandidateId, + "candidate_id", + "GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_CANDIDATE_ID", + "candidate_id must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.Title, + "title", + "GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_TITLE", + "title must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.ReferenceUri, + "reference_uri", + "GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_REFERENCE_URI", + "reference_uri must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.ArtifactKind, + "artifact_kind", + "GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_ARTIFACT_KIND", + "artifact_kind must be a non-empty string.", + issues); + ValidateOptionalText( + candidate.ContentType, + "content_type", + "GHG_SOURCE_DISCOVERY_CANDIDATE_BLANK_CONTENT_TYPE", + "content_type must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.Extension, + "extension", + "GHG_SOURCE_DISCOVERY_CANDIDATE_BLANK_EXTENSION", + "extension must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.ChecksumSha256, + "checksum_sha256", + "GHG_SOURCE_DISCOVERY_CANDIDATE_BLANK_CHECKSUM_SHA256", + "checksum_sha256 must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.VersionLabel, + "version_label", + "GHG_SOURCE_DISCOVERY_CANDIDATE_BLANK_VERSION_LABEL", + "version_label must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.DiscoveredAtLabel, + "discovered_at_label", + "GHG_SOURCE_DISCOVERY_CANDIDATE_BLANK_DISCOVERED_AT_LABEL", + "discovered_at_label must be non-empty when provided.", + issues); + ValidateOptionalPositiveInt( + candidate.DocumentYear, + "document_year", + "GHG_SOURCE_DISCOVERY_CANDIDATE_INVALID_DOCUMENT_YEAR", + "document_year must be a positive integer when provided.", + issues); + ValidateOptionalPositiveInt( + candidate.ReportingYear, + "reporting_year", + "GHG_SOURCE_DISCOVERY_CANDIDATE_INVALID_REPORTING_YEAR", + "reporting_year must be a positive integer when provided.", + issues); + + if (candidate.SourceFamily != SourceFamily.GhgProtocol) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_CANDIDATE_SOURCE_FAMILY_MISMATCH", + "source_family must match the GHG source family.", + "source_family")); + } + + if (candidate.SourceKey != GhgSourceKey) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_CANDIDATE_SOURCE_KEY_MISMATCH", + "source_key must match the GHG source key.", + "source_key")); + } + + if (candidate.ArtifactKind != ArtifactKind) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_CANDIDATE_ARTIFACT_KIND_MISMATCH", + "artifact_kind must match the GHG expected format.", + "artifact_kind")); + } + + if (candidate.Status != GhgSourceDiscoveryStatus.Declared) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_CANDIDATE_UNSUPPORTED_STATUS", + "candidate status must remain declared.", + "status")); + } + + if (candidate.DownloadAllowed) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_CANDIDATE_DOWNLOAD_NOT_ALLOWED", + "download_allowed must be false for this boundary.", + "download_allowed")); + } + + return new GhgSourceDiscoveryValidationResult(issues); + } + + public static GhgSourceDiscoveryValidationResult Validate(GhgSourceDiscoveryResult? result) + { + var issues = new List(); + + if (result is null) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_RESULT_MISSING", + "result is required.", + "result")); + return new GhgSourceDiscoveryValidationResult(issues); + } + + issues.AddRange(Validate(result.Request).Issues); + + if (!Enum.IsDefined(result.Status)) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_RESULT_INVALID_STATUS", + "status must be a defined GHG source discovery status.", + "status")); + } + + foreach (var (fieldName, value) in new[] + { + ("no_network", result.NoNetwork), + ("no_download", result.NoDownload), + ("no_parse", result.NoParse), + ("no_database_writes", result.NoDatabaseWrites), + ("no_sql", result.NoSql), + ("no_scheduler", result.NoScheduler), + }) + { + if (!value) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED", + $"{fieldName} must remain true.", + fieldName)); + } + } + + for (var index = 0; index < result.Candidates.Count; index++) + { + foreach (var issue in Validate(result.Candidates[index]).Issues) + { + issues.Add(issue with { FieldName = $"candidates[{index + 1}].{issue.FieldName}" }); + } + } + + if (result.Status == GhgSourceDiscoveryStatus.Declared && result.Issues.Count > 0) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + "declared result status must not include issue metadata.", + "issues")); + } + + if (result.Status == GhgSourceDiscoveryStatus.Declared && issues.Count > 0) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + "declared result status requires valid metadata.", + "status")); + } + + if (result.Status == GhgSourceDiscoveryStatus.Invalid && result.Issues.Count == 0) + { + issues.Add(new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_RESULT_MISSING_INVALID_ISSUES", + "invalid result status requires issue metadata.", + "issues")); + } + + return new GhgSourceDiscoveryValidationResult(issues); + } + + private static void ValidateRequiredText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (string.IsNullOrWhiteSpace(value)) + { + issues.Add(new GhgSourceDiscoveryIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is not null && string.IsNullOrWhiteSpace(value)) + { + issues.Add(new GhgSourceDiscoveryIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalPositiveInt( + int? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is <= 0) + { + issues.Add(new GhgSourceDiscoveryIssue(code, message, fieldName)); + } + } + + private static void ValidateFalse( + bool value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value) + { + issues.Add(new GhgSourceDiscoveryIssue(code, message, fieldName)); + } + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryIssue.cs new file mode 100644 index 00000000..56265344 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDiscoveryIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryMode.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryMode.cs new file mode 100644 index 00000000..abaa0f42 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryMode.cs @@ -0,0 +1,6 @@ +namespace CarbonOps.Parser.Contracts; + +public enum GhgSourceDiscoveryMode +{ + RuntimePassive = 0, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryRequest.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryRequest.cs new file mode 100644 index 00000000..31ac49d4 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryRequest.cs @@ -0,0 +1,44 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDiscoveryRequest +{ + public SourceFamily SourceFamily { get; } + + public string SourceKey { get; } + + public string DiscoveryReferenceUri { get; } + + public GhgSourceDiscoveryMode Mode { get; } + + public bool AllowNetwork { get; } + + public bool AllowDownload { get; } + + public bool AllowParse { get; } + + public bool AllowDatabaseWrites { get; } + + public bool AllowScheduler { get; } + + public GhgSourceDiscoveryRequest( + SourceFamily sourceFamily, + string sourceKey, + string discoveryReferenceUri, + GhgSourceDiscoveryMode mode = GhgSourceDiscoveryMode.RuntimePassive, + bool allowNetwork = false, + bool allowDownload = false, + bool allowParse = false, + bool allowDatabaseWrites = false, + bool allowScheduler = false) + { + SourceFamily = sourceFamily; + SourceKey = sourceKey; + DiscoveryReferenceUri = discoveryReferenceUri; + Mode = mode; + AllowNetwork = allowNetwork; + AllowDownload = allowDownload; + AllowParse = allowParse; + AllowDatabaseWrites = allowDatabaseWrites; + AllowScheduler = allowScheduler; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryResult.cs new file mode 100644 index 00000000..5ef2cc5d --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryResult.cs @@ -0,0 +1,55 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDiscoveryResult +{ + public GhgSourceDiscoveryStatus Status { get; } + + public GhgSourceDiscoveryRequest Request { get; } + + public IReadOnlyList Candidates { get; } + + public IReadOnlyList Issues { get; } + + public bool NoNetwork { get; } + + public bool NoDownload { get; } + + public bool NoParse { get; } + + public bool NoDatabaseWrites { get; } + + public bool NoSql { get; } + + public bool NoScheduler { get; } + + public int CandidateCount => Candidates.Count; + + public IReadOnlyList CandidateIds { get; } + + public GhgSourceDiscoveryResult( + GhgSourceDiscoveryStatus status, + GhgSourceDiscoveryRequest request, + IEnumerable candidates, + IEnumerable? issues = null, + bool noNetwork = true, + bool noDownload = true, + bool noParse = true, + bool noDatabaseWrites = true, + bool noSql = true, + bool noScheduler = true) + { + var candidateSnapshot = candidates.ToArray(); + + Status = status; + Request = request; + Candidates = Array.AsReadOnly(candidateSnapshot); + Issues = Array.AsReadOnly((issues ?? Array.Empty()).ToArray()); + NoNetwork = noNetwork; + NoDownload = noDownload; + NoParse = noParse; + NoDatabaseWrites = noDatabaseWrites; + NoSql = noSql; + NoScheduler = noScheduler; + CandidateIds = Array.AsReadOnly(candidateSnapshot.Select(candidate => candidate.CandidateId).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryStatus.cs new file mode 100644 index 00000000..dfb2d8d3 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryStatus.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public enum GhgSourceDiscoveryStatus +{ + Declared = 0, + Invalid = 1, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryValidationResult.cs new file mode 100644 index 00000000..6cf89ddb --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDiscoveryValidationResult.cs @@ -0,0 +1,13 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDiscoveryValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public GhgSourceDiscoveryValidationResult(IEnumerable? issues = null) + { + Issues = Array.AsReadOnly((issues ?? Array.Empty()).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDocumentCandidate.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDocumentCandidate.cs new file mode 100644 index 00000000..e1d73896 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDocumentCandidate.cs @@ -0,0 +1,68 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDocumentCandidate +{ + public SourceFamily SourceFamily { get; } + + public string SourceKey { get; } + + public string CandidateId { get; } + + public string Title { get; } + + public string ReferenceUri { get; } + + public string ArtifactKind { get; } + + public GhgSourceDiscoveryStatus Status { get; } + + public int? DocumentYear { get; } + + public int? ReportingYear { get; } + + public string? ContentType { get; } + + public string? Extension { get; } + + public string? ChecksumSha256 { get; } + + public string? VersionLabel { get; } + + public string? DiscoveredAtLabel { get; } + + public bool DownloadAllowed { get; } + + public GhgSourceDocumentCandidate( + SourceFamily sourceFamily, + string sourceKey, + string candidateId, + string title, + string referenceUri, + string artifactKind, + GhgSourceDiscoveryStatus status = GhgSourceDiscoveryStatus.Declared, + int? documentYear = null, + int? reportingYear = null, + string? contentType = null, + string? extension = null, + string? checksumSha256 = null, + string? versionLabel = null, + string? discoveredAtLabel = null, + bool downloadAllowed = false) + { + SourceFamily = sourceFamily; + SourceKey = sourceKey; + CandidateId = candidateId; + Title = title; + ReferenceUri = referenceUri; + ArtifactKind = artifactKind; + Status = status; + DocumentYear = documentYear; + ReportingYear = reportingYear; + ContentType = contentType; + Extension = extension; + ChecksumSha256 = checksumSha256; + VersionLabel = versionLabel; + DiscoveredAtLabel = discoveredAtLabel; + DownloadAllowed = downloadAllowed; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionBoundary.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionBoundary.cs new file mode 100644 index 00000000..86b11754 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionBoundary.cs @@ -0,0 +1,728 @@ +using System.Security.Cryptography; + +namespace CarbonOps.Parser.Contracts; + +public static class GhgSourceDownloadExecutionBoundary +{ + private const string GhgSourceKey = "ghg_protocol"; + + public static GhgSourceDownloadExecutionRequest CreateRequest( + GhgSourceDocumentCandidate candidate, + string targetRoot, + string targetRelativePath, + bool allowDownloadExecution = false, + bool allowFileWrite = false, + bool allowNetwork = false, + bool allowOverwrite = false) => + new( + candidate.SourceFamily, + candidate.SourceKey, + candidate.CandidateId, + candidate.Title, + candidate.ReferenceUri, + candidate.ArtifactKind, + targetRoot, + targetRelativePath, + candidate.DownloadAllowed, + allowDownloadExecution, + allowFileWrite, + allowNetwork, + allowOverwrite, + contentType: candidate.ContentType, + extension: candidate.Extension, + expectedChecksumSha256: candidate.ChecksumSha256, + documentYear: candidate.DocumentYear, + reportingYear: candidate.ReportingYear, + versionLabel: candidate.VersionLabel); + + public static GhgSourceDownloadExecutionValidationResult Validate( + GhgSourceDownloadExecutionRequest? request) + { + var issues = new List(); + + if (request is null) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_MISSING_REQUEST", + "request is required.", + "request")); + return new GhgSourceDownloadExecutionValidationResult(issues); + } + + if (!Enum.IsDefined(request.SourceFamily)) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_INVALID_SOURCE_FAMILY", + "source_family must be a defined source family.", + "source_family")); + } + + ValidateRequiredText( + request.SourceKey, + "source_key", + "GHG_SOURCE_DOWNLOAD_MISSING_SOURCE_KEY", + "source_key must be a non-empty string.", + issues); + ValidateRequiredText( + request.CandidateId, + "candidate_id", + "GHG_SOURCE_DOWNLOAD_MISSING_CANDIDATE_ID", + "candidate_id must be a non-empty string.", + issues); + ValidateRequiredText( + request.CandidateTitle, + "candidate_title", + "GHG_SOURCE_DOWNLOAD_MISSING_CANDIDATE_TITLE", + "candidate_title must be a non-empty string.", + issues); + ValidateRequiredText( + request.SourceReferenceUri, + "source_reference_uri", + "GHG_SOURCE_DOWNLOAD_MISSING_SOURCE_REFERENCE_URI", + "source_reference_uri must be a non-empty string.", + issues); + ValidateRequiredText( + request.ArtifactKind, + "artifact_kind", + "GHG_SOURCE_DOWNLOAD_MISSING_ARTIFACT_KIND", + "artifact_kind must be a non-empty string.", + issues); + ValidateRequiredText( + request.TargetRoot, + "target_root", + "GHG_SOURCE_DOWNLOAD_MISSING_TARGET_ROOT", + "target_root must be a non-empty string.", + issues); + ValidateRequiredText( + request.TargetRelativePath, + "target_relative_path", + "GHG_SOURCE_DOWNLOAD_MISSING_TARGET_RELATIVE_PATH", + "target_relative_path must be a non-empty string.", + issues); + ValidateOptionalText( + request.ContentType, + "content_type", + "GHG_SOURCE_DOWNLOAD_BLANK_CONTENT_TYPE", + "content_type must be non-empty when provided.", + issues); + ValidateOptionalText( + request.Extension, + "extension", + "GHG_SOURCE_DOWNLOAD_BLANK_EXTENSION", + "extension must be non-empty when provided.", + issues); + ValidateOptionalText( + request.ExpectedChecksumSha256, + "expected_checksum_sha256", + "GHG_SOURCE_DOWNLOAD_BLANK_EXPECTED_CHECKSUM_SHA256", + "expected_checksum_sha256 must be non-empty when provided.", + issues); + ValidateOptionalText( + request.VersionLabel, + "version_label", + "GHG_SOURCE_DOWNLOAD_BLANK_VERSION_LABEL", + "version_label must be non-empty when provided.", + issues); + ValidateOptionalPositiveInt( + request.DocumentYear, + "document_year", + "GHG_SOURCE_DOWNLOAD_INVALID_DOCUMENT_YEAR", + "document_year must be a positive integer when provided.", + issues); + ValidateOptionalPositiveInt( + request.ReportingYear, + "reporting_year", + "GHG_SOURCE_DOWNLOAD_INVALID_REPORTING_YEAR", + "reporting_year must be a positive integer when provided.", + issues); + + if (request.SourceFamily != SourceFamily.GhgProtocol) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_SOURCE_FAMILY_MISMATCH", + "source_family must be ghg_protocol.", + "source_family")); + } + + if (request.SourceKey != GhgSourceKey) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_SOURCE_KEY_MISMATCH", + "source_key must be ghg_protocol.", + "source_key")); + } + + ValidateTrue( + request.CandidateDownloadAllowed, + "candidate_download_allowed", + "GHG_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE", + "candidate metadata must explicitly allow download execution.", + issues); + ValidateTrue( + request.AllowDownloadExecution, + "allow_download_execution", + "GHG_SOURCE_DOWNLOAD_EXECUTION_NOT_ALLOWED", + "allow_download_execution must be true.", + issues); + ValidateTrue( + request.AllowFileWrite, + "allow_file_write", + "GHG_SOURCE_DOWNLOAD_FILE_WRITE_NOT_ALLOWED", + "allow_file_write must be true.", + issues); + ValidateFalse( + request.AllowParse, + "allow_parse", + "GHG_SOURCE_DOWNLOAD_PARSE_NOT_ALLOWED", + "allow_parse must be false for this boundary.", + issues); + ValidateFalse( + request.AllowDatabaseWrites, + "allow_database_writes", + "GHG_SOURCE_DOWNLOAD_DATABASE_WRITES_NOT_ALLOWED", + "allow_database_writes must be false for this boundary.", + issues); + ValidateFalse( + request.AllowScheduler, + "allow_scheduler", + "GHG_SOURCE_DOWNLOAD_SCHEDULER_NOT_ALLOWED", + "allow_scheduler must be false for this boundary.", + issues); + + ValidateSourceReferenceUri(request, issues); + ValidateTargetPaths(request, issues); + + return new GhgSourceDownloadExecutionValidationResult(issues); + } + + public static GhgSourceDownloadExecutionResult Execute( + GhgSourceDownloadExecutionRequest request, + Func transport) + { + var validation = Validate(request); + if (!validation.IsValid) + { + return new GhgSourceDownloadExecutionResult( + GhgSourceDownloadExecutionStatus.Blocked, + request, + issues: validation.Issues); + } + + var safeTarget = PrepareSafeTargetPath(request); + if (!safeTarget.Validation.IsValid) + { + return new GhgSourceDownloadExecutionResult( + GhgSourceDownloadExecutionStatus.Blocked, + request, + issues: safeTarget.Validation.Issues); + } + + GhgSourceDownloadTransportResponse response; + try + { + response = transport(request.SourceReferenceUri); + } + catch (Exception error) + { + return new GhgSourceDownloadExecutionResult( + GhgSourceDownloadExecutionStatus.Failed, + request, + issues: + [ + new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_TRANSPORT_FAILED", + $"transport failed: {error.Message}", + "source_reference_uri"), + ]); + } + + var responseValidation = ValidateTransportResponse(response); + if (!responseValidation.IsValid) + { + return new GhgSourceDownloadExecutionResult( + GhgSourceDownloadExecutionStatus.Failed, + request, + issues: responseValidation.Issues); + } + + var checksum = Convert.ToHexString(SHA256.HashData(response.Content)).ToLowerInvariant(); + if (request.ExpectedChecksumSha256 is not null + && !string.Equals(checksum, request.ExpectedChecksumSha256, StringComparison.OrdinalIgnoreCase)) + { + return new GhgSourceDownloadExecutionResult( + GhgSourceDownloadExecutionStatus.Failed, + request, + issues: + [ + new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH", + "downloaded content checksum did not match expected value.", + "expected_checksum_sha256"), + ]); + } + + safeTarget = PrepareSafeTargetPath(request); + if (!safeTarget.Validation.IsValid) + { + return new GhgSourceDownloadExecutionResult( + GhgSourceDownloadExecutionStatus.Blocked, + request, + issues: safeTarget.Validation.Issues); + } + + try + { + WriteContentToSafeTarget(safeTarget.TargetPath, response.Content, request.AllowOverwrite); + } + catch (IOException error) when (File.Exists(safeTarget.TargetPath) && !request.AllowOverwrite) + { + return WriteFailed(request, "GHG_SOURCE_DOWNLOAD_TARGET_EXISTS", error); + } + catch (Exception error) + { + return WriteFailed(request, "GHG_SOURCE_DOWNLOAD_WRITE_FAILED", error); + } + + var artifact = new GhgSourceDownloadedArtifact( + request.SourceFamily, + request.SourceKey, + request.CandidateId, + $"ghg_source_download_artifact_{request.CandidateId}", + request.ArtifactKind, + request.SourceReferenceUri, + safeTarget.TargetPath, + Path.GetFileName(safeTarget.TargetPath), + checksum, + response.Content.LongLength, + response.ContentType ?? request.ContentType, + request.Extension, + response.FinalUri, + request.DocumentYear, + request.ReportingYear, + request.VersionLabel); + + return new GhgSourceDownloadExecutionResult( + GhgSourceDownloadExecutionStatus.Downloaded, + request, + artifact); + } + + public static GhgSourceDownloadExecutionValidationResult Validate( + GhgSourceDownloadExecutionResult? result) + { + var issues = new List(); + + if (result is null) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_RESULT_MISSING", + "result is required.", + "result")); + return new GhgSourceDownloadExecutionValidationResult(issues); + } + + issues.AddRange(Validate(result.Request).Issues); + + if (!Enum.IsDefined(result.Status)) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_RESULT_INVALID_STATUS", + "status must be a defined GHG source download execution status.", + "status")); + } + + foreach (var (fieldName, value) in new[] + { + ("no_parse", result.NoParse), + ("no_database_writes", result.NoDatabaseWrites), + ("no_sql", result.NoSql), + ("no_scheduler", result.NoScheduler), + }) + { + if (!value) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED", + $"{fieldName} must remain true.", + fieldName)); + } + } + + if (result.Status == GhgSourceDownloadExecutionStatus.Downloaded && result.Artifact is null) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_RESULT_MISSING_ARTIFACT", + "downloaded results require artifact metadata.", + "artifact")); + } + else if (result.Status != GhgSourceDownloadExecutionStatus.Downloaded && result.Artifact is not null) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_RESULT_UNEXPECTED_ARTIFACT", + "non-downloaded results must not include artifact metadata.", + "artifact")); + } + + if (result.Status != GhgSourceDownloadExecutionStatus.Downloaded && result.Issues.Count == 0) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_RESULT_MISSING_ISSUES", + "blocked or failed results require issue metadata.", + "issues")); + } + + return new GhgSourceDownloadExecutionValidationResult(issues); + } + + private static GhgSourceDownloadExecutionResult WriteFailed( + GhgSourceDownloadExecutionRequest request, + string code, + Exception error) => + new( + GhgSourceDownloadExecutionStatus.Failed, + request, + issues: + [ + new GhgSourceDownloadExecutionIssue( + code, + $"target write failed: {error.Message}", + "target_relative_path"), + ]); + + private static GhgSourceDownloadExecutionValidationResult ValidateTransportResponse( + GhgSourceDownloadTransportResponse? response) + { + var issues = new List(); + + if (response is null) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING", + "transport response is required.", + "transport")); + return new GhgSourceDownloadExecutionValidationResult(issues); + } + + if (response.Content is null) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT", + "transport response content is required.", + "content")); + } + else if (response.Content.Length == 0) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_RESPONSE_EMPTY_CONTENT", + "transport response content must not be empty.", + "content")); + } + + ValidateOptionalText( + response.ContentType, + "content_type", + "GHG_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "response content_type must be non-empty when provided.", + issues); + ValidateOptionalText( + response.FinalUri, + "final_uri", + "GHG_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + "response final_uri must be non-empty when provided.", + issues); + + return new GhgSourceDownloadExecutionValidationResult(issues); + } + + private static (string TargetPath, GhgSourceDownloadExecutionValidationResult Validation) PrepareSafeTargetPath( + GhgSourceDownloadExecutionRequest request) + { + var issues = new List(); + + string root; + string targetPath; + try + { + root = Path.GetFullPath(request.TargetRoot); + targetPath = Path.GetFullPath(Path.Combine(root, request.TargetRelativePath)); + } + catch (Exception error) when (error is ArgumentException or NotSupportedException or PathTooLongException) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_TARGET_PATH_UNRESOLVED", + "target path could not be resolved safely.", + "target_relative_path")); + return (string.Empty, new GhgSourceDownloadExecutionValidationResult(issues)); + } + + if (!IsPathInsideRoot(root, targetPath)) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE", + "target_relative_path must stay within target_root.", + "target_relative_path")); + } + + if (ContainsExistingSymlink(root, targetPath)) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE", + "target path must not traverse an existing symbolic link.", + "target_relative_path")); + } + + if (File.Exists(targetPath) && !request.AllowOverwrite) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_TARGET_EXISTS", + "target path already exists and allow_overwrite is false.", + "target_relative_path")); + } + + return (targetPath, new GhgSourceDownloadExecutionValidationResult(issues)); + } + + private static void WriteContentToSafeTarget(string targetPath, byte[] content, bool allowOverwrite) + { + var parent = Path.GetDirectoryName(targetPath); + if (!string.IsNullOrWhiteSpace(parent)) + { + Directory.CreateDirectory(parent); + } + + if (IsSymlink(targetPath)) + { + throw new IOException("target path is a symbolic link."); + } + + var mode = allowOverwrite ? FileMode.Create : FileMode.CreateNew; + using var stream = new FileStream(targetPath, mode, FileAccess.Write, FileShare.None); + stream.Write(content, 0, content.Length); + } + + private static bool IsPathInsideRoot(string root, string targetPath) + { + var rootWithSeparator = Path.EndsInDirectorySeparator(root) + ? root + : root + Path.DirectorySeparatorChar; + + return targetPath.StartsWith(rootWithSeparator, StringComparison.Ordinal) + || string.Equals(root, targetPath, StringComparison.Ordinal); + } + + private static bool ContainsExistingSymlink(string root, string targetPath) + { + var relative = Path.GetRelativePath(root, targetPath); + if (relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative)) + { + return false; + } + + if (IsSymlink(root)) + { + return true; + } + + var current = root; + foreach (var segment in relative.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)) + { + if (string.IsNullOrWhiteSpace(segment)) + { + continue; + } + + current = Path.Combine(current, segment); + if ((Directory.Exists(current) || File.Exists(current)) && IsSymlink(current)) + { + return true; + } + } + + return false; + } + + private static bool IsSymlink(string path) + { + try + { + return (File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0; + } + catch (FileNotFoundException) + { + return false; + } + catch (DirectoryNotFoundException) + { + return false; + } + } + + private static void ValidateSourceReferenceUri( + GhgSourceDownloadExecutionRequest request, + ICollection issues) + { + if (string.IsNullOrWhiteSpace(request.SourceReferenceUri)) + { + return; + } + + if (!Uri.TryCreate(request.SourceReferenceUri, UriKind.Absolute, out var uri)) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + request.SourceReferenceUri.Contains("://", StringComparison.Ordinal) + ? "GHG_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI" + : "GHG_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME", + request.SourceReferenceUri.Contains("://", StringComparison.Ordinal) + ? "source_reference_uri must be a well-formed URI." + : "source_reference_uri must include a URI scheme.", + "source_reference_uri")); + return; + } + + if ((uri.Scheme == Uri.UriSchemeHttps || uri.Scheme == Uri.UriSchemeHttp) + && string.IsNullOrWhiteSpace(uri.Host)) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + "source_reference_uri must be a well-formed URI.", + "source_reference_uri")); + } + else if (uri.Scheme == "discovery") + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE", + "discovery references are not direct download references.", + "source_reference_uri")); + } + else if (uri.Scheme == Uri.UriSchemeHttps && !request.AllowNetwork) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_NETWORK_NOT_ALLOWED", + "allow_network must be true for https source references.", + "source_reference_uri")); + } + else if (uri.Scheme == Uri.UriSchemeHttp) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_INSECURE_HTTP_NOT_ALLOWED", + "http source references are not allowed.", + "source_reference_uri")); + } + else if (uri.Scheme is not "mock" and not "memory") + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI", + "source_reference_uri must use an allowed execution scheme.", + "source_reference_uri")); + } + } + + private static void ValidateTargetPaths( + GhgSourceDownloadExecutionRequest request, + ICollection issues) + { + if (!string.IsNullOrWhiteSpace(request.TargetRoot) && !Path.IsPathFullyQualified(request.TargetRoot)) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_ABSOLUTE", + "target_root must be an absolute path.", + "target_root")); + } + + if (string.IsNullOrWhiteSpace(request.TargetRelativePath)) + { + return; + } + + if (Path.IsPathFullyQualified(request.TargetRelativePath)) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_ABSOLUTE", + "target_relative_path must be relative.", + "target_relative_path")); + } + + if (Uri.TryCreate(request.TargetRelativePath, UriKind.Absolute, out var targetUri) + && !string.IsNullOrWhiteSpace(targetUri.Scheme)) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_URI", + "target_relative_path must not be a URI.", + "target_relative_path")); + } + + var segments = request.TargetRelativePath.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + if (segments.Any(segment => segment == "..")) + { + issues.Add(new GhgSourceDownloadExecutionIssue( + "GHG_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE", + "target_relative_path must not contain parent traversal.", + "target_relative_path")); + } + } + + private static void ValidateRequiredText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (string.IsNullOrWhiteSpace(value)) + { + issues.Add(new GhgSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is not null && string.IsNullOrWhiteSpace(value)) + { + issues.Add(new GhgSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalPositiveInt( + int? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is <= 0) + { + issues.Add(new GhgSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateTrue( + bool value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (!value) + { + issues.Add(new GhgSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateFalse( + bool value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value) + { + issues.Add(new GhgSourceDownloadExecutionIssue(code, message, fieldName)); + } + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionIssue.cs new file mode 100644 index 00000000..535a8154 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDownloadExecutionIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionRequest.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionRequest.cs new file mode 100644 index 00000000..5ef495f9 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionRequest.cs @@ -0,0 +1,96 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDownloadExecutionRequest +{ + public SourceFamily SourceFamily { get; init; } + + public string SourceKey { get; init; } + + public string CandidateId { get; init; } + + public string CandidateTitle { get; init; } + + public string SourceReferenceUri { get; init; } + + public string ArtifactKind { get; init; } + + public string TargetRoot { get; init; } + + public string TargetRelativePath { get; init; } + + public bool CandidateDownloadAllowed { get; init; } + + public bool AllowDownloadExecution { get; init; } + + public bool AllowFileWrite { get; init; } + + public bool AllowNetwork { get; init; } + + public bool AllowOverwrite { get; init; } + + public bool AllowParse { get; init; } + + public bool AllowDatabaseWrites { get; init; } + + public bool AllowScheduler { get; init; } + + public string? ContentType { get; init; } + + public string? Extension { get; init; } + + public string? ExpectedChecksumSha256 { get; init; } + + public int? DocumentYear { get; init; } + + public int? ReportingYear { get; init; } + + public string? VersionLabel { get; init; } + + public GhgSourceDownloadExecutionRequest( + SourceFamily sourceFamily, + string sourceKey, + string candidateId, + string candidateTitle, + string sourceReferenceUri, + string artifactKind, + string targetRoot, + string targetRelativePath, + bool candidateDownloadAllowed = false, + bool allowDownloadExecution = false, + bool allowFileWrite = false, + bool allowNetwork = false, + bool allowOverwrite = false, + bool allowParse = false, + bool allowDatabaseWrites = false, + bool allowScheduler = false, + string? contentType = null, + string? extension = null, + string? expectedChecksumSha256 = null, + int? documentYear = null, + int? reportingYear = null, + string? versionLabel = null) + { + SourceFamily = sourceFamily; + SourceKey = sourceKey; + CandidateId = candidateId; + CandidateTitle = candidateTitle; + SourceReferenceUri = sourceReferenceUri; + ArtifactKind = artifactKind; + TargetRoot = targetRoot; + TargetRelativePath = targetRelativePath; + CandidateDownloadAllowed = candidateDownloadAllowed; + AllowDownloadExecution = allowDownloadExecution; + AllowFileWrite = allowFileWrite; + AllowNetwork = allowNetwork; + AllowOverwrite = allowOverwrite; + AllowParse = allowParse; + AllowDatabaseWrites = allowDatabaseWrites; + AllowScheduler = allowScheduler; + ContentType = contentType; + Extension = extension; + ExpectedChecksumSha256 = expectedChecksumSha256; + DocumentYear = documentYear; + ReportingYear = reportingYear; + VersionLabel = versionLabel; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionResult.cs new file mode 100644 index 00000000..afba6841 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionResult.cs @@ -0,0 +1,42 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDownloadExecutionResult +{ + public GhgSourceDownloadExecutionStatus Status { get; init; } + + public GhgSourceDownloadExecutionRequest Request { get; init; } + + public GhgSourceDownloadedArtifact? Artifact { get; init; } + + public IReadOnlyList Issues { get; init; } + + public bool NoParse { get; init; } + + public bool NoDatabaseWrites { get; init; } + + public bool NoSql { get; init; } + + public bool NoScheduler { get; init; } + + public bool Downloaded => Status == GhgSourceDownloadExecutionStatus.Downloaded; + + public GhgSourceDownloadExecutionResult( + GhgSourceDownloadExecutionStatus status, + GhgSourceDownloadExecutionRequest request, + GhgSourceDownloadedArtifact? artifact = null, + IEnumerable? issues = null, + bool noParse = true, + bool noDatabaseWrites = true, + bool noSql = true, + bool noScheduler = true) + { + Status = status; + Request = request; + Artifact = artifact; + Issues = (issues ?? Array.Empty()).ToArray(); + NoParse = noParse; + NoDatabaseWrites = noDatabaseWrites; + NoSql = noSql; + NoScheduler = noScheduler; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionStatus.cs new file mode 100644 index 00000000..be6dc6a2 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionStatus.cs @@ -0,0 +1,8 @@ +namespace CarbonOps.Parser.Contracts; + +public enum GhgSourceDownloadExecutionStatus +{ + Blocked = 0, + Downloaded = 1, + Failed = 2, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionValidationResult.cs new file mode 100644 index 00000000..45f37ddd --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadExecutionValidationResult.cs @@ -0,0 +1,14 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDownloadExecutionValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public GhgSourceDownloadExecutionValidationResult( + IEnumerable? issues = null) + { + Issues = (issues ?? Array.Empty()).ToArray(); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadTransportResponse.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadTransportResponse.cs new file mode 100644 index 00000000..fa9e232d --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadTransportResponse.cs @@ -0,0 +1,6 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDownloadTransportResponse( + byte[] Content, + string? ContentType = null, + string? FinalUri = null); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadedArtifact.cs b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadedArtifact.cs new file mode 100644 index 00000000..678314b3 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/GhgSourceDownloadedArtifact.cs @@ -0,0 +1,19 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record GhgSourceDownloadedArtifact( + SourceFamily SourceFamily, + string SourceKey, + string CandidateId, + string ArtifactId, + string ArtifactKind, + string SourceReferenceUri, + string LocalPath, + string OriginalFilename, + string ChecksumSha256, + long SizeBytes, + string? ContentType = null, + string? Extension = null, + string? FinalUri = null, + int? DocumentYear = null, + int? ReportingYear = null, + string? VersionLabel = null); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccEfdbNormalizedContentParser.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccEfdbNormalizedContentParser.cs new file mode 100644 index 00000000..5b393f4b --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccEfdbNormalizedContentParser.cs @@ -0,0 +1,431 @@ +using System.Globalization; + +namespace CarbonOps.Parser.Contracts; + +public static class IpccEfdbNormalizedContentParser +{ + public static IReadOnlyList Header { get; } = Array.AsReadOnly(new[] + { + "record_type", + "source_year", + "source_version", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", + "subcategory", + "ipcc_sector", + "gas", + "region", + "technology", + "provenance", + }); + + private static readonly IReadOnlySet RequiredFields = new HashSet( + new[] + { + "source_year", + "source_version", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", + "ipcc_sector", + "gas", + "provenance", + }, + StringComparer.Ordinal); + + public static ParserAdapterRunResult Parse( + ParserAdapterRunRequest request, + IReadOnlyDictionary contentByArtifactReference) + { + var requestValidation = request.Validate(); + if (!requestValidation.IsValid) + { + return FailedResult( + request, + [], + requestValidation.Errors.Select(error => Issue( + request, + ParserValidationIssueSeverity.Error, + "IPCC_EFDB_CONTENT_INVALID_REQUEST", + error))); + } + + if (request.SourceFamily != SourceFamily.IpccEfdb) + { + return FailedResult( + request, + [], + [ + Issue( + request, + ParserValidationIssueSeverity.Error, + "IPCC_EFDB_CONTENT_SOURCE_FAMILY_MISMATCH", + "IPCC EFDB content parser only accepts ipcc_efdb source family.", + fieldKey: "source_family"), + ]); + } + + var rows = new List(); + var issues = new List(); + + foreach (var artifact in request.Artifacts) + { + if (!contentByArtifactReference.TryGetValue(artifact.ArtifactReference, out var content)) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Error, + "IPCC_EFDB_CONTENT_MISSING_ARTIFACT_CONTENT", + "IPCC EFDB parser content was not provided for an input artifact.", + artifact.ArtifactReference, + fieldKey: "artifact_reference")); + continue; + } + + ParseArtifact(request, artifact, content, rows, issues); + } + + if (issues.Any(issue => issue.Severity == ParserValidationIssueSeverity.Error)) + { + return FailedResult(request, rows, issues); + } + + if (rows.Count == 0) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Warning, + "IPCC_EFDB_CONTENT_NO_RECORDS", + "IPCC EFDB content included no parseable emission factor rows.", + fieldKey: "content")); + } + + return new ParserAdapterRunResult( + request.SourceFamily, + request.SourceKey, + request.ParserKey, + ParserRunStatus.Completed, + request.Artifacts.Select(artifact => artifact.ArtifactReference), + rows, + issues, + request.RunId, + request.CorrelationId, + request.RequestedReportingYear); + } + + private static void ParseArtifact( + ParserAdapterRunRequest request, + ParserInputArtifact artifact, + string content, + ICollection rows, + ICollection issues) + { + var csvRows = CsvRows(content).ToArray(); + if (csvRows.Length == 0) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Warning, + "IPCC_EFDB_CONTENT_EMPTY", + "IPCC EFDB content input did not include parseable content.", + artifact.ArtifactReference, + fieldKey: "content")); + return; + } + + if (!csvRows[0].SequenceEqual(Header, StringComparer.Ordinal)) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Error, + "IPCC_EFDB_CONTENT_INVALID_HEADER", + "IPCC EFDB content header must match the declared parser contract.", + artifact.ArtifactReference, + sourceRowNumber: 1, + fieldKey: "header")); + return; + } + + for (var rowIndex = 1; rowIndex < csvRows.Length; rowIndex++) + { + var sourceRowNumber = rowIndex + 1; + var values = csvRows[rowIndex]; + if (values.Count == 1 && string.IsNullOrWhiteSpace(values[0])) + { + continue; + } + + if (values.Count != Header.Count) + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Error, + "IPCC_EFDB_CONTENT_INVALID_ROW", + "IPCC EFDB content row has an unexpected column count.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: $"row[{sourceRowNumber.ToString(CultureInfo.InvariantCulture)}]")); + continue; + } + + var row = Header + .Select((field, index) => new { field, value = values[index].Trim() }) + .ToDictionary(pair => pair.field, pair => pair.value, StringComparer.Ordinal); + + if (!row.Values.Any(value => !string.IsNullOrWhiteSpace(value))) + { + continue; + } + + if (row["record_type"] != "emission_factor") + { + issues.Add(Issue( + request, + ParserValidationIssueSeverity.Warning, + "IPCC_EFDB_CONTENT_UNSUPPORTED_ROW_SKIPPED", + "IPCC EFDB content row was skipped because record_type is unsupported.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: "record_type", + context: + [ + new ParserValidationIssueContext("record_type", row["record_type"]), + ])); + continue; + } + + var rowIssues = RowIssues(request, artifact, row, sourceRowNumber).ToArray(); + foreach (var rowIssue in rowIssues) + { + issues.Add(rowIssue); + } + + if (rowIssues.Length > 0) + { + continue; + } + + rows.Add(CreateOutputRow(request, artifact, row, sourceRowNumber)); + } + } + + private static IEnumerable RowIssues( + ParserAdapterRunRequest request, + ParserInputArtifact artifact, + IReadOnlyDictionary row, + int sourceRowNumber) + { + foreach (var field in Header.Where(field => RequiredFields.Contains(field) && string.IsNullOrWhiteSpace(row[field]))) + { + yield return Issue( + request, + ParserValidationIssueSeverity.Error, + "IPCC_EFDB_CONTENT_MISSING_REQUIRED_FIELD", + $"IPCC EFDB emission factor row is missing required field: {field}.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: field, + context: + [ + new ParserValidationIssueContext("row_number", sourceRowNumber.ToString(CultureInfo.InvariantCulture)), + new ParserValidationIssueContext("field_name", field), + ]); + } + + if (!int.TryParse(row["source_year"], NumberStyles.None, CultureInfo.InvariantCulture, out var sourceYear) || + sourceYear < 1) + { + yield return Issue( + request, + ParserValidationIssueSeverity.Error, + "IPCC_EFDB_CONTENT_INVALID_SOURCE_YEAR", + "IPCC EFDB source_year must be a positive integer.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: "source_year", + context: + [ + new ParserValidationIssueContext("row_number", sourceRowNumber.ToString(CultureInfo.InvariantCulture)), + new ParserValidationIssueContext("field_name", "source_year"), + new ParserValidationIssueContext("raw_value", row["source_year"]), + ]); + } + + if (!decimal.TryParse(row["factor_value"], NumberStyles.Number, CultureInfo.InvariantCulture, out _)) + { + yield return Issue( + request, + ParserValidationIssueSeverity.Error, + "IPCC_EFDB_CONTENT_INVALID_FACTOR_VALUE", + "IPCC EFDB factor_value must be numeric.", + artifact.ArtifactReference, + sourceRowNumber: sourceRowNumber, + fieldKey: "factor_value", + context: + [ + new ParserValidationIssueContext("row_number", sourceRowNumber.ToString(CultureInfo.InvariantCulture)), + new ParserValidationIssueContext("field_name", "factor_value"), + new ParserValidationIssueContext("raw_value", row["factor_value"]), + ]); + } + } + + private static ParserNormalizedOutputRow CreateOutputRow( + ParserAdapterRunRequest request, + ParserInputArtifact artifact, + IReadOnlyDictionary row, + int sourceRowNumber) + { + var sourceYear = int.Parse(row["source_year"], CultureInfo.InvariantCulture); + var rowIdentifier = string.Join( + "_", + new[] + { + "ipcc_efdb", + row["source_year"], + row["source_version"], + row["factor_id"], + $"row_{sourceRowNumber.ToString(CultureInfo.InvariantCulture)}", + }); + var masterId = $"ipcc_master_{row["source_year"]}_{row["source_version"]}_{row["factor_id"]}"; + var detailId = $"ipcc_detail_{row["source_year"]}_{row["source_version"]}_{row["factor_id"]}"; + + return new ParserNormalizedOutputRow( + SourceFamily.IpccEfdb, + request.SourceKey, + request.ParserKey, + artifact.ArtifactReference, + rowIdentifier, + sourceRowNumber, + [ + new ParserNormalizedField("source_family", SourceFamily.IpccEfdb.ToWireName()), + new ParserNormalizedField("source_year", row["source_year"]), + new ParserNormalizedField("source_version", row["source_version"]), + new ParserNormalizedField("factor_id", row["factor_id"]), + new ParserNormalizedField("factor_name", row["factor_name"]), + new ParserNormalizedField("factor_value", row["factor_value"]), + new ParserNormalizedField("unit", row["unit"]), + new ParserNormalizedField("category", row["category"]), + new ParserNormalizedField("subcategory", NullIfEmpty(row["subcategory"])), + new ParserNormalizedField("ipcc_sector", row["ipcc_sector"]), + new ParserNormalizedField("gas", row["gas"]), + new ParserNormalizedField("region", NullIfEmpty(row["region"])), + new ParserNormalizedField("technology", NullIfEmpty(row["technology"])), + new ParserNormalizedField("provenance_artifact_reference", artifact.ArtifactReference), + new ParserNormalizedField("provenance_checksum_algorithm", artifact.ChecksumAlgorithm), + new ParserNormalizedField("provenance_checksum_value", artifact.ChecksumValue), + new ParserNormalizedField("provenance_row_number", sourceRowNumber.ToString(CultureInfo.InvariantCulture)), + new ParserNormalizedField("provenance", row["provenance"]), + new ParserNormalizedField("source_family_master_id", masterId), + new ParserNormalizedField("source_family_detail_id", detailId), + new ParserNormalizedField("master_external_key", $"{row["source_year"]}:{row["source_version"]}:{row["factor_id"]}"), + new ParserNormalizedField("detail_external_key", $"{row["factor_id"]}:{row["unit"]}:{row["gas"]}:{row["ipcc_sector"]}"), + ], + reportingYear: sourceYear); + } + + private static ParserAdapterRunResult FailedResult( + ParserAdapterRunRequest request, + IEnumerable rows, + IEnumerable issues) => + new( + request.SourceFamily, + request.SourceKey, + request.ParserKey, + ParserRunStatus.Failed, + request.Artifacts.Select(artifact => artifact.ArtifactReference), + rows, + issues, + request.RunId, + request.CorrelationId, + request.RequestedReportingYear); + + private static ParserValidationIssue Issue( + ParserAdapterRunRequest request, + ParserValidationIssueSeverity severity, + string code, + string message, + string? artifactReference = null, + string? rowIdentifier = null, + int? sourceRowNumber = null, + string? fieldKey = null, + IEnumerable? context = null) => + new( + request.SourceFamily, + request.SourceKey, + request.ParserKey, + severity, + code, + message, + artifactReference, + rowIdentifier, + sourceRowNumber, + fieldKey, + context); + + private static IEnumerable> CsvRows(string content) + { + var row = new List(); + var field = new List(); + var inQuotes = false; + + for (var index = 0; index < content.Length; index++) + { + var current = content[index]; + if (inQuotes) + { + if (current == '"' && index + 1 < content.Length && content[index + 1] == '"') + { + field.Add('"'); + index++; + continue; + } + + if (current == '"') + { + inQuotes = false; + continue; + } + + field.Add(current); + continue; + } + + switch (current) + { + case '"': + inQuotes = true; + break; + case ',': + row.Add(new string(field.ToArray())); + field.Clear(); + break; + case '\r': + break; + case '\n': + row.Add(new string(field.ToArray())); + field.Clear(); + yield return row.ToArray(); + row.Clear(); + break; + default: + field.Add(current); + break; + } + } + + if (field.Count > 0 || row.Count > 0) + { + row.Add(new string(field.ToArray())); + yield return row.ToArray(); + } + } + + private static string? NullIfEmpty(string value) => + string.IsNullOrWhiteSpace(value) ? null : value; +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryBoundary.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryBoundary.cs new file mode 100644 index 00000000..46051a32 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryBoundary.cs @@ -0,0 +1,403 @@ +namespace CarbonOps.Parser.Contracts; + +public static class IpccSourceDiscoveryBoundary +{ + private const string IpccSourceKey = "ipcc_efdb"; + private const string DiscoveryReferenceUri = "discovery://ipcc_efdb/homepage"; + private const string ArtifactKind = "discovery"; + + public static IpccSourceDiscoveryRequest CreateRequest() => + new( + SourceFamily.IpccEfdb, + IpccSourceKey, + DiscoveryReferenceUri); + + public static IpccSourceDiscoveryResult CreateResult(IpccSourceDiscoveryRequest? request = null) + { + var activeRequest = request ?? CreateRequest(); + var requestValidation = Validate(activeRequest); + if (!requestValidation.IsValid) + { + return new IpccSourceDiscoveryResult( + IpccSourceDiscoveryStatus.Invalid, + activeRequest, + Array.Empty(), + requestValidation.Issues); + } + + var candidate = new IpccSourceDocumentCandidate( + SourceFamily.IpccEfdb, + IpccSourceKey, + "ipcc_source_discovery_candidate_001_ipcc_efdb", + "IPCC EFDB", + activeRequest.DiscoveryReferenceUri, + ArtifactKind, + versionLabel: "dn049_ipcc_discovery_boundary", + discoveredAtLabel: "runtime_passive_discovery_unavailable"); + var candidateValidation = Validate(candidate); + + return new IpccSourceDiscoveryResult( + candidateValidation.IsValid ? IpccSourceDiscoveryStatus.Declared : IpccSourceDiscoveryStatus.Invalid, + activeRequest, + candidateValidation.IsValid ? new[] { candidate } : Array.Empty(), + candidateValidation.Issues); + } + + public static IpccSourceDiscoveryValidationResult Validate(IpccSourceDiscoveryRequest? request) + { + var issues = new List(); + + if (request is null) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_MISSING_REQUEST", + "request is required.", + "request")); + return new IpccSourceDiscoveryValidationResult(issues); + } + + if (!Enum.IsDefined(request.SourceFamily)) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_INVALID_SOURCE_FAMILY", + "source_family must be a defined source family.", + "source_family")); + } + + ValidateRequiredText( + request.SourceKey, + "source_key", + "IPCC_SOURCE_DISCOVERY_MISSING_SOURCE_KEY", + "source_key must be a non-empty string.", + issues); + ValidateRequiredText( + request.DiscoveryReferenceUri, + "discovery_reference_uri", + "IPCC_SOURCE_DISCOVERY_MISSING_REFERENCE_URI", + "discovery_reference_uri must be a non-empty string.", + issues); + + if (request.SourceFamily != SourceFamily.IpccEfdb) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_SOURCE_FAMILY_MISMATCH", + "source_family must be ipcc_efdb.", + "source_family")); + } + + if (request.SourceKey != IpccSourceKey) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_SOURCE_KEY_MISMATCH", + "source_key must be ipcc_efdb.", + "source_key")); + } + + if (request.Mode != IpccSourceDiscoveryMode.RuntimePassive) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_UNSUPPORTED_MODE", + "mode must remain runtime_passive.", + "mode")); + } + + ValidateFalse( + request.AllowNetwork, + "allow_network", + "IPCC_SOURCE_DISCOVERY_NETWORK_NOT_ALLOWED", + "allow_network must be false for this boundary.", + issues); + ValidateFalse( + request.AllowDownload, + "allow_download", + "IPCC_SOURCE_DISCOVERY_DOWNLOAD_NOT_ALLOWED", + "allow_download must be false for this boundary.", + issues); + ValidateFalse( + request.AllowParse, + "allow_parse", + "IPCC_SOURCE_DISCOVERY_PARSE_NOT_ALLOWED", + "allow_parse must be false for this boundary.", + issues); + ValidateFalse( + request.AllowDatabaseWrites, + "allow_database_writes", + "IPCC_SOURCE_DISCOVERY_DATABASE_WRITES_NOT_ALLOWED", + "allow_database_writes must be false for this boundary.", + issues); + ValidateFalse( + request.AllowScheduler, + "allow_scheduler", + "IPCC_SOURCE_DISCOVERY_SCHEDULER_NOT_ALLOWED", + "allow_scheduler must be false for this boundary.", + issues); + + return new IpccSourceDiscoveryValidationResult(issues); + } + + public static IpccSourceDiscoveryValidationResult Validate(IpccSourceDocumentCandidate? candidate) + { + var issues = new List(); + + if (candidate is null) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING", + "candidate is required.", + "candidate")); + return new IpccSourceDiscoveryValidationResult(issues); + } + + if (!Enum.IsDefined(candidate.SourceFamily)) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_CANDIDATE_INVALID_SOURCE_FAMILY", + "source_family must be a defined source family.", + "source_family")); + } + + ValidateRequiredText( + candidate.SourceKey, + "source_key", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_SOURCE_KEY", + "source_key must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.CandidateId, + "candidate_id", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_CANDIDATE_ID", + "candidate_id must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.Title, + "title", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_TITLE", + "title must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.ReferenceUri, + "reference_uri", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_REFERENCE_URI", + "reference_uri must be a non-empty string.", + issues); + ValidateRequiredText( + candidate.ArtifactKind, + "artifact_kind", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_ARTIFACT_KIND", + "artifact_kind must be a non-empty string.", + issues); + ValidateOptionalText( + candidate.ContentType, + "content_type", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_BLANK_CONTENT_TYPE", + "content_type must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.Extension, + "extension", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_BLANK_EXTENSION", + "extension must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.ChecksumSha256, + "checksum_sha256", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_BLANK_CHECKSUM_SHA256", + "checksum_sha256 must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.VersionLabel, + "version_label", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_BLANK_VERSION_LABEL", + "version_label must be non-empty when provided.", + issues); + ValidateOptionalText( + candidate.DiscoveredAtLabel, + "discovered_at_label", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_BLANK_DISCOVERED_AT_LABEL", + "discovered_at_label must be non-empty when provided.", + issues); + ValidateOptionalPositiveInt( + candidate.DocumentYear, + "document_year", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_INVALID_DOCUMENT_YEAR", + "document_year must be a positive integer when provided.", + issues); + ValidateOptionalPositiveInt( + candidate.ReportingYear, + "reporting_year", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_INVALID_REPORTING_YEAR", + "reporting_year must be a positive integer when provided.", + issues); + + if (candidate.SourceFamily != SourceFamily.IpccEfdb) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_CANDIDATE_SOURCE_FAMILY_MISMATCH", + "source_family must match the IPCC source family.", + "source_family")); + } + + if (candidate.SourceKey != IpccSourceKey) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_CANDIDATE_SOURCE_KEY_MISMATCH", + "source_key must match the IPCC source key.", + "source_key")); + } + + if (candidate.ArtifactKind != ArtifactKind) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_CANDIDATE_ARTIFACT_KIND_MISMATCH", + "artifact_kind must match the IPCC expected format.", + "artifact_kind")); + } + + if (candidate.Status != IpccSourceDiscoveryStatus.Declared) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_CANDIDATE_UNSUPPORTED_STATUS", + "candidate status must remain declared.", + "status")); + } + + if (candidate.DownloadAllowed) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_CANDIDATE_DOWNLOAD_NOT_ALLOWED", + "download_allowed must be false for this boundary.", + "download_allowed")); + } + + return new IpccSourceDiscoveryValidationResult(issues); + } + + public static IpccSourceDiscoveryValidationResult Validate(IpccSourceDiscoveryResult? result) + { + var issues = new List(); + + if (result is null) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_RESULT_MISSING", + "result is required.", + "result")); + return new IpccSourceDiscoveryValidationResult(issues); + } + + issues.AddRange(Validate(result.Request).Issues); + + if (!Enum.IsDefined(result.Status)) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_RESULT_INVALID_STATUS", + "status must be a defined IPCC source discovery status.", + "status")); + } + + foreach (var (fieldName, value) in new[] + { + ("no_network", result.NoNetwork), + ("no_download", result.NoDownload), + ("no_parse", result.NoParse), + ("no_database_writes", result.NoDatabaseWrites), + ("no_sql", result.NoSql), + ("no_scheduler", result.NoScheduler), + }) + { + if (!value) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED", + $"{fieldName} must remain true.", + fieldName)); + } + } + + for (var index = 0; index < result.Candidates.Count; index++) + { + foreach (var issue in Validate(result.Candidates[index]).Issues) + { + issues.Add(issue with { FieldName = $"candidates[{index + 1}].{issue.FieldName}" }); + } + } + + if (result.Status == IpccSourceDiscoveryStatus.Declared && result.Issues.Count > 0) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + "declared result status must not include issue metadata.", + "issues")); + } + + if (result.Status == IpccSourceDiscoveryStatus.Declared && issues.Count > 0) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + "declared result status requires valid metadata.", + "status")); + } + + if (result.Status == IpccSourceDiscoveryStatus.Invalid && result.Issues.Count == 0) + { + issues.Add(new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_RESULT_MISSING_INVALID_ISSUES", + "invalid result status requires issue metadata.", + "issues")); + } + + return new IpccSourceDiscoveryValidationResult(issues); + } + + private static void ValidateRequiredText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (string.IsNullOrWhiteSpace(value)) + { + issues.Add(new IpccSourceDiscoveryIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is not null && string.IsNullOrWhiteSpace(value)) + { + issues.Add(new IpccSourceDiscoveryIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalPositiveInt( + int? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is <= 0) + { + issues.Add(new IpccSourceDiscoveryIssue(code, message, fieldName)); + } + } + + private static void ValidateFalse( + bool value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value) + { + issues.Add(new IpccSourceDiscoveryIssue(code, message, fieldName)); + } + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryIssue.cs new file mode 100644 index 00000000..f478d44c --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDiscoveryIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryMode.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryMode.cs new file mode 100644 index 00000000..dd67e458 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryMode.cs @@ -0,0 +1,6 @@ +namespace CarbonOps.Parser.Contracts; + +public enum IpccSourceDiscoveryMode +{ + RuntimePassive = 0, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryRequest.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryRequest.cs new file mode 100644 index 00000000..5b265903 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryRequest.cs @@ -0,0 +1,44 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDiscoveryRequest +{ + public SourceFamily SourceFamily { get; } + + public string SourceKey { get; } + + public string DiscoveryReferenceUri { get; } + + public IpccSourceDiscoveryMode Mode { get; } + + public bool AllowNetwork { get; } + + public bool AllowDownload { get; } + + public bool AllowParse { get; } + + public bool AllowDatabaseWrites { get; } + + public bool AllowScheduler { get; } + + public IpccSourceDiscoveryRequest( + SourceFamily sourceFamily, + string sourceKey, + string discoveryReferenceUri, + IpccSourceDiscoveryMode mode = IpccSourceDiscoveryMode.RuntimePassive, + bool allowNetwork = false, + bool allowDownload = false, + bool allowParse = false, + bool allowDatabaseWrites = false, + bool allowScheduler = false) + { + SourceFamily = sourceFamily; + SourceKey = sourceKey; + DiscoveryReferenceUri = discoveryReferenceUri; + Mode = mode; + AllowNetwork = allowNetwork; + AllowDownload = allowDownload; + AllowParse = allowParse; + AllowDatabaseWrites = allowDatabaseWrites; + AllowScheduler = allowScheduler; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryResult.cs new file mode 100644 index 00000000..c2a07acb --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryResult.cs @@ -0,0 +1,55 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDiscoveryResult +{ + public IpccSourceDiscoveryStatus Status { get; } + + public IpccSourceDiscoveryRequest Request { get; } + + public IReadOnlyList Candidates { get; } + + public IReadOnlyList Issues { get; } + + public bool NoNetwork { get; } + + public bool NoDownload { get; } + + public bool NoParse { get; } + + public bool NoDatabaseWrites { get; } + + public bool NoSql { get; } + + public bool NoScheduler { get; } + + public int CandidateCount => Candidates.Count; + + public IReadOnlyList CandidateIds { get; } + + public IpccSourceDiscoveryResult( + IpccSourceDiscoveryStatus status, + IpccSourceDiscoveryRequest request, + IEnumerable candidates, + IEnumerable? issues = null, + bool noNetwork = true, + bool noDownload = true, + bool noParse = true, + bool noDatabaseWrites = true, + bool noSql = true, + bool noScheduler = true) + { + var candidateSnapshot = candidates.ToArray(); + + Status = status; + Request = request; + Candidates = Array.AsReadOnly(candidateSnapshot); + Issues = Array.AsReadOnly((issues ?? Array.Empty()).ToArray()); + NoNetwork = noNetwork; + NoDownload = noDownload; + NoParse = noParse; + NoDatabaseWrites = noDatabaseWrites; + NoSql = noSql; + NoScheduler = noScheduler; + CandidateIds = Array.AsReadOnly(candidateSnapshot.Select(candidate => candidate.CandidateId).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryStatus.cs new file mode 100644 index 00000000..3784d19d --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryStatus.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public enum IpccSourceDiscoveryStatus +{ + Declared = 0, + Invalid = 1, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryValidationResult.cs new file mode 100644 index 00000000..8b6d8ca9 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDiscoveryValidationResult.cs @@ -0,0 +1,13 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDiscoveryValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public IpccSourceDiscoveryValidationResult(IEnumerable? issues = null) + { + Issues = Array.AsReadOnly((issues ?? Array.Empty()).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDocumentCandidate.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDocumentCandidate.cs new file mode 100644 index 00000000..2a042b40 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDocumentCandidate.cs @@ -0,0 +1,68 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDocumentCandidate +{ + public SourceFamily SourceFamily { get; } + + public string SourceKey { get; } + + public string CandidateId { get; } + + public string Title { get; } + + public string ReferenceUri { get; } + + public string ArtifactKind { get; } + + public IpccSourceDiscoveryStatus Status { get; } + + public int? DocumentYear { get; } + + public int? ReportingYear { get; } + + public string? ContentType { get; } + + public string? Extension { get; } + + public string? ChecksumSha256 { get; } + + public string? VersionLabel { get; } + + public string? DiscoveredAtLabel { get; } + + public bool DownloadAllowed { get; } + + public IpccSourceDocumentCandidate( + SourceFamily sourceFamily, + string sourceKey, + string candidateId, + string title, + string referenceUri, + string artifactKind, + IpccSourceDiscoveryStatus status = IpccSourceDiscoveryStatus.Declared, + int? documentYear = null, + int? reportingYear = null, + string? contentType = null, + string? extension = null, + string? checksumSha256 = null, + string? versionLabel = null, + string? discoveredAtLabel = null, + bool downloadAllowed = false) + { + SourceFamily = sourceFamily; + SourceKey = sourceKey; + CandidateId = candidateId; + Title = title; + ReferenceUri = referenceUri; + ArtifactKind = artifactKind; + Status = status; + DocumentYear = documentYear; + ReportingYear = reportingYear; + ContentType = contentType; + Extension = extension; + ChecksumSha256 = checksumSha256; + VersionLabel = versionLabel; + DiscoveredAtLabel = discoveredAtLabel; + DownloadAllowed = downloadAllowed; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionBoundary.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionBoundary.cs new file mode 100644 index 00000000..5e00a673 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionBoundary.cs @@ -0,0 +1,850 @@ +using System.Security.Cryptography; + +namespace CarbonOps.Parser.Contracts; + +public static class IpccSourceDownloadExecutionBoundary +{ + private const string IpccSourceKey = "ipcc_efdb"; + + public static IpccSourceDownloadExecutionRequest CreateRequest( + IpccSourceDocumentCandidate candidate, + string targetRoot, + string targetRelativePath, + bool allowDownloadExecution = false, + bool allowFileWrite = false, + bool allowNetwork = false, + bool allowOverwrite = false) => + new( + candidate.SourceFamily, + candidate.SourceKey, + candidate.CandidateId, + candidate.Title, + candidate.ReferenceUri, + candidate.ArtifactKind, + targetRoot, + targetRelativePath, + candidate.DownloadAllowed, + allowDownloadExecution, + allowFileWrite, + allowNetwork, + allowOverwrite, + contentType: candidate.ContentType, + extension: candidate.Extension, + expectedChecksumSha256: candidate.ChecksumSha256, + documentYear: candidate.DocumentYear, + reportingYear: candidate.ReportingYear, + versionLabel: candidate.VersionLabel); + + public static IpccSourceDownloadExecutionValidationResult Validate( + IpccSourceDownloadExecutionRequest? request) + { + var issues = new List(); + + if (request is null) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_MISSING_REQUEST", + "request is required.", + "request")); + return new IpccSourceDownloadExecutionValidationResult(issues); + } + + if (!Enum.IsDefined(request.SourceFamily)) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_INVALID_SOURCE_FAMILY", + "source_family must be a defined source family.", + "source_family")); + } + + ValidateRequiredText( + request.SourceKey, + "source_key", + "IPCC_SOURCE_DOWNLOAD_MISSING_SOURCE_KEY", + "source_key must be a non-empty string.", + issues); + ValidateRequiredText( + request.CandidateId, + "candidate_id", + "IPCC_SOURCE_DOWNLOAD_MISSING_CANDIDATE_ID", + "candidate_id must be a non-empty string.", + issues); + ValidateRequiredText( + request.CandidateTitle, + "candidate_title", + "IPCC_SOURCE_DOWNLOAD_MISSING_CANDIDATE_TITLE", + "candidate_title must be a non-empty string.", + issues); + ValidateRequiredText( + request.SourceReferenceUri, + "source_reference_uri", + "IPCC_SOURCE_DOWNLOAD_MISSING_SOURCE_REFERENCE_URI", + "source_reference_uri must be a non-empty string.", + issues); + ValidateRequiredText( + request.ArtifactKind, + "artifact_kind", + "IPCC_SOURCE_DOWNLOAD_MISSING_ARTIFACT_KIND", + "artifact_kind must be a non-empty string.", + issues); + ValidateRequiredText( + request.TargetRoot, + "target_root", + "IPCC_SOURCE_DOWNLOAD_MISSING_TARGET_ROOT", + "target_root must be a non-empty string.", + issues); + ValidateRequiredText( + request.TargetRelativePath, + "target_relative_path", + "IPCC_SOURCE_DOWNLOAD_MISSING_TARGET_RELATIVE_PATH", + "target_relative_path must be a non-empty string.", + issues); + ValidateOptionalText( + request.ContentType, + "content_type", + "IPCC_SOURCE_DOWNLOAD_BLANK_CONTENT_TYPE", + "content_type must be non-empty when provided.", + issues); + ValidateOptionalText( + request.Extension, + "extension", + "IPCC_SOURCE_DOWNLOAD_BLANK_EXTENSION", + "extension must be non-empty when provided.", + issues); + ValidateOptionalText( + request.ExpectedChecksumSha256, + "expected_checksum_sha256", + "IPCC_SOURCE_DOWNLOAD_BLANK_EXPECTED_CHECKSUM_SHA256", + "expected_checksum_sha256 must be non-empty when provided.", + issues); + ValidateOptionalText( + request.VersionLabel, + "version_label", + "IPCC_SOURCE_DOWNLOAD_BLANK_VERSION_LABEL", + "version_label must be non-empty when provided.", + issues); + ValidateOptionalPositiveInt( + request.DocumentYear, + "document_year", + "IPCC_SOURCE_DOWNLOAD_INVALID_DOCUMENT_YEAR", + "document_year must be a positive integer when provided.", + issues); + ValidateOptionalPositiveInt( + request.ReportingYear, + "reporting_year", + "IPCC_SOURCE_DOWNLOAD_INVALID_REPORTING_YEAR", + "reporting_year must be a positive integer when provided.", + issues); + + if (request.SourceFamily != SourceFamily.IpccEfdb) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_SOURCE_FAMILY_MISMATCH", + "source_family must be ipcc_efdb.", + "source_family")); + } + + if (request.SourceKey != IpccSourceKey) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_SOURCE_KEY_MISMATCH", + "source_key must be ipcc_efdb.", + "source_key")); + } + + ValidateTrue( + request.CandidateDownloadAllowed, + "candidate_download_allowed", + "IPCC_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE", + "candidate metadata must explicitly allow download execution.", + issues); + ValidateTrue( + request.AllowDownloadExecution, + "allow_download_execution", + "IPCC_SOURCE_DOWNLOAD_EXECUTION_NOT_ALLOWED", + "allow_download_execution must be true.", + issues); + ValidateTrue( + request.AllowFileWrite, + "allow_file_write", + "IPCC_SOURCE_DOWNLOAD_FILE_WRITE_NOT_ALLOWED", + "allow_file_write must be true.", + issues); + ValidateFalse( + request.AllowParse, + "allow_parse", + "IPCC_SOURCE_DOWNLOAD_PARSE_NOT_ALLOWED", + "allow_parse must be false for this boundary.", + issues); + ValidateFalse( + request.AllowDatabaseWrites, + "allow_database_writes", + "IPCC_SOURCE_DOWNLOAD_DATABASE_WRITES_NOT_ALLOWED", + "allow_database_writes must be false for this boundary.", + issues); + ValidateFalse( + request.AllowScheduler, + "allow_scheduler", + "IPCC_SOURCE_DOWNLOAD_SCHEDULER_NOT_ALLOWED", + "allow_scheduler must be false for this boundary.", + issues); + + ValidateSourceReferenceUri(request, issues); + ValidateTargetPaths(request, issues); + + return new IpccSourceDownloadExecutionValidationResult(issues); + } + + public static IpccSourceDownloadExecutionResult Execute( + IpccSourceDownloadExecutionRequest request, + Func transport, + Func? utcNow = null) + { + var validation = Validate(request); + if (!validation.IsValid) + { + return new IpccSourceDownloadExecutionResult( + IpccSourceDownloadExecutionStatus.Blocked, + request, + issues: validation.Issues); + } + + var safeTarget = PrepareSafeTargetPath(request); + if (safeTarget.FileAlreadyKnown) + { + var retrievedAtUtc = GetRetrievedAtUtc(utcNow); + return new IpccSourceDownloadExecutionResult( + IpccSourceDownloadExecutionStatus.AlreadyKnown, + request, + CreateArtifact( + request, + safeTarget.TargetPath, + request.ExpectedChecksumSha256!, + retrievedAtUtc, + null, + null)); + } + + if (!safeTarget.Validation.IsValid) + { + return new IpccSourceDownloadExecutionResult( + IpccSourceDownloadExecutionStatus.Blocked, + request, + issues: safeTarget.Validation.Issues); + } + + IpccSourceDownloadTransportResponse response; + try + { + response = transport(request.SourceReferenceUri); + } + catch (Exception error) + { + return new IpccSourceDownloadExecutionResult( + IpccSourceDownloadExecutionStatus.Failed, + request, + issues: + [ + new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_TRANSPORT_FAILED", + $"transport failed: {error.Message}", + "source_reference_uri"), + ]); + } + + var responseValidation = ValidateTransportResponse(response); + if (!responseValidation.IsValid) + { + return new IpccSourceDownloadExecutionResult( + IpccSourceDownloadExecutionStatus.Failed, + request, + issues: responseValidation.Issues); + } + + var checksum = Convert.ToHexString(SHA256.HashData(response.Content)).ToLowerInvariant(); + if (request.ExpectedChecksumSha256 is not null + && !string.Equals(checksum, request.ExpectedChecksumSha256, StringComparison.OrdinalIgnoreCase)) + { + return new IpccSourceDownloadExecutionResult( + IpccSourceDownloadExecutionStatus.Failed, + request, + issues: + [ + new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH", + "downloaded content checksum did not match expected value.", + "expected_checksum_sha256"), + ]); + } + + safeTarget = PrepareSafeTargetPath(request); + if (safeTarget.FileAlreadyKnown) + { + var retrievedAtUtc = GetRetrievedAtUtc(utcNow); + return new IpccSourceDownloadExecutionResult( + IpccSourceDownloadExecutionStatus.AlreadyKnown, + request, + CreateArtifact( + request, + safeTarget.TargetPath, + checksum, + retrievedAtUtc, + response.ContentType, + response.FinalUri)); + } + + if (!safeTarget.Validation.IsValid) + { + return new IpccSourceDownloadExecutionResult( + IpccSourceDownloadExecutionStatus.Blocked, + request, + issues: safeTarget.Validation.Issues); + } + + try + { + WriteContentToSafeTarget(safeTarget.TargetPath, response.Content, request.AllowOverwrite); + } + catch (IOException error) when (File.Exists(safeTarget.TargetPath) && !request.AllowOverwrite) + { + return WriteFailed(request, "IPCC_SOURCE_DOWNLOAD_TARGET_EXISTS", error); + } + catch (Exception error) + { + return WriteFailed(request, "IPCC_SOURCE_DOWNLOAD_WRITE_FAILED", error); + } + + var artifact = CreateArtifact( + request, + safeTarget.TargetPath, + checksum, + GetRetrievedAtUtc(utcNow), + response.ContentType, + response.FinalUri); + + return new IpccSourceDownloadExecutionResult( + IpccSourceDownloadExecutionStatus.Downloaded, + request, + artifact); + } + + public static IpccSourceDownloadExecutionValidationResult Validate( + IpccSourceDownloadExecutionResult? result) + { + var issues = new List(); + + if (result is null) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_RESULT_MISSING", + "result is required.", + "result")); + return new IpccSourceDownloadExecutionValidationResult(issues); + } + + issues.AddRange(Validate(result.Request).Issues); + + if (!Enum.IsDefined(result.Status)) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_RESULT_INVALID_STATUS", + "status must be a defined IPCC source download execution status.", + "status")); + } + + foreach (var (fieldName, value) in new[] + { + ("no_parse", result.NoParse), + ("no_database_writes", result.NoDatabaseWrites), + ("no_sql", result.NoSql), + ("no_scheduler", result.NoScheduler), + }) + { + if (!value) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED", + $"{fieldName} must remain true.", + fieldName)); + } + } + + if ((result.Status == IpccSourceDownloadExecutionStatus.Downloaded + || result.Status == IpccSourceDownloadExecutionStatus.AlreadyKnown) + && result.Artifact is null) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_RESULT_MISSING_ARTIFACT", + "downloaded or already-known results require artifact metadata.", + "artifact")); + } + else if (result.Status != IpccSourceDownloadExecutionStatus.Downloaded + && result.Status != IpccSourceDownloadExecutionStatus.AlreadyKnown + && result.Artifact is not null) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_RESULT_UNEXPECTED_ARTIFACT", + "blocked or failed results must not include artifact metadata.", + "artifact")); + } + + if (result.Status != IpccSourceDownloadExecutionStatus.Downloaded + && result.Status != IpccSourceDownloadExecutionStatus.AlreadyKnown + && result.Issues.Count == 0) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_RESULT_MISSING_ISSUES", + "blocked or failed results require issue metadata.", + "issues")); + } + + if (result.Artifact is not null) + { + ValidateArtifact(result.Artifact, issues); + } + + return new IpccSourceDownloadExecutionValidationResult(issues); + } + + private static IpccSourceDownloadedArtifact CreateArtifact( + IpccSourceDownloadExecutionRequest request, + string targetPath, + string checksum, + DateTimeOffset retrievedAtUtc, + string? responseContentType, + string? finalUri) + { + var sizeBytes = File.Exists(targetPath) ? new FileInfo(targetPath).Length : 0; + + return new IpccSourceDownloadedArtifact( + request.SourceFamily, + request.SourceKey, + request.CandidateId, + $"ipcc_source_download_artifact_{request.CandidateId}", + request.ArtifactKind, + request.SourceReferenceUri, + targetPath, + Path.GetFileName(targetPath), + checksum, + sizeBytes, + retrievedAtUtc, + responseContentType ?? request.ContentType, + request.Extension, + finalUri, + request.DocumentYear, + request.ReportingYear, + request.VersionLabel); + } + + private static IpccSourceDownloadExecutionResult WriteFailed( + IpccSourceDownloadExecutionRequest request, + string code, + Exception error) => + new( + IpccSourceDownloadExecutionStatus.Failed, + request, + issues: + [ + new IpccSourceDownloadExecutionIssue( + code, + $"target write failed: {error.Message}", + "target_relative_path"), + ]); + + private static IpccSourceDownloadExecutionValidationResult ValidateTransportResponse( + IpccSourceDownloadTransportResponse? response) + { + var issues = new List(); + + if (response is null) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_RESPONSE_MISSING", + "transport response is required.", + "transport")); + return new IpccSourceDownloadExecutionValidationResult(issues); + } + + if (response.Content is null) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT", + "transport response content is required.", + "content")); + } + else if (response.Content.Length == 0) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_RESPONSE_EMPTY_CONTENT", + "transport response content must not be empty.", + "content")); + } + + ValidateOptionalText( + response.ContentType, + "content_type", + "IPCC_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "response content_type must be non-empty when provided.", + issues); + ValidateOptionalText( + response.FinalUri, + "final_uri", + "IPCC_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + "response final_uri must be non-empty when provided.", + issues); + + return new IpccSourceDownloadExecutionValidationResult(issues); + } + + private static (string TargetPath, bool FileAlreadyKnown, IpccSourceDownloadExecutionValidationResult Validation) + PrepareSafeTargetPath(IpccSourceDownloadExecutionRequest request) + { + var issues = new List(); + + string root; + string targetPath; + try + { + root = Path.GetFullPath(request.TargetRoot); + targetPath = Path.GetFullPath(Path.Combine(root, request.TargetRelativePath)); + } + catch (Exception error) when (error is ArgumentException or NotSupportedException or PathTooLongException) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_TARGET_PATH_UNRESOLVED", + "target path could not be resolved safely.", + "target_relative_path")); + return (string.Empty, false, new IpccSourceDownloadExecutionValidationResult(issues)); + } + + if (!IsPathInsideRoot(root, targetPath)) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE", + "target_relative_path must stay within target_root.", + "target_relative_path")); + } + + if (ContainsExistingSymlink(root, targetPath)) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE", + "target path must not traverse an existing symbolic link.", + "target_relative_path")); + } + + if (issues.Count == 0 + && File.Exists(targetPath) + && !request.AllowOverwrite + && !string.IsNullOrWhiteSpace(request.ExpectedChecksumSha256)) + { + var existingChecksum = Convert + .ToHexString(SHA256.HashData(File.ReadAllBytes(targetPath))) + .ToLowerInvariant(); + if (string.Equals(existingChecksum, request.ExpectedChecksumSha256, StringComparison.OrdinalIgnoreCase)) + { + return (targetPath, true, new IpccSourceDownloadExecutionValidationResult()); + } + } + + if (File.Exists(targetPath) && !request.AllowOverwrite) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_TARGET_EXISTS", + "target path already exists and allow_overwrite is false.", + "target_relative_path")); + } + + return (targetPath, false, new IpccSourceDownloadExecutionValidationResult(issues)); + } + + private static void WriteContentToSafeTarget(string targetPath, byte[] content, bool allowOverwrite) + { + var parent = Path.GetDirectoryName(targetPath); + if (!string.IsNullOrWhiteSpace(parent)) + { + Directory.CreateDirectory(parent); + } + + if (IsSymlink(targetPath)) + { + throw new IOException("target path is a symbolic link."); + } + + var mode = allowOverwrite ? FileMode.Create : FileMode.CreateNew; + using var stream = new FileStream(targetPath, mode, FileAccess.Write, FileShare.None); + stream.Write(content, 0, content.Length); + } + + private static bool IsPathInsideRoot(string root, string targetPath) + { + var rootWithSeparator = Path.EndsInDirectorySeparator(root) + ? root + : root + Path.DirectorySeparatorChar; + + return targetPath.StartsWith(rootWithSeparator, StringComparison.Ordinal) + || string.Equals(root, targetPath, StringComparison.Ordinal); + } + + private static bool ContainsExistingSymlink(string root, string targetPath) + { + var relative = Path.GetRelativePath(root, targetPath); + if (relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative)) + { + return false; + } + + if (IsSymlink(root)) + { + return true; + } + + var current = root; + foreach (var segment in relative.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)) + { + if (string.IsNullOrWhiteSpace(segment)) + { + continue; + } + + current = Path.Combine(current, segment); + if ((Directory.Exists(current) || File.Exists(current)) && IsSymlink(current)) + { + return true; + } + } + + return false; + } + + private static bool IsSymlink(string path) + { + try + { + return (File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0; + } + catch (FileNotFoundException) + { + return false; + } + catch (DirectoryNotFoundException) + { + return false; + } + } + + private static void ValidateSourceReferenceUri( + IpccSourceDownloadExecutionRequest request, + ICollection issues) + { + if (string.IsNullOrWhiteSpace(request.SourceReferenceUri)) + { + return; + } + + if (!Uri.TryCreate(request.SourceReferenceUri, UriKind.Absolute, out var uri)) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + request.SourceReferenceUri.Contains("://", StringComparison.Ordinal) + ? "IPCC_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI" + : "IPCC_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME", + request.SourceReferenceUri.Contains("://", StringComparison.Ordinal) + ? "source_reference_uri must be a well-formed URI." + : "source_reference_uri must include a URI scheme.", + "source_reference_uri")); + return; + } + + if ((uri.Scheme == Uri.UriSchemeHttps || uri.Scheme == Uri.UriSchemeHttp) + && string.IsNullOrWhiteSpace(uri.Host)) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + "source_reference_uri must be a well-formed URI.", + "source_reference_uri")); + } + else if (uri.Scheme == "discovery") + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE", + "discovery references are not direct download references.", + "source_reference_uri")); + } + else if (uri.Scheme == Uri.UriSchemeHttps && !request.AllowNetwork) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_NETWORK_NOT_ALLOWED", + "allow_network must be true for https source references.", + "source_reference_uri")); + } + else if (uri.Scheme == Uri.UriSchemeHttp) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_INSECURE_HTTP_NOT_ALLOWED", + "http source references are not allowed.", + "source_reference_uri")); + } + else if (uri.Scheme is not "mock" and not "memory") + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI", + "source_reference_uri must use an allowed execution scheme.", + "source_reference_uri")); + } + } + + private static void ValidateTargetPaths( + IpccSourceDownloadExecutionRequest request, + ICollection issues) + { + if (!string.IsNullOrWhiteSpace(request.TargetRoot) && !Path.IsPathFullyQualified(request.TargetRoot)) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_ABSOLUTE", + "target_root must be an absolute path.", + "target_root")); + } + + if (string.IsNullOrWhiteSpace(request.TargetRelativePath)) + { + return; + } + + if (Path.IsPathFullyQualified(request.TargetRelativePath)) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_ABSOLUTE", + "target_relative_path must be relative.", + "target_relative_path")); + } + + if (Uri.TryCreate(request.TargetRelativePath, UriKind.Absolute, out var targetUri) + && !string.IsNullOrWhiteSpace(targetUri.Scheme)) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_URI", + "target_relative_path must not be a URI.", + "target_relative_path")); + } + + var segments = request.TargetRelativePath.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + if (segments.Any(segment => segment == "..")) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE", + "target_relative_path must not contain parent traversal.", + "target_relative_path")); + } + } + + private static DateTimeOffset NormalizeRetrievedAtUtc(DateTimeOffset value) => + value.ToUniversalTime(); + + private static DateTimeOffset GetRetrievedAtUtc(Func? utcNow) => + NormalizeRetrievedAtUtc(utcNow?.Invoke() ?? DateTimeOffset.UtcNow); + + private static void ValidateArtifact( + IpccSourceDownloadedArtifact artifact, + ICollection issues) + { + if (artifact.SourceFamily != SourceFamily.IpccEfdb) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_SOURCE_FAMILY_MISMATCH", + "artifact source_family must be ipcc_efdb.", + "artifact.source_family")); + } + + if (artifact.RetrievedAtUtc.Offset != TimeSpan.Zero) + { + issues.Add(new IpccSourceDownloadExecutionIssue( + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_RETRIEVED_AT_NOT_UTC", + "retrieved_at_utc must use UTC offset semantics.", + "artifact.retrieved_at_utc")); + } + + ValidateRequiredText( + artifact.SourceReferenceUri, + "artifact.source_reference_uri", + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_MISSING_SOURCE_REFERENCE_URI", + "artifact source_reference_uri must be a non-empty string.", + issues); + ValidateRequiredText( + artifact.LocalPath, + "artifact.local_path", + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_MISSING_LOCAL_PATH", + "artifact local_path must be a non-empty string.", + issues); + ValidateRequiredText( + artifact.ChecksumSha256, + "artifact.checksum_sha256", + "IPCC_SOURCE_DOWNLOAD_ARTIFACT_MISSING_CHECKSUM_SHA256", + "artifact checksum_sha256 must be a non-empty string.", + issues); + } + + private static void ValidateRequiredText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (string.IsNullOrWhiteSpace(value)) + { + issues.Add(new IpccSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is not null && string.IsNullOrWhiteSpace(value)) + { + issues.Add(new IpccSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalPositiveInt( + int? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is <= 0) + { + issues.Add(new IpccSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateTrue( + bool value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (!value) + { + issues.Add(new IpccSourceDownloadExecutionIssue(code, message, fieldName)); + } + } + + private static void ValidateFalse( + bool value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value) + { + issues.Add(new IpccSourceDownloadExecutionIssue(code, message, fieldName)); + } + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionIssue.cs new file mode 100644 index 00000000..35bba4ab --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDownloadExecutionIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionRequest.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionRequest.cs new file mode 100644 index 00000000..ca9172f5 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionRequest.cs @@ -0,0 +1,96 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDownloadExecutionRequest +{ + public SourceFamily SourceFamily { get; init; } + + public string SourceKey { get; init; } + + public string CandidateId { get; init; } + + public string CandidateTitle { get; init; } + + public string SourceReferenceUri { get; init; } + + public string ArtifactKind { get; init; } + + public string TargetRoot { get; init; } + + public string TargetRelativePath { get; init; } + + public bool CandidateDownloadAllowed { get; init; } + + public bool AllowDownloadExecution { get; init; } + + public bool AllowFileWrite { get; init; } + + public bool AllowNetwork { get; init; } + + public bool AllowOverwrite { get; init; } + + public bool AllowParse { get; init; } + + public bool AllowDatabaseWrites { get; init; } + + public bool AllowScheduler { get; init; } + + public string? ContentType { get; init; } + + public string? Extension { get; init; } + + public string? ExpectedChecksumSha256 { get; init; } + + public int? DocumentYear { get; init; } + + public int? ReportingYear { get; init; } + + public string? VersionLabel { get; init; } + + public IpccSourceDownloadExecutionRequest( + SourceFamily sourceFamily, + string sourceKey, + string candidateId, + string candidateTitle, + string sourceReferenceUri, + string artifactKind, + string targetRoot, + string targetRelativePath, + bool candidateDownloadAllowed = false, + bool allowDownloadExecution = false, + bool allowFileWrite = false, + bool allowNetwork = false, + bool allowOverwrite = false, + bool allowParse = false, + bool allowDatabaseWrites = false, + bool allowScheduler = false, + string? contentType = null, + string? extension = null, + string? expectedChecksumSha256 = null, + int? documentYear = null, + int? reportingYear = null, + string? versionLabel = null) + { + SourceFamily = sourceFamily; + SourceKey = sourceKey; + CandidateId = candidateId; + CandidateTitle = candidateTitle; + SourceReferenceUri = sourceReferenceUri; + ArtifactKind = artifactKind; + TargetRoot = targetRoot; + TargetRelativePath = targetRelativePath; + CandidateDownloadAllowed = candidateDownloadAllowed; + AllowDownloadExecution = allowDownloadExecution; + AllowFileWrite = allowFileWrite; + AllowNetwork = allowNetwork; + AllowOverwrite = allowOverwrite; + AllowParse = allowParse; + AllowDatabaseWrites = allowDatabaseWrites; + AllowScheduler = allowScheduler; + ContentType = contentType; + Extension = extension; + ExpectedChecksumSha256 = expectedChecksumSha256; + DocumentYear = documentYear; + ReportingYear = reportingYear; + VersionLabel = versionLabel; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionResult.cs new file mode 100644 index 00000000..24bde50d --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionResult.cs @@ -0,0 +1,44 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDownloadExecutionResult +{ + public IpccSourceDownloadExecutionStatus Status { get; init; } + + public IpccSourceDownloadExecutionRequest Request { get; init; } + + public IpccSourceDownloadedArtifact? Artifact { get; init; } + + public IReadOnlyList Issues { get; init; } + + public bool NoParse { get; init; } + + public bool NoDatabaseWrites { get; init; } + + public bool NoSql { get; init; } + + public bool NoScheduler { get; init; } + + public bool Downloaded => Status == IpccSourceDownloadExecutionStatus.Downloaded; + + public bool AlreadyKnown => Status == IpccSourceDownloadExecutionStatus.AlreadyKnown; + + public IpccSourceDownloadExecutionResult( + IpccSourceDownloadExecutionStatus status, + IpccSourceDownloadExecutionRequest request, + IpccSourceDownloadedArtifact? artifact = null, + IEnumerable? issues = null, + bool noParse = true, + bool noDatabaseWrites = true, + bool noSql = true, + bool noScheduler = true) + { + Status = status; + Request = request; + Artifact = artifact; + Issues = (issues ?? Array.Empty()).ToArray(); + NoParse = noParse; + NoDatabaseWrites = noDatabaseWrites; + NoSql = noSql; + NoScheduler = noScheduler; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionStatus.cs new file mode 100644 index 00000000..dc6f4a94 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionStatus.cs @@ -0,0 +1,9 @@ +namespace CarbonOps.Parser.Contracts; + +public enum IpccSourceDownloadExecutionStatus +{ + Blocked = 0, + Downloaded = 1, + Failed = 2, + AlreadyKnown = 3, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionValidationResult.cs new file mode 100644 index 00000000..54b0b5dd --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadExecutionValidationResult.cs @@ -0,0 +1,14 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDownloadExecutionValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public IpccSourceDownloadExecutionValidationResult( + IEnumerable? issues = null) + { + Issues = (issues ?? Array.Empty()).ToArray(); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadTransportResponse.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadTransportResponse.cs new file mode 100644 index 00000000..048e6906 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadTransportResponse.cs @@ -0,0 +1,6 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDownloadTransportResponse( + byte[] Content, + string? ContentType = null, + string? FinalUri = null); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadedArtifact.cs b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadedArtifact.cs new file mode 100644 index 00000000..62110f2d --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/IpccSourceDownloadedArtifact.cs @@ -0,0 +1,20 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record IpccSourceDownloadedArtifact( + SourceFamily SourceFamily, + string SourceKey, + string CandidateId, + string ArtifactId, + string ArtifactKind, + string SourceReferenceUri, + string LocalPath, + string OriginalFilename, + string ChecksumSha256, + long SizeBytes, + DateTimeOffset RetrievedAtUtc, + string? ContentType = null, + string? Extension = null, + string? FinalUri = null, + int? DocumentYear = null, + int? ReportingYear = null, + string? VersionLabel = null); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceCommand.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceCommand.cs new file mode 100644 index 00000000..8ce0a550 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceCommand.cs @@ -0,0 +1,24 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record ParsedFactorPersistenceCommand +{ + public IReadOnlyList MasterRecords { get; } + + public IReadOnlyList DetailRecords { get; } + + public int SkippedDuplicateCount { get; } + + public IReadOnlyList Issues { get; } + + public ParsedFactorPersistenceCommand( + IEnumerable masterRecords, + IEnumerable detailRecords, + int skippedDuplicateCount = 0, + IEnumerable? issues = null) + { + MasterRecords = Array.AsReadOnly(masterRecords.ToArray()); + DetailRecords = Array.AsReadOnly(detailRecords.ToArray()); + SkippedDuplicateCount = skippedDuplicateCount; + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceIssue.cs new file mode 100644 index 00000000..b639f384 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record ParsedFactorPersistenceIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceStatus.cs new file mode 100644 index 00000000..d3a029ae --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceStatus.cs @@ -0,0 +1,8 @@ +namespace CarbonOps.Parser.Contracts; + +public enum ParsedFactorPersistenceStatus +{ + Declared = 0, + FailedValidation = 1, + NoRecords = 2, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceWriter.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceWriter.cs new file mode 100644 index 00000000..57b41449 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceWriter.cs @@ -0,0 +1,377 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace CarbonOps.Parser.Contracts; + +public static class ParsedFactorPersistenceWriter +{ + public const string DefaultTimestampLabel = "dry_run_timestamp_unavailable"; + + public static ParsedFactorPersistenceCommand BuildCommand( + ParserNormalizedOutputBatch parsedOutput, + string? sourceDocumentId = null, + string lifecycleStatus = "active", + string timestampLabel = DefaultTimestampLabel) + { + if (parsedOutput is null) + { + return new ParsedFactorPersistenceCommand( + [], + [], + issues: + [ + new ParsedFactorPersistenceIssue( + "PARSED_FACTOR_PERSISTENCE_INVALID_OUTPUT", + "parsed output must be a ParserNormalizedOutputBatch.", + "parsedOutput"), + ]); + } + + if (parsedOutput.RowCount == 0) + { + return new ParsedFactorPersistenceCommand( + [], + [], + issues: + [ + new ParsedFactorPersistenceIssue( + "PARSED_FACTOR_PERSISTENCE_NO_RECORDS", + "parsed output must include records before persistence.", + "Rows", + "warning"), + ]); + } + + var issues = new List(); + var masters = new Dictionary<(SourceFamily SourceFamily, string MasterId), SourceFamilyMasterRecord>(); + var details = new Dictionary<(SourceFamily SourceFamily, string MasterId, string DetailExternalKey), SourceFamilyDetailRecord>(); + var skippedDuplicateCount = 0; + + for (var index = 0; index < parsedOutput.Rows.Count; index++) + { + var row = parsedOutput.Rows[index]; + if (row is null) + { + issues.Add(new ParsedFactorPersistenceIssue( + "PARSED_FACTOR_PERSISTENCE_INVALID_NORMALIZED_ROW", + "ParserNormalizedOutputRow is required.", + $"Rows[{index}]")); + continue; + } + + AppendRowValidationIssues(row, index, issues); + + var mapped = MapRow( + row, + index, + sourceDocumentId, + lifecycleStatus, + timestampLabel); + issues.AddRange(mapped.Issues); + + if (mapped.MasterRecord is null || mapped.DetailRecord is null) + { + continue; + } + + var masterKey = (mapped.MasterRecord.SourceFamily, mapped.MasterRecord.SourceFamilyMasterId); + if (!masters.TryGetValue(masterKey, out var existingMaster)) + { + masters.Add(masterKey, mapped.MasterRecord); + } + else if (existingMaster == mapped.MasterRecord) + { + skippedDuplicateCount++; + } + else + { + issues.Add(new ParsedFactorPersistenceIssue( + "PARSED_FACTOR_PERSISTENCE_DUPLICATE_MASTER_CONFLICT", + "duplicate source-family master identity maps to different record content.", + $"Rows[{index}].source_family_master_id")); + } + + var detailKey = ( + mapped.DetailRecord.SourceFamily, + mapped.DetailRecord.SourceFamilyMasterId, + mapped.DetailRecord.DetailExternalKey); + if (!details.TryGetValue(detailKey, out var existingDetail)) + { + details.Add(detailKey, mapped.DetailRecord); + } + else if (existingDetail == mapped.DetailRecord) + { + skippedDuplicateCount++; + } + else + { + issues.Add(new ParsedFactorPersistenceIssue( + "PARSED_FACTOR_PERSISTENCE_DUPLICATE_DETAIL_CONFLICT", + "duplicate factor identity maps to different detail record content.", + $"Rows[{index}].detail_external_key")); + } + } + + var command = new ParsedFactorPersistenceCommand( + masters.Values, + details.Values, + skippedDuplicateCount, + issues); + var repositoryValidation = SourceFamilyRepositoryRegistry.ValidateInputs( + "parsed_factor_persistence_command", + command.MasterRecords, + command.DetailRecords); + + if (repositoryValidation.Issues.Count == 0) + { + return command; + } + + return new ParsedFactorPersistenceCommand( + command.MasterRecords, + command.DetailRecords, + command.SkippedDuplicateCount, + command.Issues.Concat(repositoryValidation.Issues.Select(FromRepositoryIssue))); + } + + public static ParsedFactorPersistenceWriterResult Persist( + ParserNormalizedOutputBatch parsedOutput, + ISourceFamilyRepository repository, + string? sourceDocumentId = null, + string lifecycleStatus = "active", + string timestampLabel = DefaultTimestampLabel) + { + var command = BuildCommand(parsedOutput, sourceDocumentId, lifecycleStatus, timestampLabel); + if (command.Issues.Count > 0) + { + var status = IsOnlyNoRecords(command.Issues) + ? ParsedFactorPersistenceStatus.NoRecords + : ParsedFactorPersistenceStatus.FailedValidation; + + return new ParsedFactorPersistenceWriterResult( + repository.ProviderName, + status, + command.MasterRecords.Count, + command.DetailRecords.Count, + 0, + 0, + command.SkippedDuplicateCount, + command.Issues, + command); + } + + var repositoryResult = repository.PersistSourceFamilyRecords( + command.MasterRecords, + command.DetailRecords); + var repositoryIssues = repositoryResult.Issues.Select(FromRepositoryIssue).ToArray(); + var resultStatus = repositoryResult.Status == SourceFamilyRepositoryPersistStatus.Declared + ? ParsedFactorPersistenceStatus.Declared + : ParsedFactorPersistenceStatus.FailedValidation; + + return new ParsedFactorPersistenceWriterResult( + repositoryResult.ProviderName, + resultStatus, + command.MasterRecords.Count, + command.DetailRecords.Count, + repositoryResult.PersistedMasterCount, + repositoryResult.PersistedDetailCount, + command.SkippedDuplicateCount, + repositoryIssues, + command); + } + + private static MappedRow MapRow( + ParserNormalizedOutputRow row, + int index, + string? explicitSourceDocumentId, + string lifecycleStatus, + string timestampLabel) + { + if (!Enum.IsDefined(row.SourceFamily)) + { + return new MappedRow( + null, + null, + [ + new ParsedFactorPersistenceIssue( + "PARSED_FACTOR_PERSISTENCE_UNSUPPORTED_SOURCE_FAMILY", + "source family must be GHG Protocol, DEFRA/DESNZ, or IPCC EFDB.", + $"Rows[{index}].SourceFamily"), + ]); + } + + var fields = row.Fields + .GroupBy(field => field.Key, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.First().Value, StringComparer.Ordinal); + var issues = new List(); + var resolvedSourceDocumentId = ResolveSourceDocumentId(row, fields, explicitSourceDocumentId); + var factorValue = TextOrNull(Field(fields, "factor_value", "value")); + var factorUnit = TextOrNull(Field(fields, "factor_unit", "unit")); + + var requiredValues = new Dictionary + { + ["source_document_id"] = resolvedSourceDocumentId, + ["factor_value"] = factorValue, + ["factor_unit"] = factorUnit, + }; + foreach (var pair in requiredValues) + { + if (pair.Value is null) + { + issues.Add(new ParsedFactorPersistenceIssue( + "PARSED_FACTOR_PERSISTENCE_MISSING_REQUIRED_FIELD", + "parsed factor persistence requires a non-empty value.", + $"Rows[{index}].{pair.Key}")); + } + } + + if (issues.Count > 0) + { + return new MappedRow(null, null, issues); + } + + var masterExternalKey = TextOrNull(Field(fields, "master_external_key")) ?? DefaultMasterExternalKey(row, fields); + var detailExternalKey = TextOrNull(Field(fields, "detail_external_key")) ?? DefaultDetailExternalKey(row, fields); + var familyPrefix = row.SourceFamily switch + { + SourceFamily.GhgProtocol => "ghg", + SourceFamily.DefraDesnz => "defra", + SourceFamily.IpccEfdb => "ipcc", + _ => row.SourceFamily.ToWireName(), + }; + var masterId = TextOrNull(Field(fields, "source_family_master_id")) + ?? $"{familyPrefix}_master_{StableDigest(familyPrefix, masterExternalKey)[..16]}"; + var detailId = TextOrNull(Field(fields, "source_family_detail_id")) + ?? $"{familyPrefix}_detail_{StableDigest(familyPrefix, masterId, detailExternalKey)[..16]}"; + + var masterRecord = new SourceFamilyMasterRecord( + row.SourceFamily, + masterId, + resolvedSourceDocumentId!, + masterExternalKey, + lifecycleStatus, + TextOrNull(Field(fields, "effective_from")), + TextOrNull(Field(fields, "effective_to")), + StableDigest("master", familyPrefix, resolvedSourceDocumentId!, masterExternalKey, lifecycleStatus), + timestampLabel, + timestampLabel); + var detailRecord = new SourceFamilyDetailRecord( + row.SourceFamily, + detailId, + masterId, + detailExternalKey, + factorValue!, + factorUnit!, + lifecycleStatus, + StableDigest("detail", familyPrefix, masterId, detailExternalKey, factorValue!, factorUnit!), + timestampLabel, + timestampLabel); + + return new MappedRow(masterRecord, detailRecord, issues); + } + + private static void AppendRowValidationIssues( + ParserNormalizedOutputRow row, + int index, + ICollection issues) + { + foreach (var error in row.Validate().Errors) + { + issues.Add(new ParsedFactorPersistenceIssue( + "PARSED_FACTOR_PERSISTENCE_INVALID_NORMALIZED_ROW", + error, + $"Rows[{index}]")); + } + } + + private static string? ResolveSourceDocumentId( + ParserNormalizedOutputRow row, + IReadOnlyDictionary fields, + string? explicitSourceDocumentId) + { + var explicitValue = TextOrNull(explicitSourceDocumentId); + if (explicitValue is not null) + { + return explicitValue; + } + + var fieldValue = TextOrNull(Field(fields, "source_document_id")); + if (fieldValue is not null) + { + return fieldValue; + } + + var artifactReference = TextOrNull(Field(fields, "provenance_artifact_reference", "artifact_reference")) + ?? TextOrNull(row.ArtifactReference); + var checksum = TextOrNull(Field(fields, "provenance_checksum_value", "source_checksum_sha256")); + if (artifactReference is null && checksum is null) + { + return null; + } + + return $"source_document_{StableDigest(row.SourceFamily.ToWireName(), row.SourceKey, artifactReference, checksum)[..24]}"; + } + + private static string DefaultMasterExternalKey( + ParserNormalizedOutputRow row, + IReadOnlyDictionary fields) + { + var sourceYear = TextOrNull(Field(fields, "source_year")) ?? "unknown-year"; + var sourceVersion = TextOrNull(Field(fields, "source_version")) ?? "unknown-version"; + var factorId = TextOrNull(Field(fields, "factor_id")) ?? row.RowIdentifier; + + return $"{sourceYear}:{sourceVersion}:{factorId}"; + } + + private static string DefaultDetailExternalKey( + ParserNormalizedOutputRow row, + IReadOnlyDictionary fields) + { + var factorId = TextOrNull(Field(fields, "factor_id")) ?? row.RowIdentifier; + var factorUnit = TextOrNull(Field(fields, "factor_unit", "unit")) ?? "unknown-unit"; + var gas = TextOrNull(Field(fields, "greenhouse_gas", "gas")); + + return gas is null + ? $"{factorId}:{factorUnit}" + : $"{factorId}:{factorUnit}:{gas}"; + } + + private static string? Field(IReadOnlyDictionary fields, params string[] names) + { + foreach (var name in names) + { + if (fields.TryGetValue(name, out var value)) + { + return value; + } + } + + return null; + } + + private static string? TextOrNull(string? value) + { + var text = value?.Trim(); + + return string.IsNullOrEmpty(text) ? null : text; + } + + private static string StableDigest(params string?[] values) + { + var payload = JsonSerializer.Serialize(values); + + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(payload))).ToLowerInvariant(); + } + + private static ParsedFactorPersistenceIssue FromRepositoryIssue(SourceFamilyRepositoryIssue issue) => + new(issue.Code, issue.Message, issue.FieldName, issue.Severity); + + private static bool IsOnlyNoRecords(IReadOnlyList issues) => + issues.Count == 1 && issues[0].Code == "PARSED_FACTOR_PERSISTENCE_NO_RECORDS"; + + private sealed record MappedRow( + SourceFamilyMasterRecord? MasterRecord, + SourceFamilyDetailRecord? DetailRecord, + IEnumerable Issues); +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceWriterResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceWriterResult.cs new file mode 100644 index 00000000..a48fbcea --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParsedFactorPersistenceWriterResult.cs @@ -0,0 +1,44 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record ParsedFactorPersistenceWriterResult +{ + public string ProviderName { get; } + + public ParsedFactorPersistenceStatus Status { get; } + + public int AttemptedMasterCount { get; } + + public int AttemptedDetailCount { get; } + + public int PersistedMasterCount { get; } + + public int PersistedDetailCount { get; } + + public int SkippedDuplicateCount { get; } + + public IReadOnlyList Issues { get; } + + public ParsedFactorPersistenceCommand? Command { get; } + + public ParsedFactorPersistenceWriterResult( + string providerName, + ParsedFactorPersistenceStatus status, + int attemptedMasterCount, + int attemptedDetailCount, + int persistedMasterCount, + int persistedDetailCount, + int skippedDuplicateCount = 0, + IEnumerable? issues = null, + ParsedFactorPersistenceCommand? command = null) + { + ProviderName = providerName; + Status = status; + AttemptedMasterCount = attemptedMasterCount; + AttemptedDetailCount = attemptedDetailCount; + PersistedMasterCount = persistedMasterCount; + PersistedDetailCount = persistedDetailCount; + SkippedDuplicateCount = skippedDuplicateCount; + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + Command = command; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepository.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepository.cs new file mode 100644 index 00000000..09792e5f --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepository.cs @@ -0,0 +1,8 @@ +namespace CarbonOps.Parser.Contracts; + +public interface IParserRunRepository +{ + string ProviderName { get; } + + ParserRunRepositoryPersistResult PersistRuns(IEnumerable runs); +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryIssue.cs new file mode 100644 index 00000000..6ec70719 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record ParserRunRepositoryIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryPersistResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryPersistResult.cs new file mode 100644 index 00000000..3394b3e8 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryPersistResult.cs @@ -0,0 +1,24 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record ParserRunRepositoryPersistResult +{ + public string ProviderName { get; } + + public ParserRunRepositoryPersistStatus Status { get; } + + public int PersistedCount { get; } + + public IReadOnlyList Issues { get; } + + public ParserRunRepositoryPersistResult( + string providerName, + ParserRunRepositoryPersistStatus status, + int persistedCount, + IEnumerable? issues = null) + { + ProviderName = providerName; + Status = status; + PersistedCount = persistedCount; + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryPersistStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryPersistStatus.cs new file mode 100644 index 00000000..9f8fb56c --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryPersistStatus.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public enum ParserRunRepositoryPersistStatus +{ + Declared = 0, + FailedValidation = 1, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryRegistry.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryRegistry.cs new file mode 100644 index 00000000..e5711e66 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryRegistry.cs @@ -0,0 +1,60 @@ +namespace CarbonOps.Parser.Contracts; + +public static class ParserRunRepositoryRegistry +{ + public static ParserRunRepositoryPersistResult CreatePersistResult( + string providerName, + IEnumerable runs, + IEnumerable? issues = null) + { + var runSnapshot = runs.ToArray(); + var collectedIssues = ValidateInputs(providerName, runSnapshot).Issues + .Select(issue => new ParserRunRepositoryIssue( + issue.Code, + issue.Message, + issue.FieldName, + issue.Severity)) + .ToList(); + collectedIssues.AddRange(issues ?? []); + + var status = collectedIssues.Count == 0 + ? ParserRunRepositoryPersistStatus.Declared + : ParserRunRepositoryPersistStatus.FailedValidation; + + return new ParserRunRepositoryPersistResult( + providerName, + status, + status == ParserRunRepositoryPersistStatus.Declared ? runSnapshot.Length : 0, + collectedIssues); + } + + public static ParserRunRepositoryValidationResult ValidateInputs( + string providerName, + IEnumerable runs) + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(providerName)) + { + errors.Add(new ParserRunRepositoryIssue( + "PARSER_RUN_REPOSITORY_MISSING_PROVIDER_NAME", + "ProviderName must be a non-empty string.", + "ProviderName")); + } + + var runSnapshot = runs.ToArray(); + for (var index = 0; index < runSnapshot.Length; index++) + { + var run = runSnapshot[index]; + if (run is null) + { + errors.Add(new ParserRunRepositoryIssue( + "PARSER_RUN_REPOSITORY_INVALID_RUN", + "Runs must contain ParserRunResult instances.", + $"Runs[{index}]")); + } + } + + return new ParserRunRepositoryValidationResult(errors); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryValidationResult.cs new file mode 100644 index 00000000..a133dccc --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ParserRunRepositoryValidationResult.cs @@ -0,0 +1,14 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record ParserRunRepositoryValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public ParserRunRepositoryValidationResult( + IEnumerable? issues = null) + { + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/Phase1IngestionOrchestrator.cs b/src/dotnet/CarbonOps.Parser.Contracts/Phase1IngestionOrchestrator.cs new file mode 100644 index 00000000..89184367 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/Phase1IngestionOrchestrator.cs @@ -0,0 +1,599 @@ +namespace CarbonOps.Parser.Contracts; + +public enum Phase1IngestionExecutionMode +{ + Sequential = 0, + BoundedParallel = 1, +} + +public enum Phase1IngestionRunStatus +{ + Completed = 0, + CompletedWithFailures = 1, + Failed = 2, + NotExecutable = 3, +} + +public enum Phase1IngestionFamilyRunStatus +{ + Completed = 0, + Failed = 1, + Skipped = 2, +} + +public sealed record Phase1IngestionFailure( + SourceFamily SourceFamily, + string SourceKey, + string Stage, + string Code, + string Message, + string? FieldName = null, + string Severity = "error"); + +public sealed record Phase1IngestionOrchestratorRequest +{ + public IReadOnlyList SourceFamilies { get; } + + public Phase1IngestionExecutionMode ExecutionMode { get; } + + public int MaxDegreeOfParallelism { get; } + + public PostgreSQLRuntimeConfigGate RuntimeConfigGate { get; } + + public PostgreSQLSchemaBootstrapReport? SchemaBootstrapReport { get; } + + public string RunId { get; } + + public string? CorrelationId { get; } + + public Phase1OperationalEventSink? OperationalEventSink { get; } + + public Phase1IngestionOrchestratorRequest( + IEnumerable sourceFamilies, + Phase1IngestionExecutionMode executionMode = Phase1IngestionExecutionMode.Sequential, + int maxDegreeOfParallelism = 1, + PostgreSQLRuntimeConfigGate? runtimeConfigGate = null, + string runId = "phase1_ingestion_orchestrator_run", + string? correlationId = null, + PostgreSQLSchemaBootstrapReport? schemaBootstrapReport = null, + Phase1OperationalEventSink? operationalEventSink = null) + { + SourceFamilies = Array.AsReadOnly(sourceFamilies.ToArray()); + ExecutionMode = executionMode; + MaxDegreeOfParallelism = maxDegreeOfParallelism; + RuntimeConfigGate = runtimeConfigGate ?? new PostgreSQLRuntimeConfigGate(); + SchemaBootstrapReport = schemaBootstrapReport; + RunId = runId; + CorrelationId = correlationId; + OperationalEventSink = operationalEventSink; + } +} + +public sealed record Phase1IngestionFamilyResult +{ + public SourceFamily SourceFamily { get; } + + public string SourceKey { get; } + + public Phase1IngestionFamilyRunStatus Status { get; } + + public SourceAcquisitionRunResult? AcquisitionRun { get; } + + public SourceAcquisitionRunRepositoryPersistResult? AcquisitionRunPersistResult { get; } + + public SourceDocumentRepositoryPersistResult? SourceDocumentPersistResult { get; } + + public ParserAdapterRunResult? ParserRun { get; } + + public ParserRunRepositoryPersistResult? ParserRunPersistResult { get; } + + public ParsedFactorPersistenceWriterResult? ParsedFactorPersistResult { get; } + + public IReadOnlyList Failures { get; } + + public int SourceCandidateCount => AcquisitionRun?.CandidateCount ?? 0; + + public int SourceDocumentMetadataCount => AcquisitionRun?.ArtifactCount ?? 0; + + public int ParserAcceptedRowCount => ParserRun?.RowCount ?? 0; + + public int ParserFailureCount => ParserRun?.ValidationIssues.Count(issue => issue.Severity == ParserValidationIssueSeverity.Error) ?? 0; + + public int PersistedMasterCount => ParsedFactorPersistResult?.PersistedMasterCount ?? 0; + + public int PersistedDetailCount => ParsedFactorPersistResult?.PersistedDetailCount ?? 0; + + public int FailureCount => Failures.Count; + + public Phase1IngestionFamilyResult( + SourceFamily sourceFamily, + string sourceKey, + Phase1IngestionFamilyRunStatus status, + SourceAcquisitionRunResult? acquisitionRun = null, + SourceAcquisitionRunRepositoryPersistResult? acquisitionRunPersistResult = null, + SourceDocumentRepositoryPersistResult? sourceDocumentPersistResult = null, + ParserAdapterRunResult? parserRun = null, + ParserRunRepositoryPersistResult? parserRunPersistResult = null, + ParsedFactorPersistenceWriterResult? parsedFactorPersistResult = null, + IEnumerable? failures = null) + { + SourceFamily = sourceFamily; + SourceKey = sourceKey; + Status = status; + AcquisitionRun = acquisitionRun; + AcquisitionRunPersistResult = acquisitionRunPersistResult; + SourceDocumentPersistResult = sourceDocumentPersistResult; + ParserRun = parserRun; + ParserRunPersistResult = parserRunPersistResult; + ParsedFactorPersistResult = parsedFactorPersistResult; + Failures = Array.AsReadOnly((failures ?? []).ToArray()); + } +} + +public sealed record Phase1IngestionOrchestratorResult +{ + public Phase1IngestionOrchestratorRequest Request { get; } + + public PostgreSQLRuntimeConfigGateDecision RuntimeConfigDecision { get; } + + public Phase1IngestionRunStatus Status { get; } + + public IReadOnlyList SelectedSourceFamilies { get; } + + public IReadOnlyList FamilyResults { get; } + + public IReadOnlyList Failures { get; } + + public int SourceFamilyCount => FamilyResults.Count; + + public int CompletedSourceFamilyCount => FamilyResults.Count(result => result.Status == Phase1IngestionFamilyRunStatus.Completed); + + public int FailedSourceFamilyCount => FamilyResults.Count(result => result.Status == Phase1IngestionFamilyRunStatus.Failed); + + public int TotalSourceDocumentMetadataCount => FamilyResults.Sum(result => result.SourceDocumentMetadataCount); + + public int TotalParserAcceptedRowCount => FamilyResults.Sum(result => result.ParserAcceptedRowCount); + + public int TotalParserFailureCount => FamilyResults.Sum(result => result.ParserFailureCount); + + public int TotalPersistedMasterCount => FamilyResults.Sum(result => result.PersistedMasterCount); + + public int TotalPersistedDetailCount => FamilyResults.Sum(result => result.PersistedDetailCount); + + public int FailureCount => Failures.Count; + + public Phase1IngestionOrchestratorResult( + Phase1IngestionOrchestratorRequest request, + PostgreSQLRuntimeConfigGateDecision runtimeConfigDecision, + IEnumerable familyResults, + IEnumerable? failures = null, + Phase1IngestionRunStatus? status = null, + IEnumerable? selectedSourceFamilies = null) + { + Request = request; + RuntimeConfigDecision = runtimeConfigDecision; + var familyResultArray = familyResults.ToArray(); + var failureArray = (failures ?? []).ToArray(); + FamilyResults = Array.AsReadOnly(familyResultArray); + Failures = Array.AsReadOnly(failureArray); + Status = status ?? StatusFromFamilyResults(familyResultArray); + SelectedSourceFamilies = Array.AsReadOnly( + (selectedSourceFamilies ?? familyResultArray.Select(result => result.SourceFamily)) + .ToArray()); + } + + private static Phase1IngestionRunStatus StatusFromFamilyResults( + IReadOnlyCollection familyResults) + { + if (familyResults.Count == 0) + { + return Phase1IngestionRunStatus.Failed; + } + + var completedCount = familyResults.Count(result => + result.Status == Phase1IngestionFamilyRunStatus.Completed); + + return completedCount == familyResults.Count + ? Phase1IngestionRunStatus.Completed + : completedCount > 0 + ? Phase1IngestionRunStatus.CompletedWithFailures + : Phase1IngestionRunStatus.Failed; + } +} + +public interface IPhase1SourceFamilyIngestionRuntime +{ + SourceFamily SourceFamily { get; } + + SourceAcquisitionRunResult DiscoverAndDownload(string runId, string? correlationId); + + ParserAdapterRunResult Normalize(ParserAdapterRunRequest request); +} + +public sealed record Phase1IngestionOrchestratorDependencies( + IEnumerable SourceRuntimes, + ISourceAcquisitionRunRepository SourceAcquisitionRunRepository, + ISourceDocumentRepository SourceDocumentRepository, + IParserRunRepository ParserRunRepository, + ISourceFamilyRepository SourceFamilyRepository); + +public sealed class Phase1IngestionOrchestrator +{ + private readonly IReadOnlyDictionary sourceRuntimes; + private readonly ISourceAcquisitionRunRepository sourceAcquisitionRunRepository; + private readonly ISourceDocumentRepository sourceDocumentRepository; + private readonly IParserRunRepository parserRunRepository; + private readonly ISourceFamilyRepository sourceFamilyRepository; + + public Phase1IngestionOrchestrator(Phase1IngestionOrchestratorDependencies dependencies) + { + sourceRuntimes = dependencies.SourceRuntimes.ToDictionary(runtime => runtime.SourceFamily); + sourceAcquisitionRunRepository = dependencies.SourceAcquisitionRunRepository; + sourceDocumentRepository = dependencies.SourceDocumentRepository; + parserRunRepository = dependencies.ParserRunRepository; + sourceFamilyRepository = dependencies.SourceFamilyRepository; + } + + public Phase1IngestionOrchestratorResult Run(Phase1IngestionOrchestratorRequest request) + { + Phase1OperationalDiagnostics.Emit( + request.OperationalEventSink, + "phase1_ingestion_orchestrator_started", + Phase1OperationalDiagnostics.SummarizeOrchestratorRequest(request)); + + var runtimeDecision = PostgreSQLRuntimeConfigGateEvaluator.Evaluate(request.RuntimeConfigGate); + var selectedSourceFamilies = request.SourceFamilies + .Where(sourceFamily => Enum.IsDefined(sourceFamily)) + .Distinct() + .ToArray(); + var readinessFailures = ValidateRequest(request) + .Concat(ValidatePostgreSQLReadiness(request, runtimeDecision)) + .ToArray(); + if (readinessFailures.Length > 0) + { + var blockedResult = new Phase1IngestionOrchestratorResult( + request, + runtimeDecision, + [], + readinessFailures, + Phase1IngestionRunStatus.NotExecutable, + selectedSourceFamilies); + Phase1OperationalDiagnostics.Emit( + request.OperationalEventSink, + "phase1_ingestion_orchestrator_completed", + Phase1OperationalDiagnostics.SummarizeOrchestratorResultForDiagnostics(blockedResult)); + return blockedResult; + } + + var familyResults = new List(); + foreach (var sourceFamily in selectedSourceFamilies) + { + var familyResult = RunSourceFamily(sourceFamily, request); + familyResults.Add(familyResult); + Phase1OperationalDiagnostics.Emit( + request.OperationalEventSink, + "phase1_source_family_completed", + Phase1OperationalDiagnostics.SummarizeFamilyResultForDiagnostics( + familyResult, + request.RunId, + request.CorrelationId)); + } + + var result = new Phase1IngestionOrchestratorResult( + request, + runtimeDecision, + familyResults, + familyResults.SelectMany(result => result.Failures), + selectedSourceFamilies: selectedSourceFamilies); + Phase1OperationalDiagnostics.Emit( + request.OperationalEventSink, + "phase1_ingestion_orchestrator_completed", + Phase1OperationalDiagnostics.SummarizeOrchestratorResultForDiagnostics(result)); + return result; + } + + private Phase1IngestionFamilyResult RunSourceFamily( + SourceFamily sourceFamily, + Phase1IngestionOrchestratorRequest request) + { + var sourceKey = sourceFamily.ToWireName(); + if (!sourceRuntimes.TryGetValue(sourceFamily, out var runtime)) + { + return Failed(sourceFamily, sourceKey, Failure(sourceFamily, sourceKey, "runtime", "PHASE1_RUNTIME_MISSING", "No ingestion runtime is registered for the selected source family.")); + } + + SourceAcquisitionRunResult? acquisitionRun = null; + SourceAcquisitionRunRepositoryPersistResult? acquisitionPersist = null; + SourceDocumentRepositoryPersistResult? documentPersist = null; + ParserAdapterRunResult? parserRun = null; + ParserRunRepositoryPersistResult? parserPersist = null; + ParsedFactorPersistenceWriterResult? factorPersist = null; + var failures = new List(); + + try + { + acquisitionRun = runtime.DiscoverAndDownload(request.RunId, request.CorrelationId); + failures.AddRange(ValidateAcquisition(sourceFamily, sourceKey, acquisitionRun)); + acquisitionPersist = sourceAcquisitionRunRepository.PersistRuns([acquisitionRun]); + failures.AddRange(FromAcquisitionRepositoryIssues(sourceFamily, sourceKey, acquisitionPersist.Issues)); + documentPersist = sourceDocumentRepository.PersistSourceDocuments(CreateSourceDocumentRecords(acquisitionRun.Artifacts)); + failures.AddRange(FromSourceDocumentRepositoryIssues(sourceFamily, sourceKey, documentPersist.Issues)); + + if (failures.Any(failure => failure.Stage is "discovery" or "download" or "source_document_persistence" or "source_acquisition_run_persistence")) + { + return BuildResult(sourceFamily, sourceKey, failures, acquisitionRun, acquisitionPersist, documentPersist); + } + + var bridgeBatch = SourceArtifactParserInputBridgeRegistry.CreateBridgeBatch(acquisitionRun.Artifacts); + var parserKey = ParserSelectionRegistry.GetParserKey(sourceFamily); + var parserRequest = new ParserAdapterRunRequest( + sourceFamily, + sourceKey, + parserKey, + bridgeBatch.Bridges.Select(bridge => bridge.ParserInputArtifact), + runId: $"{request.RunId}-{sourceKey}-parser", + correlationId: request.CorrelationId, + requestedReportingYear: acquisitionRun.ReportingYear); + parserRun = runtime.Normalize(parserRequest); + failures.AddRange(FromParserIssues(sourceFamily, sourceKey, parserRun.ValidationIssues)); + parserPersist = parserRunRepository.PersistRuns([CreateParserRunResult(parserRun)]); + failures.AddRange(FromParserRunRepositoryIssues(sourceFamily, sourceKey, parserPersist.Issues)); + + if (parserRun.Status != ParserRunStatus.Completed || + failures.Any(failure => failure.Stage is "parser" or "parser_run_persistence")) + { + return BuildResult(sourceFamily, sourceKey, failures, acquisitionRun, acquisitionPersist, documentPersist, parserRun, parserPersist); + } + + var sourceDocumentId = acquisitionRun.Artifacts.Count == 1 + ? acquisitionRun.Artifacts[0].ArtifactId + : null; + factorPersist = ParsedFactorPersistenceWriter.Persist( + new ParserNormalizedOutputBatch(parserRun.Rows), + sourceFamilyRepository, + sourceDocumentId); + failures.AddRange(FromParsedFactorPersistenceIssues(sourceFamily, sourceKey, factorPersist.Issues)); + + return BuildResult( + sourceFamily, + sourceKey, + failures, + acquisitionRun, + acquisitionPersist, + documentPersist, + parserRun, + parserPersist, + factorPersist); + } + catch (Exception exception) when (exception is InvalidOperationException or ArgumentException) + { + failures.Add(Failure(sourceFamily, sourceKey, "orchestrator", "PHASE1_ORCHESTRATOR_EXCEPTION", exception.Message)); + return BuildResult( + sourceFamily, + sourceKey, + failures, + acquisitionRun, + acquisitionPersist, + documentPersist, + parserRun, + parserPersist, + factorPersist); + } + } + + private static Phase1IngestionFamilyResult BuildResult( + SourceFamily sourceFamily, + string sourceKey, + IReadOnlyCollection failures, + SourceAcquisitionRunResult? acquisitionRun = null, + SourceAcquisitionRunRepositoryPersistResult? acquisitionRunPersistResult = null, + SourceDocumentRepositoryPersistResult? sourceDocumentPersistResult = null, + ParserAdapterRunResult? parserRun = null, + ParserRunRepositoryPersistResult? parserRunPersistResult = null, + ParsedFactorPersistenceWriterResult? parsedFactorPersistResult = null) => + new( + sourceFamily, + sourceKey, + failures.Any(failure => failure.Severity == "error") + ? Phase1IngestionFamilyRunStatus.Failed + : Phase1IngestionFamilyRunStatus.Completed, + acquisitionRun, + acquisitionRunPersistResult, + sourceDocumentPersistResult, + parserRun, + parserRunPersistResult, + parsedFactorPersistResult, + failures); + + private static Phase1IngestionFamilyResult Failed( + SourceFamily sourceFamily, + string sourceKey, + Phase1IngestionFailure failure) => + new( + sourceFamily, + sourceKey, + Phase1IngestionFamilyRunStatus.Failed, + failures: [failure]); + + private static IEnumerable ValidateRequest(Phase1IngestionOrchestratorRequest request) + { + var hasValidSourceFamily = request.SourceFamilies.Any(sourceFamily => + Enum.IsDefined(sourceFamily)); + if (request.SourceFamilies.Count == 0 || !hasValidSourceFamily) + { + yield return Failure(SourceFamily.GhgProtocol, "", "request", "PHASE1_SOURCE_FAMILY_SELECTION_REQUIRED", "At least one source family must be explicitly selected.", "SourceFamilies"); + } + + for (var index = 0; index < request.SourceFamilies.Count; index++) + { + var sourceFamily = request.SourceFamilies[index]; + if (!Enum.IsDefined(sourceFamily)) + { + yield return Failure(SourceFamily.GhgProtocol, "", "request", "PHASE1_SOURCE_FAMILY_INVALID", "Selected source families must be defined Phase 1 source families.", $"SourceFamilies[{index}]"); + } + } + + if (!Enum.IsDefined(request.ExecutionMode)) + { + yield return Failure(SourceFamily.GhgProtocol, "", "request", "PHASE1_EXECUTION_MODE_INVALID", "Execution mode must be a defined Phase 1 ingestion execution mode.", "ExecutionMode"); + } + + if (request.MaxDegreeOfParallelism < 1) + { + yield return Failure(SourceFamily.GhgProtocol, "", "request", "PHASE1_MAX_DEGREE_INVALID", "MaxDegreeOfParallelism must be at least 1.", "MaxDegreeOfParallelism"); + } + + if (request.ExecutionMode == Phase1IngestionExecutionMode.Sequential && request.MaxDegreeOfParallelism != 1) + { + yield return Failure(SourceFamily.GhgProtocol, "", "request", "PHASE1_SEQUENTIAL_MAX_DEGREE_MUST_BE_ONE", "Sequential execution must use MaxDegreeOfParallelism=1.", "MaxDegreeOfParallelism"); + } + + if (request.ExecutionMode == Phase1IngestionExecutionMode.BoundedParallel) + { + yield return Failure(SourceFamily.GhgProtocol, "", "execution_mode", "PHASE1_INGESTION_BOUNDED_PARALLEL_NOT_ENABLED", "Bounded parallel execution is a declared extension point only.", "ExecutionMode"); + } + } + + private static IEnumerable ValidatePostgreSQLReadiness( + Phase1IngestionOrchestratorRequest request, + PostgreSQLRuntimeConfigGateDecision runtimeDecision) + { + if (request.RuntimeConfigGate.Requested && !runtimeDecision.RuntimeEnabled) + { + var issue = runtimeDecision.Issues.FirstOrDefault(); + yield return Failure( + SourceFamily.GhgProtocol, + "", + "postgresql_runtime_config", + issue?.Code ?? "PHASE1_INGESTION_POSTGRESQL_RUNTIME_NOT_READY", + issue?.Message ?? "PostgreSQL runtime configuration is not ready.", + issue?.FieldName ?? "RuntimeConfigGate"); + } + + if (request.SchemaBootstrapReport is { FailOnMissing: true } report && + report.MissingTableNames.Count > 0) + { + yield return Failure( + SourceFamily.GhgProtocol, + "", + "postgresql_schema_bootstrap", + "PHASE1_INGESTION_POSTGRESQL_SCHEMA_NOT_READY", + "PostgreSQL schema bootstrap reported missing required tables.", + "schema_bootstrap_report.missing_table_names"); + } + } + + private static IEnumerable ValidateAcquisition( + SourceFamily sourceFamily, + string sourceKey, + SourceAcquisitionRunResult acquisitionRun) + { + if (acquisitionRun.SourceFamily != sourceFamily || acquisitionRun.SourceKey != sourceKey) + { + yield return Failure(sourceFamily, sourceKey, "discovery", "PHASE1_ACQUISITION_SOURCE_MISMATCH", "Acquisition result source family and source key must match the selected source family."); + } + + if (acquisitionRun.Status == SourceAcquisitionRunStatus.Failed || + acquisitionRun.Status == SourceAcquisitionRunStatus.InvalidRequest) + { + yield return Failure(sourceFamily, sourceKey, "download", "PHASE1_ACQUISITION_FAILED", "Source discovery/download did not complete successfully."); + } + + if (acquisitionRun.CandidateCount == 0) + { + yield return Failure(sourceFamily, sourceKey, "discovery", "PHASE1_DISCOVERY_NO_CANDIDATES", "Source discovery returned no candidates."); + } + + if (acquisitionRun.ArtifactCount == 0) + { + yield return Failure(sourceFamily, sourceKey, "download", "PHASE1_DOWNLOAD_NO_ARTIFACTS", "Source download returned no artifacts."); + } + } + + private static IEnumerable CreateSourceDocumentRecords( + IEnumerable artifacts) + { + foreach (var artifact in artifacts) + { + var checksum = artifact.Checksum ?? new SourceDocumentChecksum( + "not_supplied", + $"{artifact.ArtifactId}_checksum_not_supplied", + IsDryRunPlaceholder: true); + yield return new SourceDocumentPersistenceRecord( + artifact.SourceFamily, + artifact.LocalReference, + checksum.Algorithm, + checksum.Value, + checksum.IsDryRunPlaceholder); + } + } + + private static ParserRunResult CreateParserRunResult(ParserAdapterRunResult parserRun) + { + var request = new ParserRunRequest( + parserRun.SourceFamily, + parserRun.ArtifactReferences.FirstOrDefault() ?? $"{parserRun.SourceKey}_artifact_unavailable", + "not_supplied", + $"{parserRun.SourceKey}_parser_run_checksum_not_supplied", + IsDryRunChecksum: true); + var rejectedRows = parserRun.ValidationIssues.Count(issue => issue.Severity == ParserValidationIssueSeverity.Error); + return new ParserRunResult( + request, + parserRun.Status, + parserRun.RowCount + rejectedRows, + parserRun.RowCount, + rejectedRows, + parserRun.ValidationIssues.Select(issue => new ParserRunIssue( + issue.Code, + issue.Message, + issue.Severity == ParserValidationIssueSeverity.Error + ? ParserRunIssueSeverity.Error + : ParserRunIssueSeverity.Warning, + issue.FieldKey))); + } + + private static IEnumerable FromParserIssues( + SourceFamily sourceFamily, + string sourceKey, + IEnumerable issues) => + issues + .Where(issue => issue.Severity == ParserValidationIssueSeverity.Error) + .Select(issue => Failure(sourceFamily, sourceKey, "parser", issue.Code, issue.Message, issue.FieldKey)); + + private static IEnumerable FromAcquisitionRepositoryIssues( + SourceFamily sourceFamily, + string sourceKey, + IEnumerable issues) => + issues.Select(issue => Failure(sourceFamily, sourceKey, "source_acquisition_run_persistence", issue.Code, issue.Message, issue.FieldName, issue.Severity)); + + private static IEnumerable FromSourceDocumentRepositoryIssues( + SourceFamily sourceFamily, + string sourceKey, + IEnumerable issues) => + issues.Select(issue => Failure(sourceFamily, sourceKey, "source_document_persistence", issue.Code, issue.Message, issue.FieldName, issue.Severity)); + + private static IEnumerable FromParserRunRepositoryIssues( + SourceFamily sourceFamily, + string sourceKey, + IEnumerable issues) => + issues.Select(issue => Failure(sourceFamily, sourceKey, "parser_run_persistence", issue.Code, issue.Message, issue.FieldName, issue.Severity)); + + private static IEnumerable FromParsedFactorPersistenceIssues( + SourceFamily sourceFamily, + string sourceKey, + IEnumerable issues) => + issues + .Where(issue => issue.Severity == "error") + .Select(issue => Failure(sourceFamily, sourceKey, "parsed_factor_persistence", issue.Code, issue.Message, issue.FieldName, issue.Severity)); + + private static Phase1IngestionFailure Failure( + SourceFamily sourceFamily, + string sourceKey, + string stage, + string code, + string message, + string? fieldName = null, + string severity = "error") => + new(sourceFamily, sourceKey, stage, code, message, fieldName, severity); + +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/Phase1OperationalDiagnostics.cs b/src/dotnet/CarbonOps.Parser.Contracts/Phase1OperationalDiagnostics.cs new file mode 100644 index 00000000..a212d39d --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/Phase1OperationalDiagnostics.cs @@ -0,0 +1,358 @@ +using System.Collections; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace CarbonOps.Parser.Contracts; + +public delegate void Phase1OperationalEventSink(string jsonEvent); + +public static partial class Phase1OperationalDiagnostics +{ + public const string OperationalLoggerName = "CarbonOps.Parser.Phase1"; + public const string Redacted = ""; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + WriteIndented = false, + }; + + private static readonly HashSet SensitiveRuntimeOptionFields = new(StringComparer.OrdinalIgnoreCase) + { + "host", + "database", + "username", + "application_name", + "dsn", + "connection_string", + "connectionstring", + "connection_uri", + "connectionuri", + "database_url", + "databaseurl", + "api_key", + "apikey", + "access_key", + "accesskey", + "private_key", + "privatekey", + }; + + private static readonly string[] SensitiveKeyParts = + [ + "password", + "passwd", + "pwd", + "secret", + "token", + "credential", + "dsn", + "connection_string", + "connection_uri", + "database_url", + "api_key", + "access_key", + "private_key", + ]; + + private static readonly string[] CompactSensitiveKeyParts = SensitiveKeyParts + .Select(CompactFieldName) + .ToArray(); + + public static IReadOnlyDictionary BuildOperationalEvent( + string eventName, + IReadOnlyDictionary payload) + { + var redactedPayload = RedactDiagnosticValue("payload", payload); + var eventPayload = redactedPayload as IReadOnlyDictionary ?? + new SortedDictionary(StringComparer.Ordinal); + var result = new SortedDictionary(StringComparer.Ordinal) + { + ["event"] = eventName, + }; + + foreach (var item in eventPayload) + { + result[item.Key] = NormalizeDiagnosticValue(item.Value); + } + + return result; + } + + public static string SerializeOperationalEvent( + string eventName, + IReadOnlyDictionary payload) => + JsonSerializer.Serialize(BuildOperationalEvent(eventName, payload), JsonOptions); + + public static object? RedactDiagnosticValue(string fieldName, object? value) + { + if (IsSensitiveField(fieldName)) + { + return value is null ? null : Redacted; + } + + return value switch + { + null => null, + string text => SafeText(text), + IReadOnlyDictionary mapping => StableMapping(mapping), + IDictionary dictionary => StableDictionary(dictionary), + IEnumerable enumerable when value is not string => StableEnumerable(fieldName, enumerable), + Enum enumValue => enumValue.ToString(), + _ => value, + }; + } + + public static IReadOnlyDictionary SummarizePostgreSQLOptionsForDiagnostics( + PostgreSQLPersistenceOptions options) => + new SortedDictionary(StringComparer.Ordinal) + { + ["application_name"] = options.ApplicationName is null ? null : Redacted, + ["connect_timeout_seconds"] = options.ConnectTimeoutSeconds, + ["database"] = Redacted, + ["host"] = Redacted, + ["password_set"] = options.PasswordSet, + ["port"] = options.Port, + ["ssl_mode"] = options.SslMode, + ["username"] = Redacted, + }; + + public static IReadOnlyDictionary SummarizeOrchestratorRequest( + Phase1IngestionOrchestratorRequest request) => + new SortedDictionary(StringComparer.Ordinal) + { + ["correlation_id"] = SafeText(request.CorrelationId), + ["execution_mode"] = Phase1ExecutionModeWireName(request.ExecutionMode), + ["max_degree_of_parallelism"] = request.MaxDegreeOfParallelism, + ["run_id"] = SafeText(request.RunId), + ["source_families"] = request.SourceFamilies.Select(sourceFamily => sourceFamily.ToWireName()).ToArray(), + }; + + public static IReadOnlyDictionary SummarizeFamilyResultForDiagnostics( + Phase1IngestionFamilyResult familyResult, + string? runId = null, + string? correlationId = null) => + new SortedDictionary(StringComparer.Ordinal) + { + ["correlation_id"] = SafeText(correlationId), + ["documents"] = DocumentSummaries(familyResult.AcquisitionRun?.Artifacts ?? []), + ["failures"] = FailureSummaries(familyResult.Failures), + ["parser"] = new SortedDictionary(StringComparer.Ordinal) + { + ["accepted_row_count"] = familyResult.ParserAcceptedRowCount, + ["failure_count"] = familyResult.ParserFailureCount, + ["result_status"] = familyResult.ParserRun?.Status.ToWireName(), + ["run_id"] = SafeText(familyResult.ParserRun?.RunId), + ["validation_issue_count"] = familyResult.ParserRun?.IssueCount ?? 0, + }, + ["persistence"] = new SortedDictionary(StringComparer.Ordinal) + { + ["parsed_factor_detail_count"] = familyResult.PersistedDetailCount, + ["parsed_factor_master_count"] = familyResult.PersistedMasterCount, + ["parser_run_count"] = familyResult.ParserRunPersistResult?.PersistedCount ?? 0, + ["source_document_count"] = familyResult.SourceDocumentPersistResult?.PersistedCount ?? 0, + ["source_run_count"] = familyResult.AcquisitionRunPersistResult?.PersistedCount ?? 0, + }, + ["run_id"] = SafeText(runId ?? familyResult.AcquisitionRun?.RunId), + ["source_family"] = familyResult.SourceFamily.ToWireName(), + ["source_key"] = familyResult.SourceKey, + ["status"] = Phase1FamilyRunStatusWireName(familyResult.Status), + }; + + public static IReadOnlyDictionary SummarizeOrchestratorResultForDiagnostics( + Phase1IngestionOrchestratorResult result) => + new SortedDictionary(StringComparer.Ordinal) + { + ["correlation_id"] = SafeText(result.Request.CorrelationId), + ["failures"] = FailureSummaries(result.Failures), + ["run_id"] = SafeText(result.Request.RunId), + ["selected_source_families"] = result.SelectedSourceFamilies + .Select(sourceFamily => sourceFamily.ToWireName()) + .ToArray(), + ["source_family_statuses"] = result.FamilyResults + .Select(familyResult => new SortedDictionary(StringComparer.Ordinal) + { + ["source_family"] = familyResult.SourceFamily.ToWireName(), + ["status"] = Phase1FamilyRunStatusWireName(familyResult.Status), + }) + .ToArray(), + ["status"] = Phase1RunStatusWireName(result.Status), + ["summary"] = new SortedDictionary(StringComparer.Ordinal) + { + ["completed_family_count"] = result.CompletedSourceFamilyCount, + ["failed_family_count"] = result.FailedSourceFamilyCount, + ["failure_count"] = result.FailureCount, + ["parsed_factor_row_count"] = result.TotalParserAcceptedRowCount, + ["parser_run_count"] = result.FamilyResults.Count(familyResult => familyResult.ParserRun is not null), + ["persisted_detail_count"] = result.TotalPersistedDetailCount, + ["persisted_master_count"] = result.TotalPersistedMasterCount, + ["persisted_parser_run_count"] = result.FamilyResults.Sum(familyResult => + familyResult.ParserRunPersistResult?.PersistedCount ?? 0), + ["persisted_source_document_count"] = result.FamilyResults.Sum(familyResult => + familyResult.SourceDocumentPersistResult?.PersistedCount ?? 0), + ["persisted_source_run_count"] = result.FamilyResults.Sum(familyResult => + familyResult.AcquisitionRunPersistResult?.PersistedCount ?? 0), + ["requested_family_count"] = result.Request.SourceFamilies.Count, + ["source_artifact_count"] = result.TotalSourceDocumentMetadataCount, + ["source_candidate_count"] = result.FamilyResults.Sum(familyResult => familyResult.SourceCandidateCount), + }, + }; + + internal static void Emit( + Phase1OperationalEventSink? sink, + string eventName, + IReadOnlyDictionary payload) + { + sink?.Invoke(SerializeOperationalEvent(eventName, payload)); + } + + private static IReadOnlyDictionary[] DocumentSummaries( + IEnumerable artifacts) => + artifacts + .Select(artifact => new SortedDictionary(StringComparer.Ordinal) + { + ["checksum_sha256"] = SafeChecksum(artifact.Checksum?.Value), + ["document_id"] = SafeText(artifact.ArtifactId), + ["source_family"] = artifact.SourceFamily.ToWireName(), + ["source_key"] = artifact.SourceKey, + }) + .ToArray(); + + private static IReadOnlyDictionary[] FailureSummaries( + IEnumerable failures) => + failures + .Select(failure => new SortedDictionary(StringComparer.Ordinal) + { + ["code"] = failure.Code, + ["field_name"] = failure.FieldName, + ["message"] = SafeText(failure.Message), + ["severity"] = failure.Severity, + ["source_family"] = failure.SourceFamily.ToWireName(), + ["source_key"] = failure.SourceKey, + ["stage"] = failure.Stage, + }) + .ToArray(); + + private static IReadOnlyDictionary StableMapping( + IReadOnlyDictionary mapping) + { + var result = new SortedDictionary(StringComparer.Ordinal); + foreach (var item in mapping) + { + result[item.Key] = RedactDiagnosticValue(item.Key, item.Value); + } + + return result; + } + + private static IReadOnlyDictionary StableDictionary(IDictionary dictionary) + { + var result = new SortedDictionary(StringComparer.Ordinal); + foreach (DictionaryEntry item in dictionary) + { + var key = Convert.ToString(item.Key) ?? string.Empty; + result[key] = RedactDiagnosticValue(key, item.Value); + } + + return result; + } + + private static object?[] StableEnumerable(string fieldName, IEnumerable enumerable) + { + var result = new List(); + foreach (var item in enumerable) + { + result.Add(RedactDiagnosticValue(fieldName, item)); + } + + return result.ToArray(); + } + + private static object? NormalizeDiagnosticValue(object? value) => + value switch + { + IReadOnlyDictionary mapping => StableMapping(mapping), + IDictionary dictionary => StableDictionary(dictionary), + IEnumerable enumerable when value is not string => StableEnumerable("item", enumerable), + Enum enumValue => enumValue.ToString(), + _ => value, + }; + + private static string? SafeText(string? value) + { + if (value is null) + { + return null; + } + + var withoutUserInfo = UserInfoUriPattern().Replace(value, $"//{Redacted}@"); + return SensitiveAssignmentPattern().Replace( + withoutUserInfo, + match => $"{match.Groups[1].Value}={Redacted}"); + } + + private static string? SafeChecksum(string? value) + { + if (value is null || !ChecksumPattern().IsMatch(value)) + { + return null; + } + + return value.ToLowerInvariant(); + } + + private static bool IsSensitiveField(string fieldName) + { + var normalized = fieldName.Trim().ToLowerInvariant(); + var compact = CompactFieldName(normalized); + if (normalized == "password_set") + { + return false; + } + + return SensitiveRuntimeOptionFields.Contains(normalized) || + SensitiveKeyParts.Any(part => normalized.Contains(part, StringComparison.Ordinal)) || + CompactSensitiveKeyParts.Any(part => compact.Contains(part, StringComparison.Ordinal)); + } + + private static string CompactFieldName(string fieldName) => + fieldName.Replace("_", string.Empty, StringComparison.Ordinal) + .Replace("-", string.Empty, StringComparison.Ordinal); + + private static string Phase1ExecutionModeWireName(Phase1IngestionExecutionMode value) => + value switch + { + Phase1IngestionExecutionMode.Sequential => "sequential", + Phase1IngestionExecutionMode.BoundedParallel => "bounded_parallel", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown Phase 1 execution mode."), + }; + + private static string Phase1RunStatusWireName(Phase1IngestionRunStatus value) => + value switch + { + Phase1IngestionRunStatus.Completed => "completed", + Phase1IngestionRunStatus.CompletedWithFailures => "completed_with_failures", + Phase1IngestionRunStatus.Failed => "failed", + Phase1IngestionRunStatus.NotExecutable => "not_executable", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown Phase 1 run status."), + }; + + private static string Phase1FamilyRunStatusWireName(Phase1IngestionFamilyRunStatus value) => + value switch + { + Phase1IngestionFamilyRunStatus.Completed => "completed", + Phase1IngestionFamilyRunStatus.Failed => "failed", + Phase1IngestionFamilyRunStatus.Skipped => "skipped", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown Phase 1 family run status."), + }; + + [GeneratedRegex("^[0-9a-fA-F]{64}$")] + private static partial Regex ChecksumPattern(); + + [GeneratedRegex("//[^/\\s:@]+:[^@\\s/]+@")] + private static partial Regex UserInfoUriPattern(); + + [GeneratedRegex("(?i)\\b(password|passwd|pwd|secret|token|dsn|connection[_-]?string)=([^\\s;,]+)")] + private static partial Regex SensitiveAssignmentPattern(); +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/Phase1ServiceHost.cs b/src/dotnet/CarbonOps.Parser.Contracts/Phase1ServiceHost.cs new file mode 100644 index 00000000..409649e4 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/Phase1ServiceHost.cs @@ -0,0 +1,511 @@ +namespace CarbonOps.Parser.Contracts; + +public enum Phase1ServiceHostStatus +{ + Created = 0, + Ready = 1, + Blocked = 2, + Running = 3, + ShutdownRequested = 4, + Stopped = 5, +} + +public enum Phase1ScheduledRunStatus +{ + Started = 0, + SkippedNotStarted = 1, + SkippedAlreadyRunning = 2, + SkippedShuttingDown = 3, +} + +public sealed record Phase1ServiceHostConfig +{ + public IReadOnlyList SourceFamilies { get; } + + public PostgreSQLPersistenceOptions PostgreSQLOptions { get; } + + public string RunIdPrefix { get; } + + public int ScheduleIntervalSeconds { get; } + + public Phase1IngestionExecutionMode ExecutionMode { get; } + + public int MaxDegreeOfParallelism { get; } + + public PostgreSQLSchemaBootstrapMode SchemaBootstrapMode { get; } + + public bool FailOnMissingSchema { get; } + + public PostgreSQLRuntimeConfigGate RuntimeConfigGate { get; } + + public Phase1ServiceHostConfig( + IEnumerable sourceFamilies, + PostgreSQLPersistenceOptions postgreSQLOptions, + string runIdPrefix = "phase1-scheduled", + int scheduleIntervalSeconds = 3600, + Phase1IngestionExecutionMode executionMode = Phase1IngestionExecutionMode.Sequential, + int maxDegreeOfParallelism = 1, + PostgreSQLSchemaBootstrapMode schemaBootstrapMode = PostgreSQLSchemaBootstrapMode.CheckOnly, + bool failOnMissingSchema = true, + PostgreSQLRuntimeConfigGate? runtimeConfigGate = null) + { + SourceFamilies = Array.AsReadOnly(sourceFamilies.ToArray()); + PostgreSQLOptions = postgreSQLOptions; + RunIdPrefix = runIdPrefix; + ScheduleIntervalSeconds = scheduleIntervalSeconds; + ExecutionMode = executionMode; + MaxDegreeOfParallelism = maxDegreeOfParallelism; + SchemaBootstrapMode = schemaBootstrapMode; + FailOnMissingSchema = failOnMissingSchema; + RuntimeConfigGate = runtimeConfigGate ?? new PostgreSQLRuntimeConfigGate(); + } +} + +public sealed record Phase1ServiceHostIssue( + string Code, + string Message, + string? FieldName = null, + string Severity = "error"); + +public sealed record Phase1ServiceHostStartupResult +{ + public Phase1ServiceHostStatus Status { get; } + + public IReadOnlyList Issues { get; } + + public PostgreSQLSchemaBootstrapReport? SchemaBootstrapReport { get; } + + public bool IsReady => Status == Phase1ServiceHostStatus.Ready; + + public Phase1ServiceHostStartupResult( + Phase1ServiceHostStatus status, + IEnumerable issues, + PostgreSQLSchemaBootstrapReport? schemaBootstrapReport = null) + { + Status = status; + Issues = Array.AsReadOnly(issues.ToArray()); + SchemaBootstrapReport = schemaBootstrapReport; + } +} + +public sealed record Phase1ScheduledRunResult +{ + public Phase1ScheduledRunStatus Status { get; } + + public string? RunId { get; } + + public Phase1IngestionOrchestratorResult? OrchestratorResult { get; } + + public IReadOnlyList Issues { get; } + + public Phase1ScheduledRunResult( + Phase1ScheduledRunStatus status, + string? runId = null, + Phase1IngestionOrchestratorResult? orchestratorResult = null, + IEnumerable? issues = null) + { + Status = status; + RunId = runId; + OrchestratorResult = orchestratorResult; + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} + +public delegate PostgreSQLSchemaBootstrapReport Phase1SchemaBootstrapChecker( + PostgreSQLSchemaBootstrapMode mode, + bool failOnMissing); + +public delegate Phase1IngestionOrchestratorResult Phase1OrchestratorRunner( + Phase1IngestionOrchestratorRequest request); + +public sealed class Phase1ScheduledIngestionServiceHost +{ + private readonly object syncRoot = new(); + private readonly Phase1SchemaBootstrapChecker schemaBootstrapChecker; + private readonly Phase1OrchestratorRunner orchestratorRunner; + private readonly Phase1OperationalEventSink? operationalEventSink; + private PostgreSQLSchemaBootstrapReport? schemaBootstrapReport; + private bool runInProgress; + private bool shutdownRequested; + private int runSequence; + + public Phase1ServiceHostConfig Config { get; } + + public Phase1ServiceHostStatus Status { get; private set; } = Phase1ServiceHostStatus.Created; + + public Phase1ScheduledIngestionServiceHost( + Phase1ServiceHostConfig config, + Phase1OrchestratorRunner orchestratorRunner, + Phase1SchemaBootstrapChecker? schemaBootstrapChecker = null, + Phase1OperationalEventSink? operationalEventSink = null) + { + Config = config; + this.orchestratorRunner = orchestratorRunner; + this.schemaBootstrapChecker = schemaBootstrapChecker ?? DefaultSchemaBootstrapChecker; + this.operationalEventSink = operationalEventSink; + } + + public Phase1ServiceHostStartupResult Start() + { + Phase1OperationalDiagnostics.Emit( + operationalEventSink, + "phase1_service_host_starting", + StartingDiagnosticPayload(Config)); + + var issues = ValidateConfig(Config).ToList(); + PostgreSQLSchemaBootstrapReport? report = null; + + if (issues.Count == 0) + { + report = schemaBootstrapChecker(Config.SchemaBootstrapMode, Config.FailOnMissingSchema); + issues.AddRange(SchemaBootstrapIssues(report)); + } + + var status = issues.Count == 0 + ? Phase1ServiceHostStatus.Ready + : Phase1ServiceHostStatus.Blocked; + var result = new Phase1ServiceHostStartupResult(status, issues, report); + + lock (syncRoot) + { + if (shutdownRequested) + { + Status = Phase1ServiceHostStatus.Stopped; + var blockedResult = new Phase1ServiceHostStartupResult( + Phase1ServiceHostStatus.Blocked, + [ + new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_SHUTDOWN_REQUESTED", + "Service host startup is blocked after shutdown.", + "status"), + ], + report); + Phase1OperationalDiagnostics.Emit( + operationalEventSink, + "phase1_service_host_started", + StartupDiagnosticPayload(blockedResult)); + return blockedResult; + } + + Status = status; + schemaBootstrapReport = report; + } + + Phase1OperationalDiagnostics.Emit( + operationalEventSink, + "phase1_service_host_started", + StartupDiagnosticPayload(result)); + return result; + } + + public Phase1ScheduledRunResult TriggerScheduledRun() + { + string runId; + Phase1IngestionOrchestratorRequest request; + + lock (syncRoot) + { + if (shutdownRequested) + { + var result = new Phase1ScheduledRunResult( + Phase1ScheduledRunStatus.SkippedShuttingDown, + issues: [ShutdownIssue()]); + Phase1OperationalDiagnostics.Emit( + operationalEventSink, + "phase1_service_host_scheduled_run_skipped", + ScheduledRunDiagnosticPayload(result)); + return result; + } + + if (runInProgress) + { + var result = new Phase1ScheduledRunResult( + Phase1ScheduledRunStatus.SkippedAlreadyRunning, + issues: + [ + new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_RUN_ALREADY_IN_PROGRESS", + "Scheduled trigger skipped while a run is active.", + "run_in_progress"), + ]); + Phase1OperationalDiagnostics.Emit( + operationalEventSink, + "phase1_service_host_scheduled_run_skipped", + ScheduledRunDiagnosticPayload(result)); + return result; + } + + if (Status != Phase1ServiceHostStatus.Ready) + { + var result = new Phase1ScheduledRunResult( + Phase1ScheduledRunStatus.SkippedNotStarted, + issues: + [ + new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_NOT_READY", + "Service host must start successfully first.", + "status"), + ]); + Phase1OperationalDiagnostics.Emit( + operationalEventSink, + "phase1_service_host_scheduled_run_skipped", + ScheduledRunDiagnosticPayload(result)); + return result; + } + + runInProgress = true; + Status = Phase1ServiceHostStatus.Running; + runSequence++; + runId = $"{Config.RunIdPrefix}-{runSequence:000000}"; + request = BuildOrchestratorRequest(runId); + } + + Phase1OperationalDiagnostics.Emit( + operationalEventSink, + "phase1_service_host_scheduled_run_started", + new SortedDictionary(StringComparer.Ordinal) + { + ["correlation_id"] = request.CorrelationId, + ["run_id"] = runId, + ["source_families"] = request.SourceFamilies.Select(sourceFamily => sourceFamily.ToWireName()).ToArray(), + }); + + Phase1IngestionOrchestratorResult orchestratorResult; + try + { + orchestratorResult = orchestratorRunner(request); + } + finally + { + lock (syncRoot) + { + runInProgress = false; + Status = shutdownRequested + ? Phase1ServiceHostStatus.Stopped + : Phase1ServiceHostStatus.Ready; + } + } + + var scheduledRunResult = new Phase1ScheduledRunResult( + Phase1ScheduledRunStatus.Started, + runId, + orchestratorResult); + Phase1OperationalDiagnostics.Emit( + operationalEventSink, + "phase1_service_host_scheduled_run_completed", + ScheduledRunDiagnosticPayload(scheduledRunResult)); + return scheduledRunResult; + } + + public Phase1ServiceHostStatus RequestShutdown() + { + lock (syncRoot) + { + shutdownRequested = true; + Status = runInProgress + ? Phase1ServiceHostStatus.ShutdownRequested + : Phase1ServiceHostStatus.Stopped; + return Status; + } + } + + public static IReadOnlyList ValidateConfig( + Phase1ServiceHostConfig config) + { + var issues = new List(); + + if (config.SourceFamilies.Count == 0) + { + issues.Add(new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_MISSING_SOURCE_FAMILIES", + "At least one Phase 1 source family must be configured.", + "source_families")); + } + + for (var index = 0; index < config.SourceFamilies.Count; index++) + { + if (!Enum.IsDefined(config.SourceFamilies[index])) + { + issues.Add(new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_UNSUPPORTED_SOURCE_FAMILY", + "Configured source family must be a Phase 1 family.", + $"source_families[{index}]")); + } + } + + if (string.IsNullOrWhiteSpace(config.RunIdPrefix)) + { + issues.Add(new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_MISSING_RUN_ID_PREFIX", + "run_id_prefix must be a non-empty string.", + "run_id_prefix")); + } + + if (config.ScheduleIntervalSeconds <= 0) + { + issues.Add(new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_INVALID_SCHEDULE_INTERVAL", + "schedule_interval_seconds must be a positive integer.", + "schedule_interval_seconds")); + } + + if (config.ExecutionMode == Phase1IngestionExecutionMode.Sequential) + { + if (config.MaxDegreeOfParallelism != 1) + { + issues.Add(new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_INVALID_SEQUENTIAL_PARALLELISM", + "Sequential scheduled execution requires MaxDegreeOfParallelism=1.", + "max_degree_of_parallelism")); + } + } + else + { + issues.Add(new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_UNSUPPORTED_EXECUTION_MODE", + "Only sequential scheduled execution is enabled.", + "execution_mode")); + } + + foreach (var optionIssue in PostgreSQLPersistenceOptionsValidator.Validate(config.PostgreSQLOptions).Issues) + { + issues.Add(new Phase1ServiceHostIssue( + optionIssue.Code, + optionIssue.Message, + $"postgresql_options.{optionIssue.FieldName}", + optionIssue.Severity)); + } + + if (!config.PostgreSQLOptions.PasswordSet) + { + issues.Add(new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_POSTGRESQL_PASSWORD_NOT_CONFIRMED", + "postgresql_options.password_set must confirm that a credential is available outside this config object.", + "postgresql_options.password_set")); + } + + return Array.AsReadOnly(issues.ToArray()); + } + + private Phase1IngestionOrchestratorRequest BuildOrchestratorRequest(string runId) => + new( + Config.SourceFamilies, + Config.ExecutionMode, + Config.MaxDegreeOfParallelism, + Config.RuntimeConfigGate, + runId, + schemaBootstrapReport: schemaBootstrapReport, + operationalEventSink: operationalEventSink); + + private static IEnumerable SchemaBootstrapIssues( + PostgreSQLSchemaBootstrapReport report) + { + if (!report.FailOnMissing || report.MissingTableNames.Count == 0) + { + yield break; + } + + yield return new Phase1ServiceHostIssue( + "PHASE1_SERVICE_HOST_POSTGRESQL_SCHEMA_NOT_READY", + "Required Phase 1 PostgreSQL tables are missing.", + "schema_bootstrap_report.missing_table_names"); + } + + private static PostgreSQLSchemaBootstrapReport DefaultSchemaBootstrapChecker( + PostgreSQLSchemaBootstrapMode mode, + bool failOnMissing) => + PostgreSQLSchemaBootstrapBoundary.BuildReport(mode, failOnMissing: failOnMissing); + + private static Phase1ServiceHostIssue ShutdownIssue() => + new( + "PHASE1_SERVICE_HOST_SHUTTING_DOWN", + "Scheduled trigger skipped because shutdown was requested.", + "status"); + + private static IReadOnlyDictionary StartingDiagnosticPayload( + Phase1ServiceHostConfig config) => + new SortedDictionary(StringComparer.Ordinal) + { + ["postgresql_options"] = Phase1OperationalDiagnostics.SummarizePostgreSQLOptionsForDiagnostics( + config.PostgreSQLOptions), + ["run_id_prefix"] = config.RunIdPrefix, + ["schedule_interval_seconds"] = config.ScheduleIntervalSeconds, + ["source_families"] = config.SourceFamilies.Select(sourceFamily => sourceFamily.ToWireName()).ToArray(), + }; + + private static IReadOnlyDictionary StartupDiagnosticPayload( + Phase1ServiceHostStartupResult result) + { + var report = result.SchemaBootstrapReport; + return new SortedDictionary(StringComparer.Ordinal) + { + ["issues"] = result.Issues.Select(ServiceHostIssuePayload).ToArray(), + ["schema_bootstrap"] = new SortedDictionary(StringComparer.Ordinal) + { + ["fail_on_missing"] = report?.FailOnMissing, + ["missing_table_count"] = report?.MissingTableNames.Count, + ["mode"] = report is null ? null : SchemaBootstrapModeWireName(report.Mode), + ["status"] = null, + }, + ["status"] = ServiceHostStatusWireName(result.Status), + }; + } + + private static IReadOnlyDictionary ScheduledRunDiagnosticPayload( + Phase1ScheduledRunResult result) + { + var payload = new SortedDictionary(StringComparer.Ordinal) + { + ["issues"] = result.Issues.Select(ServiceHostIssuePayload).ToArray(), + ["run_id"] = result.RunId, + ["status"] = ScheduledRunStatusWireName(result.Status), + }; + if (result.OrchestratorResult is not null) + { + payload["orchestrator"] = Phase1OperationalDiagnostics.SummarizeOrchestratorResultForDiagnostics( + result.OrchestratorResult); + } + + return payload; + } + + private static IReadOnlyDictionary ServiceHostIssuePayload( + Phase1ServiceHostIssue issue) => + new SortedDictionary(StringComparer.Ordinal) + { + ["code"] = issue.Code, + ["field_name"] = issue.FieldName, + ["message"] = issue.Message, + ["severity"] = issue.Severity, + }; + + private static string ServiceHostStatusWireName(Phase1ServiceHostStatus status) => + status switch + { + Phase1ServiceHostStatus.Created => "created", + Phase1ServiceHostStatus.Ready => "ready", + Phase1ServiceHostStatus.Blocked => "blocked", + Phase1ServiceHostStatus.Running => "running", + Phase1ServiceHostStatus.ShutdownRequested => "shutdown_requested", + Phase1ServiceHostStatus.Stopped => "stopped", + _ => throw new ArgumentOutOfRangeException(nameof(status), status, "Unknown Phase 1 service host status."), + }; + + private static string ScheduledRunStatusWireName(Phase1ScheduledRunStatus status) => + status switch + { + Phase1ScheduledRunStatus.Started => "started", + Phase1ScheduledRunStatus.SkippedNotStarted => "skipped_not_started", + Phase1ScheduledRunStatus.SkippedAlreadyRunning => "skipped_already_running", + Phase1ScheduledRunStatus.SkippedShuttingDown => "skipped_shutting_down", + _ => throw new ArgumentOutOfRangeException(nameof(status), status, "Unknown Phase 1 scheduled run status."), + }; + + private static string SchemaBootstrapModeWireName(PostgreSQLSchemaBootstrapMode mode) => + mode switch + { + PostgreSQLSchemaBootstrapMode.CheckOnly => "check_only", + PostgreSQLSchemaBootstrapMode.CreateMissing => "create_missing", + _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, "Unknown PostgreSQL schema bootstrap mode."), + }; +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLPersistenceOptions.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLPersistenceOptions.cs new file mode 100644 index 00000000..5e0b5a56 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLPersistenceOptions.cs @@ -0,0 +1,126 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record PostgreSQLPersistenceOptions( + string Host, + int Port, + string Database, + string Username, + bool PasswordSet = false, + string? SslMode = null, + string? ApplicationName = null, + int? ConnectTimeoutSeconds = null); + +public sealed record PostgreSQLPersistenceOptionsValidationIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); + +public sealed record PostgreSQLPersistenceOptionsValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public PostgreSQLPersistenceOptionsValidationResult( + IEnumerable? issues = null) + { + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} + +public static class PostgreSQLPersistenceOptionsValidator +{ + public static PostgreSQLPersistenceOptionsValidationResult Validate( + PostgreSQLPersistenceOptions options) + { + var issues = new List(); + + ValidateRequiredText( + options.Host, + "host", + "POSTGRESQL_OPTIONS_MISSING_HOST", + "host must be a non-empty string.", + issues); + ValidatePort(options.Port, issues); + ValidateRequiredText( + options.Database, + "database", + "POSTGRESQL_OPTIONS_MISSING_DATABASE", + "database must be a non-empty string.", + issues); + ValidateRequiredText( + options.Username, + "username", + "POSTGRESQL_OPTIONS_MISSING_USERNAME", + "username must be a non-empty string.", + issues); + ValidateOptionalText( + options.SslMode, + "ssl_mode", + "POSTGRESQL_OPTIONS_BLANK_SSL_MODE", + "ssl_mode must be non-empty when provided.", + issues); + ValidateOptionalText( + options.ApplicationName, + "application_name", + "POSTGRESQL_OPTIONS_BLANK_APPLICATION_NAME", + "application_name must be non-empty when provided.", + issues); + ValidateTimeout(options.ConnectTimeoutSeconds, issues); + + return new PostgreSQLPersistenceOptionsValidationResult(issues); + } + + private static void ValidateRequiredText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (string.IsNullOrWhiteSpace(value)) + { + issues.Add(new PostgreSQLPersistenceOptionsValidationIssue(code, message, fieldName)); + } + } + + private static void ValidateOptionalText( + string? value, + string fieldName, + string code, + string message, + ICollection issues) + { + if (value is not null && string.IsNullOrWhiteSpace(value)) + { + issues.Add(new PostgreSQLPersistenceOptionsValidationIssue(code, message, fieldName)); + } + } + + private static void ValidatePort( + int value, + ICollection issues) + { + if (value is < 1 or > 65535) + { + issues.Add(new PostgreSQLPersistenceOptionsValidationIssue( + "POSTGRESQL_OPTIONS_INVALID_PORT", + "port must be an integer between 1 and 65535.", + "port")); + } + } + + private static void ValidateTimeout( + int? value, + ICollection issues) + { + if (value is <= 0) + { + issues.Add(new PostgreSQLPersistenceOptionsValidationIssue( + "POSTGRESQL_OPTIONS_INVALID_CONNECT_TIMEOUT", + "connect_timeout_seconds must be a positive integer when provided.", + "connect_timeout_seconds")); + } + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGate.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGate.cs new file mode 100644 index 00000000..132aa927 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGate.cs @@ -0,0 +1,8 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record PostgreSQLRuntimeConfigGate( + bool Requested = false, + bool SafetyGateApproved = false, + bool OptionsContractAvailable = false, + bool ExplicitRuntimeOptIn = false, + bool SecretSourceApproved = false); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateDecision.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateDecision.cs new file mode 100644 index 00000000..b038193e --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateDecision.cs @@ -0,0 +1,56 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record PostgreSQLRuntimeConfigGateDecision +{ + public PostgreSQLRuntimeConfigGateStatus Status { get; } + + public bool Requested { get; } + + public string Reason { get; } + + public bool ConfigLoadingEnabled { get; } + + public bool RuntimeEnabled { get; } + + public bool LoadsEnvironment { get; } + + public bool LoadsConfigFiles { get; } + + public bool LoadsCredentials { get; } + + public IReadOnlyList RequiredFutureComponents { get; } + + public IReadOnlyList SafeOperationalNotes { get; } + + public IReadOnlyList Issues { get; } + + public int RequiredFutureComponentCount => RequiredFutureComponents.Count; + + public int IssueCount => Issues.Count; + + public PostgreSQLRuntimeConfigGateDecision( + PostgreSQLRuntimeConfigGateStatus status, + bool requested, + string reason, + bool configLoadingEnabled, + bool runtimeEnabled, + bool loadsEnvironment, + bool loadsConfigFiles, + bool loadsCredentials, + IEnumerable requiredFutureComponents, + IEnumerable safeOperationalNotes, + IEnumerable? issues = null) + { + Status = status; + Requested = requested; + Reason = reason; + ConfigLoadingEnabled = configLoadingEnabled; + RuntimeEnabled = runtimeEnabled; + LoadsEnvironment = loadsEnvironment; + LoadsConfigFiles = loadsConfigFiles; + LoadsCredentials = loadsCredentials; + RequiredFutureComponents = Array.AsReadOnly(requiredFutureComponents.ToArray()); + SafeOperationalNotes = Array.AsReadOnly(safeOperationalNotes.ToArray()); + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateDescription.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateDescription.cs new file mode 100644 index 00000000..c627fd1f --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateDescription.cs @@ -0,0 +1,44 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record PostgreSQLRuntimeConfigGateDescription +{ + public PostgreSQLRuntimeConfigGateStatus DefaultStatus { get; } + + public bool DisabledByDefault { get; } + + public bool AcceptsCallerIntent { get; } + + public bool LoadsEnvironment { get; } + + public bool LoadsConfigFiles { get; } + + public bool LoadsCredentials { get; } + + public bool OpensConnection { get; } + + public bool RunsSql { get; } + + public IReadOnlyList Notes { get; } + + public PostgreSQLRuntimeConfigGateDescription( + PostgreSQLRuntimeConfigGateStatus defaultStatus, + bool disabledByDefault, + bool acceptsCallerIntent, + bool loadsEnvironment, + bool loadsConfigFiles, + bool loadsCredentials, + bool opensConnection, + bool runsSql, + IEnumerable notes) + { + DefaultStatus = defaultStatus; + DisabledByDefault = disabledByDefault; + AcceptsCallerIntent = acceptsCallerIntent; + LoadsEnvironment = loadsEnvironment; + LoadsConfigFiles = loadsConfigFiles; + LoadsCredentials = loadsCredentials; + OpensConnection = opensConnection; + RunsSql = runsSql; + Notes = Array.AsReadOnly(notes.ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateEvaluator.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateEvaluator.cs new file mode 100644 index 00000000..51a0c562 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateEvaluator.cs @@ -0,0 +1,129 @@ +namespace CarbonOps.Parser.Contracts; + +public static class PostgreSQLRuntimeConfigGateEvaluator +{ + public static PostgreSQLRuntimeConfigGateDecision Evaluate(PostgreSQLRuntimeConfigGate? gate = null) + { + var activeGate = gate ?? new PostgreSQLRuntimeConfigGate(); + var requiredComponents = RequiredFutureComponents(activeGate); + + if (!activeGate.Requested) + { + return new PostgreSQLRuntimeConfigGateDecision( + PostgreSQLRuntimeConfigGateStatus.Disabled, + requested: false, + reason: "PostgreSQL runtime configuration loading is disabled by default.", + configLoadingEnabled: false, + runtimeEnabled: false, + loadsEnvironment: false, + loadsConfigFiles: false, + loadsCredentials: false, + requiredFutureComponents: requiredComponents, + safeOperationalNotes: SafeOperationalNotes(), + issues: + [ + new PostgreSQLRuntimeConfigGateIssue( + "POSTGRESQL_RUNTIME_CONFIG_DISABLED_BY_DEFAULT", + "Runtime PostgreSQL configuration loading requires explicit future enablement and remains disabled."), + ]); + } + + var readyMetadataOnly = requiredComponents.Count == 0; + + return new PostgreSQLRuntimeConfigGateDecision( + readyMetadataOnly + ? PostgreSQLRuntimeConfigGateStatus.NotEnabled + : PostgreSQLRuntimeConfigGateStatus.Blocked, + requested: true, + reason: RequestedReason(readyMetadataOnly), + configLoadingEnabled: false, + runtimeEnabled: false, + loadsEnvironment: false, + loadsConfigFiles: false, + loadsCredentials: false, + requiredFutureComponents: requiredComponents, + safeOperationalNotes: SafeOperationalNotes(), + issues: RequestedIssues(requiredComponents)); + } + + public static PostgreSQLRuntimeConfigGateDescription Describe() => + new( + PostgreSQLRuntimeConfigGateStatus.Disabled, + disabledByDefault: true, + acceptsCallerIntent: true, + loadsEnvironment: false, + loadsConfigFiles: false, + loadsCredentials: false, + opensConnection: false, + runsSql: false, + [ + "Runtime configuration gate metadata only.", + "Default decision is disabled/no-loading.", + "Requested runtime configuration remains blocked in this boundary.", + "No environment/config file/credential loading occurs.", + ]); + + private static IReadOnlyList RequiredFutureComponents(PostgreSQLRuntimeConfigGate gate) + { + var requiredComponents = new List(); + + if (!gate.SafetyGateApproved) + { + requiredComponents.Add("postgresql_implementation_safety_gate"); + } + + if (!gate.OptionsContractAvailable) + { + requiredComponents.Add("postgresql_persistence_options_contract"); + } + + if (!gate.ExplicitRuntimeOptIn) + { + requiredComponents.Add("explicit_runtime_configuration_opt_in"); + } + + if (!gate.SecretSourceApproved) + { + requiredComponents.Add("approved_secret_source"); + } + + return Array.AsReadOnly(requiredComponents.ToArray()); + } + + private static IReadOnlyList RequestedIssues( + IReadOnlyList requiredComponents) + { + if (requiredComponents.Count > 0) + { + return + [ + new PostgreSQLRuntimeConfigGateIssue( + "POSTGRESQL_RUNTIME_CONFIG_BLOCKED", + "Runtime PostgreSQL configuration loading remains blocked until future safety-gated components are complete.", + FieldName: "requested"), + ]; + } + + return + [ + new PostgreSQLRuntimeConfigGateIssue( + "POSTGRESQL_RUNTIME_CONFIG_NOT_ENABLED", + "All supplied gate metadata is marked complete, but this boundary still does not enable runtime config loading.", + FieldName: "requested"), + ]; + } + + private static string RequestedReason(bool readyMetadataOnly) => + readyMetadataOnly + ? "PostgreSQL runtime configuration was requested, but this boundary does not enable runtime config loading." + : "PostgreSQL runtime configuration was requested, but this boundary does not enable config loading and required future components are not complete."; + + private static IReadOnlyList SafeOperationalNotes() => + [ + "No environment variables are loaded.", + "No config files are read.", + "No credentials are loaded.", + "No PostgreSQL connection is opened.", + "No SQL runtime is created.", + ]; +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateIssue.cs new file mode 100644 index 00000000..e01196b0 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record PostgreSQLRuntimeConfigGateIssue( + string Code, + string Message, + string? FieldName = null, + string Severity = "warning"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateStatus.cs new file mode 100644 index 00000000..6dfe6a57 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConfigGateStatus.cs @@ -0,0 +1,8 @@ +namespace CarbonOps.Parser.Contracts; + +public enum PostgreSQLRuntimeConfigGateStatus +{ + Disabled, + Blocked, + NotEnabled, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConnection.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConnection.cs new file mode 100644 index 00000000..bfd8b638 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeConnection.cs @@ -0,0 +1,183 @@ +using Npgsql; + +namespace CarbonOps.Parser.Contracts; + +public sealed record PostgreSQLRuntimeConnectionSettings( + string Host, + int Port, + string Database, + string Username, + string Password, + string Schema, + string ApplicationName = "carbonops-parser-dotnet", + int ConnectTimeoutSeconds = 15) +{ + public PostgreSQLPersistenceOptions ToSafeOptions() => + new( + Host, + Port, + Database, + Username, + PasswordSet: !string.IsNullOrWhiteSpace(Password), + ApplicationName: ApplicationName, + ConnectTimeoutSeconds: ConnectTimeoutSeconds); +} + +public sealed record PostgreSQLRuntimeConnectionSettingsValidationIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); + +public sealed record PostgreSQLRuntimeConnectionSettingsValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public PostgreSQLRuntimeConnectionSettingsValidationResult( + IEnumerable? issues = null) + { + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} + +public static class PostgreSQLRuntimeConnectionBoundary +{ + public static PostgreSQLRuntimeConnectionSettingsValidationResult Validate( + PostgreSQLRuntimeConnectionSettings settings) + { + var issues = new List(); + var optionsValidation = PostgreSQLPersistenceOptionsValidator.Validate(settings.ToSafeOptions()); + + foreach (var issue in optionsValidation.Issues) + { + issues.Add(new PostgreSQLRuntimeConnectionSettingsValidationIssue( + issue.Code, + issue.Message, + issue.FieldName, + issue.Severity)); + } + + if (string.IsNullOrWhiteSpace(settings.Password)) + { + issues.Add(new PostgreSQLRuntimeConnectionSettingsValidationIssue( + "POSTGRESQL_RUNTIME_MISSING_PASSWORD", + "password must be configured for explicit PostgreSQL runtime commands.", + "password")); + } + + ValidateIdentifier(settings.Schema, "schema", issues); + + return new PostgreSQLRuntimeConnectionSettingsValidationResult(issues); + } + + public static bool TryCreateFromProductionConfig( + IReadOnlyDictionary values, + out PostgreSQLRuntimeConnectionSettings? settings, + out IReadOnlyList issues) + { + settings = null; + var collectedIssues = new List(); + + if (!int.TryParse(Get(values, "CARBONOPS_PARSER_POSTGRES_PORT"), out var port)) + { + port = 0; + } + + settings = new PostgreSQLRuntimeConnectionSettings( + Get(values, "CARBONOPS_PARSER_POSTGRES_HOST") ?? string.Empty, + port, + Get(values, "CARBONOPS_PARSER_POSTGRES_DATABASE") ?? string.Empty, + Get(values, "CARBONOPS_PARSER_POSTGRES_USERNAME") ?? string.Empty, + Get(values, "CARBONOPS_PARSER_POSTGRES_PASSWORD") ?? string.Empty, + Get(values, "CARBONOPS_PARSER_POSTGRES_SCHEMA") ?? string.Empty); + + collectedIssues.AddRange(Validate(settings).Issues); + issues = Array.AsReadOnly(collectedIssues.ToArray()); + return collectedIssues.Count == 0; + } + + public static IReadOnlyDictionary BuildSafeDiagnostics( + PostgreSQLRuntimeConnectionSettings settings) => + new Dictionary(StringComparer.Ordinal) + { + ["postgresql_host"] = settings.Host, + ["postgresql_port"] = settings.Port.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["postgresql_database"] = settings.Database, + ["postgresql_username"] = settings.Username, + ["postgresql_schema"] = settings.Schema, + ["postgresql_password_set"] = (!string.IsNullOrWhiteSpace(settings.Password)).ToString(), + ["postgresql_password"] = "[redacted]", + ["postgresql_connection_string"] = "[redacted]", + }; + + public static string BuildConnectionString(PostgreSQLRuntimeConnectionSettings settings) + { + var validation = Validate(settings); + if (!validation.IsValid) + { + throw new ArgumentException("PostgreSQL runtime settings are invalid.", nameof(settings)); + } + + var builder = new NpgsqlConnectionStringBuilder + { + Host = settings.Host, + Port = settings.Port, + Database = settings.Database, + Username = settings.Username, + Password = settings.Password, + SearchPath = settings.Schema, + ApplicationName = settings.ApplicationName, + Timeout = settings.ConnectTimeoutSeconds, + IncludeErrorDetail = false, + }; + + return builder.ConnectionString; + } + + internal static string RenderIdentifier(string identifier, string fieldName) + { + if (!IsValidIdentifier(identifier)) + { + throw new ArgumentException($"{fieldName} must be a PostgreSQL-safe identifier.", fieldName); + } + + return identifier; + } + + private static void ValidateIdentifier( + string? value, + string fieldName, + ICollection issues) + { + if (!IsValidIdentifier(value)) + { + issues.Add(new PostgreSQLRuntimeConnectionSettingsValidationIssue( + "POSTGRESQL_RUNTIME_INVALID_IDENTIFIER", + $"{fieldName} must contain only lowercase letters, digits, and underscores, and must start with a letter.", + fieldName)); + } + } + + private static bool IsValidIdentifier(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + if (value.Length > 63 || !char.IsAsciiLetterLower(value[0])) + { + return false; + } + + return value.All(character => + char.IsAsciiLetterLower(character) || + char.IsAsciiDigit(character) || + character == '_'); + } + + private static string? Get(IReadOnlyDictionary values, string key) => + values.TryGetValue(key, out var value) ? value : null; +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeSchemaBootstrap.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeSchemaBootstrap.cs new file mode 100644 index 00000000..79f1224c --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeSchemaBootstrap.cs @@ -0,0 +1,190 @@ +using Npgsql; + +namespace CarbonOps.Parser.Contracts; + +public sealed record PostgreSQLRuntimeSchemaBootstrapExecutionResult( + IReadOnlyList RequiredTableNames, + IReadOnlyList PresentTableNames, + IReadOnlyList MissingTableNames, + IReadOnlyList CreatedTableNames, + int StatementCount); + +public static class PostgreSQLRuntimeSchemaDdl +{ + public static IReadOnlyList RenderIdempotentSchemaStatements() => + Array.AsReadOnly( + PostgreSQLRuntimeSchemaCatalog.Tables + .Select(RenderCreateTableStatement) + .Concat(PostgreSQLRuntimeSchemaCatalog.Tables.SelectMany(RenderIndexStatements)) + .ToArray()); + + public static IReadOnlyList DestructiveSqlTokens { get; } = Array.AsReadOnly( + new[] + { + "DROP ", + "TRUNCATE ", + "DELETE ", + "ALTER TABLE ", + "RENAME ", + }); + + public static bool ContainsDestructiveSql(string statement) => + DestructiveSqlTokens.Any(token => statement.Contains(token, StringComparison.OrdinalIgnoreCase)); + + private static string RenderCreateTableStatement(PostgreSQLRuntimeTable table) + { + var lines = new List(); + var primaryKeyColumns = new List(); + + foreach (var column in table.Columns) + { + var columnSql = $"{column.Name} {RenderDataType(column.DataType)}"; + if (!column.Nullable) + { + columnSql += " NOT NULL"; + } + + lines.Add(columnSql); + if (column.IsPrimaryKey) + { + primaryKeyColumns.Add(column.Name); + } + } + + if (primaryKeyColumns.Count > 0) + { + lines.Add($"CONSTRAINT pk_{table.Name} PRIMARY KEY ({string.Join(", ", primaryKeyColumns)})"); + } + + foreach (var uniqueConstraint in table.UniqueConstraints ?? []) + { + lines.Add( + $"CONSTRAINT {uniqueConstraint.Name} UNIQUE ({string.Join(", ", uniqueConstraint.ColumnNames)})"); + } + + foreach (var foreignKey in table.ForeignKeys ?? []) + { + lines.Add( + $"CONSTRAINT fk_{table.Name}_{foreignKey.ColumnName} FOREIGN KEY ({foreignKey.ColumnName}) " + + $"REFERENCES {foreignKey.ReferencedTableName} ({foreignKey.ReferencedColumnName})"); + } + + return $"CREATE TABLE IF NOT EXISTS {table.Name} (\n {string.Join(",\n ", lines)}\n);"; + } + + private static IEnumerable RenderIndexStatements(PostgreSQLRuntimeTable table) + { + foreach (var index in table.Indexes ?? []) + { + var uniquePrefix = index.Unique ? "UNIQUE " : string.Empty; + yield return + $"CREATE {uniquePrefix}INDEX IF NOT EXISTS {index.Name} " + + $"ON {table.Name} ({string.Join(", ", index.ColumnNames)});"; + } + } + + private static string RenderDataType(PostgreSQLRuntimeColumnType dataType) => + dataType switch + { + PostgreSQLRuntimeColumnType.Uuid => "uuid", + PostgreSQLRuntimeColumnType.Text => "text", + PostgreSQLRuntimeColumnType.Integer => "integer", + PostgreSQLRuntimeColumnType.Numeric => "numeric", + PostgreSQLRuntimeColumnType.TimestampWithTimeZone => "timestamp with time zone", + PostgreSQLRuntimeColumnType.Jsonb => "jsonb", + _ => throw new ArgumentOutOfRangeException(nameof(dataType), dataType, null), + }; +} + +public sealed class PostgreSQLRuntimeSchemaBootstrapper +{ + public async Task BootstrapAsync( + PostgreSQLRuntimeConnectionSettings settings, + CancellationToken cancellationToken = default) + { + var validation = PostgreSQLRuntimeConnectionBoundary.Validate(settings); + if (!validation.IsValid) + { + throw new ArgumentException("PostgreSQL runtime settings are invalid.", nameof(settings)); + } + + var statements = PostgreSQLRuntimeSchemaDdl.RenderIdempotentSchemaStatements(); + if (statements.Any(PostgreSQLRuntimeSchemaDdl.ContainsDestructiveSql)) + { + throw new InvalidOperationException("Schema bootstrap contains destructive SQL."); + } + + await using var dataSource = NpgsqlDataSource.Create( + PostgreSQLRuntimeConnectionBoundary.BuildConnectionString(settings)); + await using var connection = await dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + + var requiredTableNames = PostgreSQLRuntimeSchemaCatalog.RequiredTableNames; + var schema = PostgreSQLRuntimeConnectionBoundary.RenderIdentifier(settings.Schema, "schema"); + await using (var schemaCommand = connection.CreateCommand()) + { + schemaCommand.CommandText = $"CREATE SCHEMA IF NOT EXISTS {schema};"; + await schemaCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + await using (var searchPathCommand = connection.CreateCommand()) + { + searchPathCommand.CommandText = $"SET search_path TO {schema};"; + await searchPathCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + var presentBefore = await FetchPresentTableNamesAsync(connection, requiredTableNames, cancellationToken) + .ConfigureAwait(false); + + foreach (var statement in statements) + { + await using var command = connection.CreateCommand(); + command.CommandText = statement; + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + var presentAfter = await FetchPresentTableNamesAsync(connection, requiredTableNames, cancellationToken) + .ConfigureAwait(false); + var created = requiredTableNames + .Where(tableName => presentAfter.Contains(tableName, StringComparer.Ordinal) && + !presentBefore.Contains(tableName, StringComparer.Ordinal)) + .ToArray(); + var missing = requiredTableNames + .Where(tableName => !presentAfter.Contains(tableName, StringComparer.Ordinal)) + .ToArray(); + + return new PostgreSQLRuntimeSchemaBootstrapExecutionResult( + requiredTableNames, + Array.AsReadOnly(presentAfter.ToArray()), + Array.AsReadOnly(missing), + Array.AsReadOnly(created), + statements.Count + 2); + } + + private static async Task> FetchPresentTableNamesAsync( + NpgsqlConnection connection, + IReadOnlyList requiredTableNames, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name = ANY($1) + ORDER BY table_name + """; + command.Parameters.AddWithValue(requiredTableNames.ToArray()); + + var present = new HashSet(StringComparer.Ordinal); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + present.Add(reader.GetString(0)); + } + + return Array.AsReadOnly( + requiredTableNames + .Where(tableName => present.Contains(tableName)) + .ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeSchemaCatalog.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeSchemaCatalog.cs new file mode 100644 index 00000000..078c06e1 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLRuntimeSchemaCatalog.cs @@ -0,0 +1,245 @@ +namespace CarbonOps.Parser.Contracts; + +public enum PostgreSQLRuntimeColumnType +{ + Uuid = 0, + Text = 1, + Integer = 2, + Numeric = 3, + TimestampWithTimeZone = 4, + Jsonb = 5, +} + +public sealed record PostgreSQLRuntimeColumn( + string Name, + PostgreSQLRuntimeColumnType DataType, + bool Nullable, + bool IsPrimaryKey = false); + +public sealed record PostgreSQLRuntimeForeignKey( + string ColumnName, + string ReferencedTableName, + string ReferencedColumnName); + +public sealed record PostgreSQLRuntimeUniqueConstraint( + string Name, + IReadOnlyList ColumnNames); + +public sealed record PostgreSQLRuntimeIndex( + string Name, + IReadOnlyList ColumnNames, + bool Unique = false); + +public sealed record PostgreSQLRuntimeTable( + string Name, + IReadOnlyList Columns, + IReadOnlyList? ForeignKeys = null, + IReadOnlyList? UniqueConstraints = null, + IReadOnlyList? Indexes = null); + +public static class PostgreSQLRuntimeSchemaCatalog +{ + public static IReadOnlyList Tables { get; } = Array.AsReadOnly( + SharedTables() + .Concat(SourceFamilyTables("ghg")) + .Concat(SourceFamilyTables("defra")) + .Concat(SourceFamilyTables("ipcc")) + .ToArray()); + + public static IReadOnlyList RequiredTableNames { get; } = Array.AsReadOnly( + Tables + .Select(table => table.Name) + .Order(StringComparer.Ordinal) + .ToArray()); + + public static string ToPostgreSQLRuntimeValue(this SourceFamily sourceFamily) => + sourceFamily switch + { + SourceFamily.GhgProtocol => "ghg_protocol", + SourceFamily.DefraDesnz => "defra_desnz", + SourceFamily.IpccEfdb => "ipcc_efdb", + _ => throw new ArgumentOutOfRangeException(nameof(sourceFamily), sourceFamily, null), + }; + + private static IReadOnlyList SharedTables() => + [ + new( + "ingestion_runs", + [ + new("ingestion_run_id", PostgreSQLRuntimeColumnType.Uuid, Nullable: false, IsPrimaryKey: true), + new("run_status", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("created_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + new("updated_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + ], + Indexes: [new("idx_ingestion_runs_run_status", ["run_status"])]), + new( + "source_documents", + [ + new("source_document_id", PostgreSQLRuntimeColumnType.Uuid, Nullable: false, IsPrimaryKey: true), + new("ingestion_run_id", PostgreSQLRuntimeColumnType.Uuid, Nullable: false), + new("source_family", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("source_document_uri", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("source_checksum_sha256", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("acquisition_status", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("acquired_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: true), + new("created_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + new("updated_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + ], + ForeignKeys: [new("ingestion_run_id", "ingestion_runs", "ingestion_run_id")], + UniqueConstraints: + [ + new( + "uq_source_documents_family_uri_checksum", + ["source_family", "source_document_uri", "source_checksum_sha256"]), + ], + Indexes: [new("idx_source_documents_ingestion_run_id", ["ingestion_run_id"])]), + new( + "parser_runs", + [ + new("parser_run_id", PostgreSQLRuntimeColumnType.Uuid, Nullable: false, IsPrimaryKey: true), + new("source_document_id", PostgreSQLRuntimeColumnType.Uuid, Nullable: false), + new("parser_status", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("error_details", PostgreSQLRuntimeColumnType.Jsonb, Nullable: true), + new("created_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + new("updated_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + ], + ForeignKeys: [new("source_document_id", "source_documents", "source_document_id")], + Indexes: [new("idx_parser_runs_source_document_id", ["source_document_id"])]), + new( + "schema_bootstrap_states", + [ + new("schema_bootstrap_state_id", PostgreSQLRuntimeColumnType.Uuid, Nullable: false, IsPrimaryKey: true), + new("schema_contract_version", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("bootstrap_status", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("created_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + new("updated_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + ], + UniqueConstraints: + [ + new("uq_schema_bootstrap_states_contract_version", ["schema_contract_version"]), + ]), + new( + "source_family_year_states", + [ + new("source_family_year_state_id", PostgreSQLRuntimeColumnType.Uuid, Nullable: false, IsPrimaryKey: true), + new("source_family", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("ingested_year", PostgreSQLRuntimeColumnType.Integer, Nullable: false), + new("created_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + new("updated_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + ], + UniqueConstraints: + [ + new("uq_source_family_year_states_family_year", ["source_family", "ingested_year"]), + ], + Indexes: [new("idx_source_family_year_states_family_year", ["source_family", "ingested_year"])]), + new( + "normalized_factor_records", + [ + new("normalized_factor_record_id", PostgreSQLRuntimeColumnType.Text, Nullable: false, IsPrimaryKey: true), + new("idempotency_key_sha256", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("source_family", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("source_id", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("source_year", PostgreSQLRuntimeColumnType.Integer, Nullable: true), + new("source_version", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("record_id", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("source_row_number", PostgreSQLRuntimeColumnType.Integer, Nullable: true), + new("source_document_reference", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("source_artifact_reference", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("source_checksum_sha256", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("factor_id", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("factor_name", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("factor_value", PostgreSQLRuntimeColumnType.Numeric, Nullable: false), + new("factor_unit", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("validation_status", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("run_id", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("parser_key", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("metadata", PostgreSQLRuntimeColumnType.Jsonb, Nullable: false), + new("normalized_fields", PostgreSQLRuntimeColumnType.Jsonb, Nullable: false), + new("warnings", PostgreSQLRuntimeColumnType.Jsonb, Nullable: false), + new("errors", PostgreSQLRuntimeColumnType.Jsonb, Nullable: false), + new("created_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + new("updated_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + ], + UniqueConstraints: + [ + new("uq_normalized_factor_records_idempotency_key", ["idempotency_key_sha256"]), + ], + Indexes: [new("idx_normalized_factor_records_source_year", ["source_family", "source_id", "source_year"])]), + ]; + + private static IReadOnlyList SourceFamilyTables(string family) + { + var masterTable = $"{family}_emission_factor_masters"; + var detailTable = $"{family}_emission_factor_details"; + var masterId = $"{family}_emission_factor_master_id"; + var detailId = $"{family}_emission_factor_detail_id"; + + return + [ + new( + masterTable, + [ + new(masterId, PostgreSQLRuntimeColumnType.Uuid, Nullable: false, IsPrimaryKey: true), + new("source_family", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("source_year", PostgreSQLRuntimeColumnType.Integer, Nullable: false), + new("source_version", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("source_release", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("source_document_id", PostgreSQLRuntimeColumnType.Uuid, Nullable: false), + new("ingestion_run_id", PostgreSQLRuntimeColumnType.Uuid, Nullable: true), + new("run_id", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("master_external_key", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("status", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("artifact_reference", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("artifact_checksum_sha256", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("archive_reference", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("archive_checksum_sha256", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("effective_from", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: true), + new("effective_to", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: true), + new("record_checksum_sha256", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("metadata", PostgreSQLRuntimeColumnType.Jsonb, Nullable: false), + new("created_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + new("updated_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + ], + ForeignKeys: + [ + new("source_document_id", "source_documents", "source_document_id"), + new("ingestion_run_id", "ingestion_runs", "ingestion_run_id"), + ], + UniqueConstraints: + [ + new( + $"uq_{masterTable}_family_year_version_key", + ["source_family", "source_year", "source_version", "master_external_key"]), + ], + Indexes: + [ + new($"idx_{masterTable}_source_year", ["source_family", "source_year", "source_version"]), + new($"idx_{masterTable}_ingestion_run_id", ["ingestion_run_id"]), + ]), + new( + detailTable, + [ + new(detailId, PostgreSQLRuntimeColumnType.Uuid, Nullable: false, IsPrimaryKey: true), + new(masterId, PostgreSQLRuntimeColumnType.Uuid, Nullable: false), + new("detail_external_key", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("source_row_number", PostgreSQLRuntimeColumnType.Integer, Nullable: true), + new("factor_id", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("factor_name", PostgreSQLRuntimeColumnType.Text, Nullable: true), + new("factor_value", PostgreSQLRuntimeColumnType.Numeric, Nullable: false), + new("factor_unit", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("status", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("record_checksum_sha256", PostgreSQLRuntimeColumnType.Text, Nullable: false), + new("raw_fields", PostgreSQLRuntimeColumnType.Jsonb, Nullable: false), + new("normalized_fields", PostgreSQLRuntimeColumnType.Jsonb, Nullable: false), + new("created_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + new("updated_at", PostgreSQLRuntimeColumnType.TimestampWithTimeZone, Nullable: false), + ], + ForeignKeys: [new(masterId, masterTable, masterId)], + UniqueConstraints: + [ + new($"uq_{detailTable}_master_detail_external_key", [masterId, "detail_external_key"]), + ], + Indexes: [new($"idx_{detailTable}_{masterId}", [masterId])]), + ]; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLSchemaBootstrap.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLSchemaBootstrap.cs new file mode 100644 index 00000000..3ce01853 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLSchemaBootstrap.cs @@ -0,0 +1,214 @@ +namespace CarbonOps.Parser.Contracts; + +public enum PostgreSQLSchemaBootstrapMode +{ + CheckOnly = 0, + CreateMissing = 1, +} + +public enum PostgreSQLSchemaBootstrapTableStatus +{ + Required = 0, + Present = 1, + Missing = 2, + Created = 3, + Skipped = 4, +} + +public sealed record PostgreSQLSchemaBootstrapRequest +{ + public PostgreSQLSchemaBootstrapMode Mode { get; } + + public IReadOnlyList RequiredTableNames { get; } + + public bool FailOnMissing { get; } + + public PostgreSQLSchemaBootstrapRequest( + PostgreSQLSchemaBootstrapMode mode, + IEnumerable requiredTableNames, + bool failOnMissing = true) + { + Mode = mode; + RequiredTableNames = Array.AsReadOnly(requiredTableNames.ToArray()); + FailOnMissing = failOnMissing; + } +} + +public sealed record PostgreSQLSchemaBootstrapTableResult( + string TableName, + PostgreSQLSchemaBootstrapTableStatus Status, + string Reason = ""); + +public sealed record PostgreSQLSchemaBootstrapReport +{ + public PostgreSQLSchemaBootstrapMode Mode { get; } + + public IReadOnlyList RequiredTableNames { get; } + + public IReadOnlyList TableResults { get; } + + public bool FailOnMissing { get; } + + public bool NoExecution { get; } + + public bool OpensConnection { get; } + + public bool RunsSql { get; } + + public bool CreatesTablesNow { get; } + + public bool RunsMigrations { get; } + + public bool ReadsEnvironment { get; } + + public bool WritesFiles { get; } + + public bool PerformsNetworkCalls { get; } + + public IReadOnlyList MissingTableNames => Array.AsReadOnly( + TableResults + .Where(result => result.Status == PostgreSQLSchemaBootstrapTableStatus.Missing) + .Select(result => result.TableName) + .ToArray()); + + public IReadOnlyList PresentTableNames => Array.AsReadOnly( + TableResults + .Where(result => result.Status == PostgreSQLSchemaBootstrapTableStatus.Present) + .Select(result => result.TableName) + .ToArray()); + + public IReadOnlyList CreatedTableNames => Array.AsReadOnly( + TableResults + .Where(result => result.Status == PostgreSQLSchemaBootstrapTableStatus.Created) + .Select(result => result.TableName) + .ToArray()); + + public IReadOnlyList SkippedTableNames => Array.AsReadOnly( + TableResults + .Where(result => result.Status == PostgreSQLSchemaBootstrapTableStatus.Skipped) + .Select(result => result.TableName) + .ToArray()); + + public PostgreSQLSchemaBootstrapReport( + PostgreSQLSchemaBootstrapMode mode, + IEnumerable requiredTableNames, + IEnumerable tableResults, + bool failOnMissing, + bool noExecution, + bool opensConnection, + bool runsSql, + bool createsTablesNow, + bool runsMigrations, + bool readsEnvironment, + bool writesFiles, + bool performsNetworkCalls) + { + Mode = mode; + RequiredTableNames = Array.AsReadOnly(requiredTableNames.ToArray()); + TableResults = Array.AsReadOnly(tableResults.ToArray()); + FailOnMissing = failOnMissing; + NoExecution = noExecution; + OpensConnection = opensConnection; + RunsSql = runsSql; + CreatesTablesNow = createsTablesNow; + RunsMigrations = runsMigrations; + ReadsEnvironment = readsEnvironment; + WritesFiles = writesFiles; + PerformsNetworkCalls = performsNetworkCalls; + } +} + +public static class PostgreSQLSchemaBootstrapBoundary +{ + public static IReadOnlyList RequiredPhase1TableNames { get; } = Array.AsReadOnly( + PostgreSQLRuntimeSchemaCatalog.RequiredTableNames.ToArray()); + + public static PostgreSQLSchemaBootstrapRequest CreateRequest( + PostgreSQLSchemaBootstrapMode mode = PostgreSQLSchemaBootstrapMode.CheckOnly, + bool failOnMissing = true) => + new(mode, RequiredPhase1TableNames, failOnMissing); + + public static PostgreSQLSchemaBootstrapReport BuildReport( + PostgreSQLSchemaBootstrapMode mode = PostgreSQLSchemaBootstrapMode.CheckOnly, + IEnumerable? presentTableNames = null, + IEnumerable? createdTableNames = null, + IEnumerable? skippedTableNames = null, + bool failOnMissing = true) + { + var request = CreateRequest(mode, failOnMissing); + var presentTables = (presentTableNames ?? []).ToHashSet(StringComparer.Ordinal); + var createdTables = (createdTableNames ?? []).ToHashSet(StringComparer.Ordinal); + var skippedTables = (skippedTableNames ?? []).ToHashSet(StringComparer.Ordinal); + + var tableResults = request.RequiredTableNames.Select(tableName => new PostgreSQLSchemaBootstrapTableResult( + tableName, + ResolveTableStatus(tableName, request.Mode, presentTables, createdTables, skippedTables), + ResolveTableReason(tableName, request.Mode, presentTables, createdTables, skippedTables))); + + return new PostgreSQLSchemaBootstrapReport( + request.Mode, + request.RequiredTableNames, + tableResults, + request.FailOnMissing, + noExecution: true, + opensConnection: false, + runsSql: false, + createsTablesNow: false, + runsMigrations: false, + readsEnvironment: false, + writesFiles: false, + performsNetworkCalls: false); + } + + private static PostgreSQLSchemaBootstrapTableStatus ResolveTableStatus( + string tableName, + PostgreSQLSchemaBootstrapMode mode, + ISet presentTableNames, + ISet createdTableNames, + ISet skippedTableNames) + { + if (presentTableNames.Contains(tableName)) + { + return PostgreSQLSchemaBootstrapTableStatus.Present; + } + + if (mode == PostgreSQLSchemaBootstrapMode.CreateMissing && + createdTableNames.Contains(tableName)) + { + return PostgreSQLSchemaBootstrapTableStatus.Created; + } + + if (skippedTableNames.Contains(tableName)) + { + return PostgreSQLSchemaBootstrapTableStatus.Skipped; + } + + return PostgreSQLSchemaBootstrapTableStatus.Missing; + } + + private static string ResolveTableReason( + string tableName, + PostgreSQLSchemaBootstrapMode mode, + ISet presentTableNames, + ISet createdTableNames, + ISet skippedTableNames) + { + if (presentTableNames.Contains(tableName)) + { + return "Required table was reported present by caller metadata."; + } + + if (mode == PostgreSQLSchemaBootstrapMode.CreateMissing && + createdTableNames.Contains(tableName)) + { + return "Required table was reported created by caller metadata."; + } + + if (skippedTableNames.Contains(tableName)) + { + return "Required table was reported skipped by caller metadata."; + } + + return "Required table was not reported present or created."; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLSourceFamilyYearStateRepository.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLSourceFamilyYearStateRepository.cs new file mode 100644 index 00000000..7ce4606c --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLSourceFamilyYearStateRepository.cs @@ -0,0 +1,130 @@ +using Npgsql; + +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceFamilyYearState( + SourceFamily SourceFamily, + int? LatestYear, + int NextYear, + int InitialYear); + +public interface IPostgreSQLSourceFamilyYearStateSession +{ + Task LatestSuccessfulYearAsync(SourceFamily sourceFamily, CancellationToken cancellationToken = default); + + Task RecordSuccessfulYearAsync( + SourceFamily sourceFamily, + int ingestedYear, + CancellationToken cancellationToken = default); +} + +public sealed class PostgreSQLSourceFamilyYearStateRepository +{ + public const int DefaultInitialYear = 2024; + + private readonly IPostgreSQLSourceFamilyYearStateSession _session; + + public PostgreSQLSourceFamilyYearStateRepository( + IPostgreSQLSourceFamilyYearStateSession session, + int initialYear = DefaultInitialYear) + { + if (initialYear < 1) + { + throw new ArgumentOutOfRangeException(nameof(initialYear), initialYear, "initialYear must be positive."); + } + + _session = session; + InitialYear = initialYear; + } + + public string ProviderName => "postgresql"; + + public int InitialYear { get; } + + public async Task LatestSuccessfulYearAsync( + SourceFamily sourceFamily, + CancellationToken cancellationToken = default) => + await _session.LatestSuccessfulYearAsync(sourceFamily, cancellationToken).ConfigureAwait(false); + + public async Task NextTargetYearAsync( + SourceFamily sourceFamily, + CancellationToken cancellationToken = default) + { + var latestYear = await LatestSuccessfulYearAsync(sourceFamily, cancellationToken).ConfigureAwait(false); + return latestYear is null ? InitialYear : latestYear.Value + 1; + } + + public async Task GetYearStateAsync( + SourceFamily sourceFamily, + CancellationToken cancellationToken = default) + { + var latestYear = await LatestSuccessfulYearAsync(sourceFamily, cancellationToken).ConfigureAwait(false); + return new SourceFamilyYearState( + sourceFamily, + latestYear, + latestYear is null ? InitialYear : latestYear.Value + 1, + InitialYear); + } + + public async Task RecordSuccessfulYearAsync( + SourceFamily sourceFamily, + int ingestedYear, + CancellationToken cancellationToken = default) + { + if (ingestedYear < 1) + { + throw new ArgumentOutOfRangeException(nameof(ingestedYear), ingestedYear, "ingestedYear must be positive."); + } + + await _session.RecordSuccessfulYearAsync(sourceFamily, ingestedYear, cancellationToken).ConfigureAwait(false); + } +} + +public sealed class NpgsqlSourceFamilyYearStateSession : IPostgreSQLSourceFamilyYearStateSession +{ + private readonly NpgsqlDataSource _dataSource; + + public NpgsqlSourceFamilyYearStateSession(NpgsqlDataSource dataSource) + { + _dataSource = dataSource; + } + + public async Task LatestSuccessfulYearAsync( + SourceFamily sourceFamily, + CancellationToken cancellationToken = default) + { + await using var command = _dataSource.CreateCommand(""" + SELECT MAX(ingested_year) + FROM source_family_year_states + WHERE source_family = $1 + """); + command.Parameters.AddWithValue(sourceFamily.ToPostgreSQLRuntimeValue()); + + var value = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return value is null or DBNull ? null : Convert.ToInt32(value, System.Globalization.CultureInfo.InvariantCulture); + } + + public async Task RecordSuccessfulYearAsync( + SourceFamily sourceFamily, + int ingestedYear, + CancellationToken cancellationToken = default) + { + await using var command = _dataSource.CreateCommand(""" + INSERT INTO source_family_year_states ( + source_family_year_state_id, + source_family, + ingested_year, + created_at, + updated_at + ) + VALUES ($1, $2, $3, NOW(), NOW()) + ON CONFLICT (source_family, ingested_year) + DO UPDATE SET updated_at = EXCLUDED.updated_at + """); + command.Parameters.AddWithValue(Guid.NewGuid()); + command.Parameters.AddWithValue(sourceFamily.ToPostgreSQLRuntimeValue()); + command.Parameters.AddWithValue(ingestedYear); + + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLSourceSpecificFactorPersistence.cs b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLSourceSpecificFactorPersistence.cs new file mode 100644 index 00000000..e8eb7c6d --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/PostgreSQLSourceSpecificFactorPersistence.cs @@ -0,0 +1,800 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Npgsql; + +namespace CarbonOps.Parser.Contracts; + +public enum PostgreSQLSourceSpecificFactorPersistenceStatus +{ + Inserted = 0, + NoRecords = 1, + FailedValidation = 2, + FailedDatabase = 3, +} + +public sealed record PostgreSQLSourceSpecificFactorPersistenceIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); + +public sealed record PostgreSQLSourceSpecificFactorPersistenceCounts( + int MasterInserted, + int MasterSkippedDuplicate, + int DetailInserted, + int DetailSkippedDuplicate, + int ValidationFailed); + +public sealed record PostgreSQLSourceSpecificFactorPersistenceResult( + string ProviderName, + PostgreSQLSourceSpecificFactorPersistenceStatus Status, + PostgreSQLSourceSpecificFactorPersistenceCounts Counts, + IReadOnlyList Issues) +{ + public int MasterInserted => Counts.MasterInserted; + + public int MasterSkippedDuplicate => Counts.MasterSkippedDuplicate; + + public int DetailInserted => Counts.DetailInserted; + + public int DetailSkippedDuplicate => Counts.DetailSkippedDuplicate; + + public int ValidationFailed => Counts.ValidationFailed; +} + +public sealed record PostgreSQLSourceSpecificMasterRecord( + SourceFamily SourceFamily, + Guid SourceFamilyMasterId, + int SourceYear, + string SourceVersion, + string? SourceRelease, + Guid SourceDocumentId, + Guid? IngestionRunId, + string? RunId, + string MasterExternalKey, + string Status, + string? ArtifactReference, + string? ArtifactChecksumSha256, + string? ArchiveReference, + string? ArchiveChecksumSha256, + string? EffectiveFrom, + string? EffectiveTo, + string RecordChecksumSha256, + IReadOnlyDictionary Metadata); + +public sealed record PostgreSQLSourceSpecificDetailRecord( + SourceFamily SourceFamily, + Guid SourceFamilyDetailId, + Guid SourceFamilyMasterId, + string DetailExternalKey, + int? SourceRowNumber, + string? FactorId, + string? FactorName, + decimal FactorValue, + string FactorUnit, + string Status, + string RecordChecksumSha256, + IReadOnlyDictionary RawFields, + IReadOnlyDictionary NormalizedFields); + +public sealed record PostgreSQLSourceSpecificFactorPersistenceBatch( + SourceFamily SourceFamily, + int SourceYear, + IReadOnlyList MasterRecords, + IReadOnlyList DetailRecords); + +public interface IPostgreSQLSourceSpecificFactorPersistenceSession +{ + string ProviderName { get; } + + Task PersistSourceFamilyYearAsync( + PostgreSQLSourceSpecificFactorPersistenceBatch batch, + CancellationToken cancellationToken = default); +} + +public sealed class PostgreSQLSourceSpecificFactorPersistenceRepository +{ + private readonly IPostgreSQLSourceSpecificFactorPersistenceSession _session; + + public PostgreSQLSourceSpecificFactorPersistenceRepository( + IPostgreSQLSourceSpecificFactorPersistenceSession session) + { + _session = session ?? throw new ArgumentNullException(nameof(session)); + } + + public async Task PersistAsync( + ParserNormalizedOutputBatch parsedOutput, + CancellationToken cancellationToken = default) + { + var mapped = PostgreSQLSourceSpecificFactorPersistenceMapper.Map(parsedOutput); + if (mapped.Issues.Count > 0) + { + var noRecords = mapped.Issues.Count == 1 && + mapped.Issues[0].Code == "POSTGRESQL_SOURCE_SPECIFIC_NO_RECORDS"; + return new PostgreSQLSourceSpecificFactorPersistenceResult( + _session.ProviderName, + noRecords + ? PostgreSQLSourceSpecificFactorPersistenceStatus.NoRecords + : PostgreSQLSourceSpecificFactorPersistenceStatus.FailedValidation, + new PostgreSQLSourceSpecificFactorPersistenceCounts(0, 0, 0, 0, mapped.Issues.Count), + mapped.Issues); + } + + var total = new PostgreSQLSourceSpecificFactorPersistenceCounts(0, 0, 0, 0, 0); + foreach (var batch in mapped.Batches) + { + try + { + var counts = await _session.PersistSourceFamilyYearAsync(batch, cancellationToken) + .ConfigureAwait(false); + total = total with + { + MasterInserted = total.MasterInserted + counts.MasterInserted, + MasterSkippedDuplicate = total.MasterSkippedDuplicate + counts.MasterSkippedDuplicate, + DetailInserted = total.DetailInserted + counts.DetailInserted, + DetailSkippedDuplicate = total.DetailSkippedDuplicate + counts.DetailSkippedDuplicate, + ValidationFailed = total.ValidationFailed + counts.ValidationFailed, + }; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return new PostgreSQLSourceSpecificFactorPersistenceResult( + _session.ProviderName, + PostgreSQLSourceSpecificFactorPersistenceStatus.FailedDatabase, + total, + [ + new( + "POSTGRESQL_SOURCE_SPECIFIC_DATABASE_ERROR", + RedactSensitiveText(ex.Message), + "database"), + ]); + } + } + + return new PostgreSQLSourceSpecificFactorPersistenceResult( + _session.ProviderName, + PostgreSQLSourceSpecificFactorPersistenceStatus.Inserted, + total, + []); + } + private static string RedactSensitiveText(string value) + { + var withoutUriSecret = System.Text.RegularExpressions.Regex.Replace( + value, + @"postgresql://[^@\s]+@", + "postgresql://***@", + System.Text.RegularExpressions.RegexOptions.IgnoreCase); + return System.Text.RegularExpressions.Regex.Replace( + withoutUriSecret, + @"password=([^;\s]+)", + "password=***", + System.Text.RegularExpressions.RegexOptions.IgnoreCase); + } +} + +public static class PostgreSQLSourceSpecificFactorPersistenceMapper +{ + public static PostgreSQLSourceSpecificFactorPersistenceMapResult Map(ParserNormalizedOutputBatch? parsedOutput) + { + if (parsedOutput is null) + { + return new PostgreSQLSourceSpecificFactorPersistenceMapResult( + [], + [ + new( + "POSTGRESQL_SOURCE_SPECIFIC_INVALID_OUTPUT", + "parsed output must be a ParserNormalizedOutputBatch.", + "parsedOutput"), + ]); + } + + if (parsedOutput.RowCount == 0) + { + return new PostgreSQLSourceSpecificFactorPersistenceMapResult( + [], + [ + new( + "POSTGRESQL_SOURCE_SPECIFIC_NO_RECORDS", + "parsed output must include records before source-specific persistence.", + "Rows", + "warning"), + ]); + } + + var issues = new List(); + var masters = new Dictionary<(SourceFamily SourceFamily, int SourceYear, string SourceVersion, string MasterKey), + PostgreSQLSourceSpecificMasterRecord>(); + var details = new Dictionary<(SourceFamily SourceFamily, Guid MasterId, string DetailKey), + PostgreSQLSourceSpecificDetailRecord>(); + + for (var index = 0; index < parsedOutput.Rows.Count; index++) + { + var mapped = MapRow(parsedOutput.Rows[index], index); + issues.AddRange(mapped.Issues); + if (mapped.MasterRecord is null || mapped.DetailRecord is null) + { + continue; + } + + var masterKey = ( + mapped.MasterRecord.SourceFamily, + mapped.MasterRecord.SourceYear, + mapped.MasterRecord.SourceVersion, + mapped.MasterRecord.MasterExternalKey); + if (!masters.TryAdd(masterKey, mapped.MasterRecord) && + masters[masterKey] != mapped.MasterRecord) + { + issues.Add(new( + "POSTGRESQL_SOURCE_SPECIFIC_DUPLICATE_MASTER_CONFLICT", + "duplicate source-specific master identity maps to different record content.", + $"Rows[{index}].master_external_key")); + } + + var detailKey = ( + mapped.DetailRecord.SourceFamily, + mapped.DetailRecord.SourceFamilyMasterId, + mapped.DetailRecord.DetailExternalKey); + if (!details.TryAdd(detailKey, mapped.DetailRecord) && + details[detailKey] != mapped.DetailRecord) + { + issues.Add(new( + "POSTGRESQL_SOURCE_SPECIFIC_DUPLICATE_DETAIL_CONFLICT", + "duplicate source-specific detail identity maps to different record content.", + $"Rows[{index}].detail_external_key")); + } + } + + if (issues.Count > 0) + { + return new PostgreSQLSourceSpecificFactorPersistenceMapResult([], issues); + } + + var batches = masters.Values + .GroupBy(master => (master.SourceFamily, master.SourceYear)) + .Select(group => + { + var masterIds = group.Select(master => master.SourceFamilyMasterId).ToHashSet(); + var batchDetails = details.Values + .Where(detail => detail.SourceFamily == group.Key.SourceFamily && + masterIds.Contains(detail.SourceFamilyMasterId)) + .OrderBy(detail => detail.DetailExternalKey, StringComparer.Ordinal) + .ToArray(); + return new PostgreSQLSourceSpecificFactorPersistenceBatch( + group.Key.SourceFamily, + group.Key.SourceYear, + Array.AsReadOnly(group.OrderBy(master => master.MasterExternalKey, StringComparer.Ordinal).ToArray()), + Array.AsReadOnly(batchDetails)); + }) + .OrderBy(batch => batch.SourceFamily.ToWireName(), StringComparer.Ordinal) + .ThenBy(batch => batch.SourceYear) + .ToArray(); + + return new PostgreSQLSourceSpecificFactorPersistenceMapResult(Array.AsReadOnly(batches), []); + } + + private static MappedRow MapRow(ParserNormalizedOutputRow? row, int index) + { + if (row is null) + { + return new MappedRow( + null, + null, + [new("POSTGRESQL_SOURCE_SPECIFIC_INVALID_ROW", "ParserNormalizedOutputRow is required.", $"Rows[{index}]")]); + } + + var fields = row.Fields + .GroupBy(field => field.Key, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.First().Value, StringComparer.Ordinal); + var issues = new List(); + foreach (var error in row.Validate().Errors) + { + issues.Add(new( + "POSTGRESQL_SOURCE_SPECIFIC_INVALID_ROW", + error, + $"Rows[{index}]")); + } + + var sourceYear = TryParsePositiveInt(TextOrNull(Field(fields, "source_year")) ?? row.ReportingYear?.ToString(CultureInfo.InvariantCulture)); + var sourceVersion = TextOrNull(Field(fields, "source_version")) ?? "version-unavailable"; + var factorValueText = TextOrNull(Field(fields, "factor_value", "value")); + var factorUnit = TextOrNull(Field(fields, "factor_unit", "unit")); + var factorValue = TryParseDecimal(factorValueText); + + if (sourceYear is null) + { + issues.Add(new("POSTGRESQL_SOURCE_SPECIFIC_MISSING_SOURCE_YEAR", "source_year must be a positive integer.", $"Rows[{index}].source_year")); + } + + if (factorValue is null) + { + issues.Add(new("POSTGRESQL_SOURCE_SPECIFIC_INVALID_FACTOR_VALUE", "factor_value must be a decimal value.", $"Rows[{index}].factor_value")); + } + + if (factorUnit is null) + { + issues.Add(new("POSTGRESQL_SOURCE_SPECIFIC_MISSING_FACTOR_UNIT", "factor_unit must be a non-empty value.", $"Rows[{index}].factor_unit")); + } + + if (issues.Count > 0) + { + return new MappedRow(null, null, issues); + } + + var resolvedSourceYear = sourceYear.GetValueOrDefault(); + var resolvedFactorValue = factorValue.GetValueOrDefault(); + var masterExternalKey = TextOrNull(Field(fields, "master_external_key")) ?? + $"{resolvedSourceYear}:{sourceVersion}:{TextOrNull(Field(fields, "factor_id")) ?? row.RowIdentifier}"; + var detailExternalKey = TextOrNull(Field(fields, "detail_external_key")) ?? + $"{TextOrNull(Field(fields, "factor_id")) ?? row.RowIdentifier}:{factorUnit}"; + var sourceDocumentKey = TextOrNull(Field(fields, "source_document_id")) ?? + StableDigest("source_document", row.SourceFamily.ToWireName(), row.SourceKey, row.ArtifactReference, Field(fields, "provenance_checksum_value")); + var runId = TextOrNull(Field(fields, "run_id")); + var artifactReference = TextOrNull(Field(fields, "provenance_artifact_reference", "artifact_reference")) ?? + TextOrNull(row.ArtifactReference); + var artifactChecksum = TextOrNull(Field(fields, "provenance_checksum_value", "source_checksum_sha256")); + var normalizedFields = NormalizeFields(fields); + var masterMetadata = new Dictionary(StringComparer.Ordinal) + { + ["source_key"] = row.SourceKey, + ["parser_key"] = row.ParserKey.Value, + ["row_identifier"] = row.RowIdentifier, + ["source_release"] = TextOrNull(Field(fields, "source_release")), + ["fields"] = normalizedFields, + }; + + var sourceFamilyMasterId = StableUuid( + "master", + row.SourceFamily.ToWireName(), + resolvedSourceYear.ToString(CultureInfo.InvariantCulture), + sourceVersion, + masterExternalKey); + var sourceFamilyDetailId = StableUuid( + "detail", + row.SourceFamily.ToWireName(), + sourceFamilyMasterId.ToString("D"), + detailExternalKey); + var master = new PostgreSQLSourceSpecificMasterRecord( + row.SourceFamily, + sourceFamilyMasterId, + resolvedSourceYear, + sourceVersion, + TextOrNull(Field(fields, "source_release")), + StableUuid("source_document", row.SourceFamily.ToWireName(), sourceDocumentKey), + StableUuid("ingestion_run", row.SourceFamily.ToWireName(), runId ?? $"{resolvedSourceYear}:{sourceVersion}"), + runId, + masterExternalKey, + TextOrNull(Field(fields, "status", "validation_status")) ?? "active", + artifactReference, + artifactChecksum, + TextOrNull(Field(fields, "archive_reference")), + TextOrNull(Field(fields, "archive_checksum_sha256")), + TextOrNull(Field(fields, "effective_from")), + TextOrNull(Field(fields, "effective_to")), + StableDigest("master", row.SourceFamily.ToWireName(), resolvedSourceYear.ToString(CultureInfo.InvariantCulture), sourceVersion, masterExternalKey), + masterMetadata); + var detail = new PostgreSQLSourceSpecificDetailRecord( + row.SourceFamily, + sourceFamilyDetailId, + sourceFamilyMasterId, + detailExternalKey, + row.SourceRowNumber ?? TryParsePositiveInt(TextOrNull(Field(fields, "provenance_row_number"))), + TextOrNull(Field(fields, "factor_id")), + TextOrNull(Field(fields, "factor_name")), + resolvedFactorValue, + factorUnit!, + TextOrNull(Field(fields, "status", "validation_status")) ?? "active", + StableDigest("detail", row.SourceFamily.ToWireName(), sourceFamilyMasterId.ToString("D"), detailExternalKey, factorValueText, factorUnit), + normalizedFields, + normalizedFields); + + return new MappedRow(master, detail, []); + } + + private static IReadOnlyDictionary NormalizeFields(IReadOnlyDictionary fields) => + fields + .OrderBy(pair => pair.Key, StringComparer.Ordinal) + .ToDictionary(pair => pair.Key, pair => (object?)pair.Value, StringComparer.Ordinal); + + private static string? Field(IReadOnlyDictionary fields, params string[] names) + { + foreach (var name in names) + { + if (fields.TryGetValue(name, out var value)) + { + return value; + } + } + + return null; + } + + private static string? TextOrNull(string? value) + { + var text = value?.Trim(); + return string.IsNullOrEmpty(text) ? null : text; + } + + private static int? TryParsePositiveInt(string? value) => + int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var parsed) && parsed > 0 + ? parsed + : null; + + private static decimal? TryParseDecimal(string? value) => + decimal.TryParse(value, NumberStyles.Number, CultureInfo.InvariantCulture, out var parsed) + ? parsed + : null; + + private static Guid StableUuid(params string?[] values) + { + var namespaceBytes = Guid.Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8").ToByteArray(); + SwapGuidByteOrder(namespaceBytes); + var nameBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(values)); + var payload = namespaceBytes.Concat(nameBytes).ToArray(); + var hash = SHA1.HashData(payload); + var guidBytes = hash[..16]; + guidBytes[6] = (byte)((guidBytes[6] & 0x0f) | 0x50); + guidBytes[8] = (byte)((guidBytes[8] & 0x3f) | 0x80); + SwapGuidByteOrder(guidBytes); + return new Guid(guidBytes); + } + + private static void SwapGuidByteOrder(byte[] bytes) + { + Array.Reverse(bytes, 0, 4); + Array.Reverse(bytes, 4, 2); + Array.Reverse(bytes, 6, 2); + } + + private static string StableDigest(params string?[] values) => + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(values)))).ToLowerInvariant(); + + private sealed record MappedRow( + PostgreSQLSourceSpecificMasterRecord? MasterRecord, + PostgreSQLSourceSpecificDetailRecord? DetailRecord, + IEnumerable Issues); +} + +public sealed record PostgreSQLSourceSpecificFactorPersistenceMapResult( + IReadOnlyList Batches, + IReadOnlyList Issues); + +public sealed record PostgreSQLSourceSpecificFactorPersistenceSqlStep( + string Name, + string CommandText); + +public sealed record PostgreSQLSourceSpecificFactorPersistenceTransactionBoundary( + bool OpensTransaction, + bool CommitsTransaction, + bool RollsBackOnFailure); + +public sealed class NpgsqlSourceSpecificFactorPersistenceSession : IPostgreSQLSourceSpecificFactorPersistenceSession +{ + private readonly NpgsqlDataSource _dataSource; + + public NpgsqlSourceSpecificFactorPersistenceSession(NpgsqlDataSource dataSource) + { + _dataSource = dataSource; + } + + public string ProviderName => "postgresql"; + + public async Task PersistSourceFamilyYearAsync( + PostgreSQLSourceSpecificFactorPersistenceBatch batch, + CancellationToken cancellationToken = default) + { + var sqlFlow = RenderSourceFamilyYearPersistenceSqlFlow(batch.SourceFamily); + var masterInsertSql = sqlFlow.Single(step => step.Name == "master_insert").CommandText; + var detailInsertSql = sqlFlow.Single(step => step.Name == "detail_insert").CommandText; + var yearStateInsertSql = sqlFlow.Single(step => step.Name == "year_state_insert").CommandText; + + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + try + { + var masterInserted = 0; + foreach (var master in batch.MasterRecords) + { + await EnsureIngestionRunAsync(connection, transaction, master, cancellationToken).ConfigureAwait(false); + await EnsureSourceDocumentAsync(connection, transaction, master, cancellationToken).ConfigureAwait(false); + masterInserted += await ExecuteInsertAsync( + connection, + transaction, + masterInsertSql, + MasterParameters(master), + cancellationToken).ConfigureAwait(false); + } + + var detailInserted = 0; + foreach (var detail in batch.DetailRecords) + { + detailInserted += await ExecuteInsertAsync( + connection, + transaction, + detailInsertSql, + DetailParameters(detail), + cancellationToken).ConfigureAwait(false); + } + + await RecordSuccessfulYearAsync( + connection, + transaction, + batch, + yearStateInsertSql, + cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + + return new PostgreSQLSourceSpecificFactorPersistenceCounts( + masterInserted, + batch.MasterRecords.Count - masterInserted, + detailInserted, + batch.DetailRecords.Count - detailInserted, + 0); + } + catch + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + throw; + } + } + + public static string RenderMasterInsertSql(SourceFamily sourceFamily) + { + var table = SourceFamilyRepositoryRegistry.GetTableNames(sourceFamily).MasterTableName; + var masterId = MasterIdColumn(sourceFamily); + return $$""" + INSERT INTO {{table}} ( + {{masterId}}, + source_family, + source_year, + source_version, + source_release, + source_document_id, + ingestion_run_id, + run_id, + master_external_key, + status, + artifact_reference, + artifact_checksum_sha256, + archive_reference, + archive_checksum_sha256, + effective_from, + effective_to, + record_checksum_sha256, + metadata, + created_at, + updated_at + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, $16, $17, $18::jsonb, NOW(), NOW() + ) + ON CONFLICT (source_family, source_year, source_version, master_external_key) + DO NOTHING + RETURNING {{masterId}} + """; + } + + public static string RenderDetailInsertSql(SourceFamily sourceFamily) + { + var table = SourceFamilyRepositoryRegistry.GetTableNames(sourceFamily).DetailTableName; + var masterId = MasterIdColumn(sourceFamily); + var detailId = DetailIdColumn(sourceFamily); + return $$""" + INSERT INTO {{table}} ( + {{detailId}}, + {{masterId}}, + detail_external_key, + source_row_number, + factor_id, + factor_name, + factor_value, + factor_unit, + status, + record_checksum_sha256, + raw_fields, + normalized_fields, + created_at, + updated_at + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11::jsonb, $12::jsonb, NOW(), NOW() + ) + ON CONFLICT ({{masterId}}, detail_external_key) + DO NOTHING + RETURNING {{detailId}} + """; + } + + public static string RenderYearStateInsertSql() + { + return """ + INSERT INTO source_family_year_states ( + source_family_year_state_id, + source_family, + ingested_year, + created_at, + updated_at + ) + VALUES ($1, $2, $3, NOW(), NOW()) + ON CONFLICT (source_family, ingested_year) + DO UPDATE SET updated_at = EXCLUDED.updated_at + RETURNING source_family_year_state_id + """; + } + + public static IReadOnlyList RenderSourceFamilyYearPersistenceSqlFlow( + SourceFamily sourceFamily) => + [ + new("master_insert", RenderMasterInsertSql(sourceFamily)), + new("detail_insert", RenderDetailInsertSql(sourceFamily)), + new("year_state_insert", RenderYearStateInsertSql()), + ]; + + public static PostgreSQLSourceSpecificFactorPersistenceTransactionBoundary DescribeTransactionBoundary() => + new( + OpensTransaction: true, + CommitsTransaction: true, + RollsBackOnFailure: true); + + private static async Task EnsureIngestionRunAsync( + NpgsqlConnection connection, + NpgsqlTransaction transaction, + PostgreSQLSourceSpecificMasterRecord master, + CancellationToken cancellationToken) + { + if (master.IngestionRunId is null) + { + return; + } + + await ExecuteInsertAsync( + connection, + transaction, + """ + INSERT INTO ingestion_runs ( + ingestion_run_id, + run_status, + created_at, + updated_at + ) + VALUES ($1, $2, NOW(), NOW()) + ON CONFLICT (ingestion_run_id) DO NOTHING + RETURNING ingestion_run_id + """, + [master.IngestionRunId.Value, "completed"], + cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureSourceDocumentAsync( + NpgsqlConnection connection, + NpgsqlTransaction transaction, + PostgreSQLSourceSpecificMasterRecord master, + CancellationToken cancellationToken) + { + await ExecuteInsertAsync( + connection, + transaction, + """ + INSERT INTO source_documents ( + source_document_id, + ingestion_run_id, + source_family, + source_document_uri, + source_checksum_sha256, + acquisition_status, + acquired_at, + created_at, + updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, NOW(), NOW(), NOW()) + ON CONFLICT (source_family, source_document_uri, source_checksum_sha256) + DO NOTHING + RETURNING source_document_id + """, + [ + master.SourceDocumentId, + master.IngestionRunId, + master.SourceFamily.ToPostgreSQLRuntimeValue(), + master.ArtifactReference ?? master.SourceDocumentId.ToString("D"), + master.ArtifactChecksumSha256 ?? "checksum-unavailable", + "downloaded", + ], + cancellationToken).ConfigureAwait(false); + } + + private static async Task RecordSuccessfulYearAsync( + NpgsqlConnection connection, + NpgsqlTransaction transaction, + PostgreSQLSourceSpecificFactorPersistenceBatch batch, + string commandText, + CancellationToken cancellationToken) + { + await ExecuteInsertAsync( + connection, + transaction, + commandText, + [Guid.NewGuid(), batch.SourceFamily.ToPostgreSQLRuntimeValue(), batch.SourceYear], + cancellationToken).ConfigureAwait(false); + } + + private static async Task ExecuteInsertAsync( + NpgsqlConnection connection, + NpgsqlTransaction transaction, + string commandText, + IReadOnlyList parameters, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = commandText; + foreach (var parameter in parameters) + { + command.Parameters.AddWithValue(parameter ?? DBNull.Value); + } + + var inserted = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return inserted is null ? 0 : 1; + } + + private static IReadOnlyList MasterParameters(PostgreSQLSourceSpecificMasterRecord record) => + [ + record.SourceFamilyMasterId, + record.SourceFamily.ToPostgreSQLRuntimeValue(), + record.SourceYear, + record.SourceVersion, + record.SourceRelease, + record.SourceDocumentId, + record.IngestionRunId, + record.RunId, + record.MasterExternalKey, + record.Status, + record.ArtifactReference, + record.ArtifactChecksumSha256, + record.ArchiveReference, + record.ArchiveChecksumSha256, + record.EffectiveFrom, + record.EffectiveTo, + record.RecordChecksumSha256, + JsonSerializer.Serialize(record.Metadata), + ]; + + private static IReadOnlyList DetailParameters(PostgreSQLSourceSpecificDetailRecord record) => + [ + record.SourceFamilyDetailId, + record.SourceFamilyMasterId, + record.DetailExternalKey, + record.SourceRowNumber, + record.FactorId, + record.FactorName, + record.FactorValue, + record.FactorUnit, + record.Status, + record.RecordChecksumSha256, + JsonSerializer.Serialize(record.RawFields), + JsonSerializer.Serialize(record.NormalizedFields), + ]; + + private static string MasterIdColumn(SourceFamily sourceFamily) + { + var prefix = SourceFamilyPrefix(sourceFamily); + return $"{prefix}_emission_factor_master_id"; + } + + private static string DetailIdColumn(SourceFamily sourceFamily) + { + var prefix = SourceFamilyPrefix(sourceFamily); + return $"{prefix}_emission_factor_detail_id"; + } + + private static string SourceFamilyPrefix(SourceFamily sourceFamily) => + sourceFamily switch + { + SourceFamily.GhgProtocol => "ghg", + SourceFamily.DefraDesnz => "defra", + SourceFamily.IpccEfdb => "ipcc", + _ => throw new ArgumentOutOfRangeException(nameof(sourceFamily), sourceFamily, "Unknown source family."), + }; +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/ProductionConfigBoundary.cs b/src/dotnet/CarbonOps.Parser.Contracts/ProductionConfigBoundary.cs new file mode 100644 index 00000000..c390cb8f --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/ProductionConfigBoundary.cs @@ -0,0 +1,307 @@ +using System.Text.Json; + +namespace CarbonOps.Parser.Contracts; + +public sealed record ProductionConfigValidationIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); + +public sealed record ProductionConfigValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public ProductionConfigValidationResult( + IEnumerable? issues = null) + { + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} + +public sealed record ProductionConfigBoundaryDescription( + IReadOnlyList RequiredEnvironmentVariables, + IReadOnlyList SecretEnvironmentVariables, + string Provider, + bool LoadsEnvironment, + bool LoadsConfigFiles, + bool LoadsCredentials, + bool LogsSecretValues, + IReadOnlyList Notes); + +public static class ProductionConfigBoundary +{ + public const string RawPostgreSQLConnectionStringKey = "CARBONOPS_PARSER_POSTGRES_CONNECTION_STRING"; + + public static readonly IReadOnlyList RequiredEnvironmentVariables = Array.AsReadOnly(new[] + { + "CARBONOPS_PARSER_ENV", + "CARBONOPS_PARSER_DATABASE_PROVIDER", + "CARBONOPS_PARSER_POSTGRES_HOST", + "CARBONOPS_PARSER_POSTGRES_PORT", + "CARBONOPS_PARSER_POSTGRES_DATABASE", + "CARBONOPS_PARSER_POSTGRES_USERNAME", + "CARBONOPS_PARSER_POSTGRES_PASSWORD", + "CARBONOPS_PARSER_POSTGRES_SCHEMA", + "CARBONOPS_PARSER_RAW_ARCHIVE_PATH", + "CARBONOPS_PARSER_LOG_LEVEL", + }); + + public static readonly IReadOnlyList SecretEnvironmentVariables = Array.AsReadOnly(new[] + { + "CARBONOPS_PARSER_POSTGRES_PASSWORD", + }); + + public static readonly IReadOnlyList KnownConfigurationKeys = Array.AsReadOnly( + RequiredEnvironmentVariables + .Append(RawPostgreSQLConnectionStringKey) + .ToArray()); + + private static readonly HashSet ValidLogLevels = new(StringComparer.OrdinalIgnoreCase) + { + "debug", + "info", + "warning", + "error", + "critical", + }; + + public static ProductionConfigBoundaryDescription Describe() => + new( + RequiredEnvironmentVariables, + SecretEnvironmentVariables, + "postgres", + LoadsEnvironment: true, + LoadsConfigFiles: true, + LoadsCredentials: true, + LogsSecretValues: false, + [ + "The .NET service entrypoint loads an optional explicit JSON config file and process environment.", + "Environment values override config file values for known production keys.", + "CARBONOPS_PARSER_POSTGRES_PASSWORD is required but never returned.", + "Connection strings are not accepted as production config values.", + "Validation messages name keys only and do not echo configured values.", + ]); + + public static ProductionConfigValidationResult Validate( + IReadOnlyDictionary values) + { + var issues = new List(); + + foreach (var envVar in RequiredEnvironmentVariables) + { + if (!HasText(Get(values, envVar))) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_MISSING_REQUIRED_ENV_VAR", + $"{envVar} must be set for production startup.", + envVar)); + } + } + + var provider = Get(values, "CARBONOPS_PARSER_DATABASE_PROVIDER"); + if (provider is not null && + HasText(provider) && + !string.Equals(provider.Trim(), "postgres", StringComparison.OrdinalIgnoreCase)) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_UNSUPPORTED_DATABASE_PROVIDER", + "Unsupported database provider. Phase 1 supports postgres only.", + "CARBONOPS_PARSER_DATABASE_PROVIDER")); + } + + ValidatePort(Get(values, "CARBONOPS_PARSER_POSTGRES_PORT"), issues); + ValidateLogLevel(Get(values, "CARBONOPS_PARSER_LOG_LEVEL"), issues); + + if (HasText(Get(values, RawPostgreSQLConnectionStringKey))) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_RAW_CONNECTION_STRING_NOT_ALLOWED", + "Raw PostgreSQL connection strings are not accepted; use split non-secret fields and CARBONOPS_PARSER_POSTGRES_PASSWORD.", + RawPostgreSQLConnectionStringKey)); + } + + return new ProductionConfigValidationResult(issues); + } + + private static void ValidatePort( + string? rawValue, + ICollection issues) + { + if (!HasText(rawValue)) + { + return; + } + + if (!int.TryParse(rawValue, out var port) || port is < 1 or > 65535) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_INVALID_POSTGRES_PORT", + "CARBONOPS_PARSER_POSTGRES_PORT must be an integer between 1 and 65535.", + "CARBONOPS_PARSER_POSTGRES_PORT")); + } + } + + private static void ValidateLogLevel( + string? rawValue, + ICollection issues) + { + if (!HasText(rawValue)) + { + return; + } + + if (rawValue is not null && !ValidLogLevels.Contains(rawValue.Trim())) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_INVALID_LOG_LEVEL", + "CARBONOPS_PARSER_LOG_LEVEL must be one of debug, info, warning, error, or critical.", + "CARBONOPS_PARSER_LOG_LEVEL")); + } + } + + private static bool HasText(string? value) => !string.IsNullOrWhiteSpace(value); + + private static string? Get(IReadOnlyDictionary values, string key) => + values.TryGetValue(key, out var value) ? value : null; +} + +public sealed record ProductionConfigLoadResult( + IReadOnlyDictionary Values, + IReadOnlyList Issues, + bool ConfigFileLoaded, + bool EnvironmentLoaded); + +public static class ProductionConfigLoader +{ + public static ProductionConfigLoadResult Load( + string? configPath, + IReadOnlyDictionary environment) + { + var values = new Dictionary(StringComparer.Ordinal); + var issues = new List(); + var configFileLoaded = false; + + foreach (var key in ProductionConfigBoundary.KnownConfigurationKeys) + { + values[key] = null; + } + + if (!string.IsNullOrWhiteSpace(configPath)) + { + configFileLoaded = LoadFileValues(configPath, values, issues); + } + + foreach (var key in ProductionConfigBoundary.KnownConfigurationKeys) + { + if (environment.TryGetValue(key, out var value)) + { + values[key] = value; + } + } + + return new ProductionConfigLoadResult( + values, + Array.AsReadOnly(issues.ToArray()), + configFileLoaded, + EnvironmentLoaded: true); + } + + private static bool LoadFileValues( + string configPath, + IDictionary values, + ICollection issues) + { + if (!File.Exists(configPath)) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_FILE_NOT_FOUND", + "Config file was not found.", + "config")); + return false; + } + + try + { + using var stream = File.OpenRead(configPath); + using var document = JsonDocument.Parse(stream); + + if (document.RootElement.ValueKind != JsonValueKind.Object) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_FILE_INVALID_SHAPE", + "Config file root must be a JSON object.", + "config")); + return false; + } + + foreach (var property in document.RootElement.EnumerateObject().OrderBy(item => item.Name, StringComparer.Ordinal)) + { + if (!values.ContainsKey(property.Name)) + { + continue; + } + + if (!TryReadScalar(property.Value, out var value)) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_FILE_INVALID_VALUE", + "Config file values must be scalar strings, numbers, booleans, or null.", + property.Name)); + continue; + } + + values[property.Name] = value; + } + + return true; + } + catch (JsonException) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_FILE_INVALID_JSON", + "Config file must be valid JSON.", + "config")); + return false; + } + catch (IOException) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_FILE_READ_FAILED", + "Config file could not be read.", + "config")); + return false; + } + catch (UnauthorizedAccessException) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_FILE_READ_FAILED", + "Config file could not be read.", + "config")); + return false; + } + } + + private static bool TryReadScalar(JsonElement element, out string? value) + { + switch (element.ValueKind) + { + case JsonValueKind.String: + value = element.GetString(); + return true; + case JsonValueKind.Number: + case JsonValueKind.True: + case JsonValueKind.False: + value = element.GetRawText(); + return true; + case JsonValueKind.Null: + value = null; + return true; + default: + value = null; + return false; + } + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepository.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepository.cs new file mode 100644 index 00000000..7f0f4a1c --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepository.cs @@ -0,0 +1,8 @@ +namespace CarbonOps.Parser.Contracts; + +public interface ISourceAcquisitionRunRepository +{ + string ProviderName { get; } + + SourceAcquisitionRunRepositoryPersistResult PersistRuns(IEnumerable runs); +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryIssue.cs new file mode 100644 index 00000000..1dfc4690 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceAcquisitionRunRepositoryIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryPersistResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryPersistResult.cs new file mode 100644 index 00000000..3d547b60 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryPersistResult.cs @@ -0,0 +1,24 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceAcquisitionRunRepositoryPersistResult +{ + public string ProviderName { get; } + + public SourceAcquisitionRunRepositoryPersistStatus Status { get; } + + public int PersistedCount { get; } + + public IReadOnlyList Issues { get; } + + public SourceAcquisitionRunRepositoryPersistResult( + string providerName, + SourceAcquisitionRunRepositoryPersistStatus status, + int persistedCount, + IEnumerable? issues = null) + { + ProviderName = providerName; + Status = status; + PersistedCount = persistedCount; + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryPersistStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryPersistStatus.cs new file mode 100644 index 00000000..11659433 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryPersistStatus.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public enum SourceAcquisitionRunRepositoryPersistStatus +{ + Declared = 0, + FailedValidation = 1, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryRegistry.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryRegistry.cs new file mode 100644 index 00000000..f7cd6244 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryRegistry.cs @@ -0,0 +1,60 @@ +namespace CarbonOps.Parser.Contracts; + +public static class SourceAcquisitionRunRepositoryRegistry +{ + public static SourceAcquisitionRunRepositoryPersistResult CreatePersistResult( + string providerName, + IEnumerable runs, + IEnumerable? issues = null) + { + var runSnapshot = runs.ToArray(); + var collectedIssues = ValidateInputs(providerName, runSnapshot).Issues + .Select(issue => new SourceAcquisitionRunRepositoryIssue( + issue.Code, + issue.Message, + issue.FieldName, + issue.Severity)) + .ToList(); + collectedIssues.AddRange(issues ?? []); + + var status = collectedIssues.Count == 0 + ? SourceAcquisitionRunRepositoryPersistStatus.Declared + : SourceAcquisitionRunRepositoryPersistStatus.FailedValidation; + + return new SourceAcquisitionRunRepositoryPersistResult( + providerName, + status, + status == SourceAcquisitionRunRepositoryPersistStatus.Declared ? runSnapshot.Length : 0, + collectedIssues); + } + + public static SourceAcquisitionRunRepositoryValidationResult ValidateInputs( + string providerName, + IEnumerable runs) + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(providerName)) + { + errors.Add(new SourceAcquisitionRunRepositoryIssue( + "SOURCE_ACQUISITION_RUN_REPOSITORY_MISSING_PROVIDER_NAME", + "ProviderName must be a non-empty string.", + "ProviderName")); + } + + var runSnapshot = runs.ToArray(); + for (var index = 0; index < runSnapshot.Length; index++) + { + var run = runSnapshot[index]; + if (run is null) + { + errors.Add(new SourceAcquisitionRunRepositoryIssue( + "SOURCE_ACQUISITION_RUN_REPOSITORY_INVALID_RUN", + "Runs must contain SourceAcquisitionRunResult instances.", + $"Runs[{index}]")); + } + } + + return new SourceAcquisitionRunRepositoryValidationResult(errors); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryValidationResult.cs new file mode 100644 index 00000000..c30c7fa7 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceAcquisitionRunRepositoryValidationResult.cs @@ -0,0 +1,14 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceAcquisitionRunRepositoryValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public SourceAcquisitionRunRepositoryValidationResult( + IEnumerable? issues = null) + { + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceCycleOrchestrator.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceCycleOrchestrator.cs new file mode 100644 index 00000000..0cbdd00e --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceCycleOrchestrator.cs @@ -0,0 +1,437 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text.Json; + +namespace CarbonOps.Parser.Contracts; + +public enum SourceCycleRunStatus +{ + Ready = 0, + Blocked = 1, + NoAvailableSourceYear = 2, + ParserNotAvailable = 3, + Parsed = 4, + PersistenceNotImplemented = 5, + NotImplemented = 6, +} + +public sealed record SourceCycleArtifact( + SourceFamily SourceFamily, + int ReportingYear, + string ArtifactReference, + string ContentType, + string? Extension = null, + string? VersionLabel = null); + +public sealed record SourceCycleRunSummary( + SourceFamily SourceFamily, + int? LatestSuccessfulYear, + int TargetYear, + SourceCycleRunStatus Status, + SourceCycleRunStatus ParserStatus, + SourceCycleRunStatus PersistenceStatus, + string StatusReason, + string? ArtifactReference, + int ArtifactCount, + int ParsedRowCount, + int ParserIssueCount, + ParserRunStatus? ParserRunStatus); + +public sealed record SourceCyclePreviewResult( + IReadOnlyList Runs, + bool PostgreSQLConnectionOpened, + bool PostgreSQLSqlExecuted, + bool RecordsInserted, + bool YearStateAdvanced, + bool NetworkAccessAttempted, + bool SecretValuesPrinted) +{ + public int RunCount => Runs.Count; +} + +public sealed class SourceCycleOrchestrator +{ + private readonly PostgreSQLSourceFamilyYearStateRepository _yearStateRepository; + private readonly IReadOnlyList _enabledSourceFamilies; + private readonly IReadOnlyDictionary<(SourceFamily SourceFamily, int ReportingYear), SourceCycleArtifact> _artifacts; + private readonly Func _loadLocalArtifactContent; + + public SourceCycleOrchestrator( + PostgreSQLSourceFamilyYearStateRepository yearStateRepository, + IEnumerable? enabledSourceFamilies = null, + IEnumerable? artifacts = null, + Func? loadLocalArtifactContent = null) + { + _yearStateRepository = yearStateRepository; + _enabledSourceFamilies = Array.AsReadOnly((enabledSourceFamilies ?? SourceFamilyRegistry.SupportedFamilies) + .Distinct() + .OrderBy(family => family.ToWireName(), StringComparer.Ordinal) + .ToArray()); + _artifacts = (artifacts ?? []) + .GroupBy(artifact => (artifact.SourceFamily, artifact.ReportingYear)) + .ToDictionary(group => group.Key, group => group.OrderBy(item => item.ArtifactReference, StringComparer.Ordinal).First()); + _loadLocalArtifactContent = loadLocalArtifactContent ?? LoadLocalArtifactContent; + } + + public async Task PreviewAsync(CancellationToken cancellationToken = default) + { + var runs = new List(); + + foreach (var sourceFamily in _enabledSourceFamilies) + { + var yearState = await _yearStateRepository.GetYearStateAsync(sourceFamily, cancellationToken).ConfigureAwait(false); + runs.Add(RunSourceFamily(yearState)); + } + + return new SourceCyclePreviewResult( + Array.AsReadOnly(runs.ToArray()), + PostgreSQLConnectionOpened: false, + PostgreSQLSqlExecuted: false, + RecordsInserted: false, + YearStateAdvanced: false, + NetworkAccessAttempted: false, + SecretValuesPrinted: false); + } + + private SourceCycleRunSummary RunSourceFamily(SourceFamilyYearState yearState) + { + if (!_artifacts.TryGetValue((yearState.SourceFamily, yearState.NextYear), out var artifact)) + { + return Summary( + yearState, + SourceCycleRunStatus.NoAvailableSourceYear, + SourceCycleRunStatus.NoAvailableSourceYear, + "no configured local artifact exists for the target source-family year", + artifactReference: null, + artifactCount: 0, + parsedRowCount: 0, + parserIssueCount: 0, + parserRunStatus: null); + } + + if (!IsLocalTextArtifact(artifact)) + { + return Summary( + yearState, + SourceCycleRunStatus.ParserNotAvailable, + SourceCycleRunStatus.ParserNotAvailable, + "parser handoff is available only for configured local CSV/text artifacts", + artifact.ArtifactReference, + artifactCount: 1, + parsedRowCount: 0, + parserIssueCount: 0, + parserRunStatus: null); + } + + var content = _loadLocalArtifactContent(artifact); + if (content is null) + { + return Summary( + yearState, + SourceCycleRunStatus.NoAvailableSourceYear, + SourceCycleRunStatus.NoAvailableSourceYear, + "configured artifact is missing or unavailable", + artifact.ArtifactReference, + artifactCount: 0, + parsedRowCount: 0, + parserIssueCount: 0, + parserRunStatus: null); + } + + var parserResult = ParseArtifact(artifact, content); + var parserStatus = parserResult.Status == ParserRunStatus.Completed + ? SourceCycleRunStatus.Parsed + : SourceCycleRunStatus.ParserNotAvailable; + var status = parserResult.Status == ParserRunStatus.Completed + ? SourceCycleRunStatus.PersistenceNotImplemented + : SourceCycleRunStatus.ParserNotAvailable; + + return Summary( + yearState, + status, + parserStatus, + parserResult.Status == ParserRunStatus.Completed + ? "parser completed; source-specific master/detail persistence is not implemented" + : "parser failed closed for the configured artifact", + artifact.ArtifactReference, + artifactCount: 1, + parserResult.RowCount, + parserResult.IssueCount, + parserResult.Status); + } + + private static SourceCycleRunSummary Summary( + SourceFamilyYearState yearState, + SourceCycleRunStatus status, + SourceCycleRunStatus parserStatus, + string statusReason, + string? artifactReference, + int artifactCount, + int parsedRowCount, + int parserIssueCount, + ParserRunStatus? parserRunStatus) => + new( + yearState.SourceFamily, + yearState.LatestYear, + yearState.NextYear, + status, + parserStatus, + SourceCycleRunStatus.PersistenceNotImplemented, + statusReason, + artifactReference, + artifactCount, + parsedRowCount, + parserIssueCount, + parserRunStatus); + + private static ParserAdapterRunResult ParseArtifact(SourceCycleArtifact artifact, string content) + { + var parserKey = ParserSelectionRegistry.GetParserKey(artifact.SourceFamily); + var parserArtifact = new ParserInputArtifact( + artifact.SourceFamily, + artifact.SourceFamily.ToWireName(), + parserKey, + ParserSourceFormat.DiscoveryReference, + artifact.ArtifactReference, + Path.GetFileName(artifact.ArtifactReference), + "sha256", + Sha256Hex(content), + isDryRunChecksum: false, + artifact.ContentType, + artifact.Extension, + artifact.ReportingYear); + var request = new ParserAdapterRunRequest( + artifact.SourceFamily, + artifact.SourceFamily.ToWireName(), + parserKey, + [parserArtifact], + requestedReportingYear: artifact.ReportingYear); + var contentByArtifactReference = new Dictionary(StringComparer.Ordinal) + { + [artifact.ArtifactReference] = content, + }; + + return artifact.SourceFamily switch + { + SourceFamily.GhgProtocol => GhgProtocolNormalizedContentParser.Parse(request, contentByArtifactReference), + SourceFamily.DefraDesnz => DefraDesnzNormalizedContentParser.Parse(request, contentByArtifactReference), + SourceFamily.IpccEfdb => IpccEfdbNormalizedContentParser.Parse(request, contentByArtifactReference), + _ => throw new ArgumentOutOfRangeException(nameof(artifact), artifact.SourceFamily, "Unknown source family."), + }; + } + + private static string? LoadLocalArtifactContent(SourceCycleArtifact artifact) + { + if (!TryGetLocalPath(artifact.ArtifactReference, out var localPath) || !File.Exists(localPath)) + { + return null; + } + + return File.ReadAllText(localPath); + } + + private static bool IsLocalTextArtifact(SourceCycleArtifact artifact) => + IsLocalPath(artifact.ArtifactReference) && + (string.Equals(artifact.Extension, ".csv", StringComparison.OrdinalIgnoreCase) || + string.Equals(artifact.ContentType, "text/csv", StringComparison.OrdinalIgnoreCase) || + string.Equals(artifact.ContentType, "text/plain", StringComparison.OrdinalIgnoreCase)); + + private static bool IsLocalPath(string reference) => + !Uri.TryCreate(reference, UriKind.Absolute, out var uri) || uri.IsFile; + + private static bool TryGetLocalPath(string reference, out string localPath) + { + if (Uri.TryCreate(reference, UriKind.Absolute, out var uri)) + { + if (!uri.IsFile) + { + localPath = string.Empty; + return false; + } + + localPath = uri.LocalPath; + return true; + } + + localPath = reference; + return true; + } + + private static string Sha256Hex(string content) + { + var hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(content)); + return Convert.ToHexString(hash).ToLowerInvariant(); + } +} + +public static class SourceCycleConfiguration +{ + public static IReadOnlyList LoadArtifacts(string? configPath) + { + if (string.IsNullOrWhiteSpace(configPath) || !File.Exists(configPath)) + { + return Array.Empty(); + } + + using var document = TryLoadJsonDocument(configPath); + if (document is null) + { + return Array.Empty(); + } + + if (document.RootElement.ValueKind != JsonValueKind.Object || + !TryGetArtifactRoot(document.RootElement, out var root) || + root.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + var artifacts = new List(); + foreach (var familyProperty in root.EnumerateObject().OrderBy(item => item.Name, StringComparer.Ordinal)) + { + if (!ContractWireNames.TryParseSourceFamilyWireName(familyProperty.Name, out var sourceFamily) || + familyProperty.Value.ValueKind != JsonValueKind.Object) + { + continue; + } + + foreach (var yearProperty in familyProperty.Value.EnumerateObject().OrderBy(item => item.Name, StringComparer.Ordinal)) + { + if (!int.TryParse(yearProperty.Name, NumberStyles.None, CultureInfo.InvariantCulture, out var year) || + year < 1 || + !TryReadArtifactReference(yearProperty.Value, out var reference, out var contentType, out var extension, out var versionLabel)) + { + continue; + } + + artifacts.Add(new SourceCycleArtifact(sourceFamily, year, reference, contentType, extension, versionLabel)); + } + } + + return Array.AsReadOnly(artifacts.ToArray()); + } + + public static IReadOnlyList LoadEnabledSourceFamilies(string? configPath) + { + if (string.IsNullOrWhiteSpace(configPath) || !File.Exists(configPath)) + { + return SourceFamilyRegistry.SupportedFamilies; + } + + using var document = TryLoadJsonDocument(configPath); + if (document is null) + { + return SourceFamilyRegistry.SupportedFamilies; + } + + if (document.RootElement.ValueKind != JsonValueKind.Object || + !document.RootElement.TryGetProperty("enabled_source_families", out var root) || + root.ValueKind != JsonValueKind.Array) + { + return SourceFamilyRegistry.SupportedFamilies; + } + + var families = root + .EnumerateArray() + .Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() : null) + .Where(item => ContractWireNames.TryParseSourceFamilyWireName(item, out _)) + .Select(item => + { + ContractWireNames.TryParseSourceFamilyWireName(item, out var sourceFamily); + return sourceFamily; + }) + .Distinct() + .ToArray(); + + return families.Length == 0 ? SourceFamilyRegistry.SupportedFamilies : Array.AsReadOnly(families); + } + + private static JsonDocument? TryLoadJsonDocument(string configPath) + { + try + { + using var stream = File.OpenRead(configPath); + return JsonDocument.Parse(stream); + } + catch (IOException) + { + return null; + } + catch (JsonException) + { + return null; + } + } + + private static bool TryGetArtifactRoot(JsonElement documentRoot, out JsonElement artifactRoot) + { + if (documentRoot.TryGetProperty("source_artifacts", out artifactRoot)) + { + return true; + } + + return documentRoot.TryGetProperty("source_years", out artifactRoot); + } + + private static bool TryReadArtifactReference( + JsonElement element, + out string reference, + out string contentType, + out string? extension, + out string? versionLabel) + { + reference = string.Empty; + contentType = "text/csv"; + extension = ".csv"; + versionLabel = null; + + if (element.ValueKind == JsonValueKind.String) + { + reference = element.GetString() ?? string.Empty; + return !string.IsNullOrWhiteSpace(reference); + } + + if (element.ValueKind != JsonValueKind.Object || + !TryReadReferenceProperty(element, out var path)) + { + return false; + } + + reference = path; + if (element.TryGetProperty("content_type", out var contentTypeElement) && + contentTypeElement.ValueKind == JsonValueKind.String) + { + contentType = contentTypeElement.GetString() ?? contentType; + } + + if (element.TryGetProperty("extension", out var extensionElement) && + extensionElement.ValueKind == JsonValueKind.String) + { + extension = extensionElement.GetString(); + } + + if (element.TryGetProperty("version_label", out var versionLabelElement) && + versionLabelElement.ValueKind == JsonValueKind.String) + { + versionLabel = versionLabelElement.GetString(); + } + + return !string.IsNullOrWhiteSpace(reference); + } + + private static bool TryReadReferenceProperty(JsonElement element, out string reference) + { + foreach (var propertyName in new[] { "path", "uri", "artifact_url" }) + { + if (element.TryGetProperty(propertyName, out var property) && + property.ValueKind == JsonValueKind.String) + { + reference = property.GetString() ?? string.Empty; + return !string.IsNullOrWhiteSpace(reference); + } + } + + reference = string.Empty; + return false; + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepository.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepository.cs new file mode 100644 index 00000000..abea2745 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepository.cs @@ -0,0 +1,9 @@ +namespace CarbonOps.Parser.Contracts; + +public interface ISourceDocumentRepository +{ + string ProviderName { get; } + + SourceDocumentRepositoryPersistResult PersistSourceDocuments( + IEnumerable records); +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryIssue.cs new file mode 100644 index 00000000..2633764f --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceDocumentRepositoryIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryPersistResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryPersistResult.cs new file mode 100644 index 00000000..e260fd72 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryPersistResult.cs @@ -0,0 +1,24 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceDocumentRepositoryPersistResult +{ + public string ProviderName { get; } + + public SourceDocumentRepositoryPersistStatus Status { get; } + + public int PersistedCount { get; } + + public IReadOnlyList Issues { get; } + + public SourceDocumentRepositoryPersistResult( + string providerName, + SourceDocumentRepositoryPersistStatus status, + int persistedCount, + IEnumerable? issues = null) + { + ProviderName = providerName; + Status = status; + PersistedCount = persistedCount; + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryPersistStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryPersistStatus.cs new file mode 100644 index 00000000..7539bae4 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryPersistStatus.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public enum SourceDocumentRepositoryPersistStatus +{ + Declared = 0, + FailedValidation = 1, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryRegistry.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryRegistry.cs new file mode 100644 index 00000000..32e0c2e2 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryRegistry.cs @@ -0,0 +1,60 @@ +namespace CarbonOps.Parser.Contracts; + +public static class SourceDocumentRepositoryRegistry +{ + public static SourceDocumentRepositoryPersistResult CreatePersistResult( + string providerName, + IEnumerable records, + IEnumerable? issues = null) + { + var recordSnapshot = records.ToArray(); + var collectedIssues = ValidateInputs(providerName, recordSnapshot).Issues + .Select(issue => new SourceDocumentRepositoryIssue( + issue.Code, + issue.Message, + issue.FieldName, + issue.Severity)) + .ToList(); + collectedIssues.AddRange(issues ?? []); + + var status = collectedIssues.Count == 0 + ? SourceDocumentRepositoryPersistStatus.Declared + : SourceDocumentRepositoryPersistStatus.FailedValidation; + + return new SourceDocumentRepositoryPersistResult( + providerName, + status, + status == SourceDocumentRepositoryPersistStatus.Declared ? recordSnapshot.Length : 0, + collectedIssues); + } + + public static SourceDocumentRepositoryValidationResult ValidateInputs( + string providerName, + IEnumerable records) + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(providerName)) + { + errors.Add(new SourceDocumentRepositoryIssue( + "SOURCE_DOCUMENT_REPOSITORY_MISSING_PROVIDER_NAME", + "ProviderName must be a non-empty string.", + "ProviderName")); + } + + var recordSnapshot = records.ToArray(); + for (var index = 0; index < recordSnapshot.Length; index++) + { + var record = recordSnapshot[index]; + if (record is null) + { + errors.Add(new SourceDocumentRepositoryIssue( + "SOURCE_DOCUMENT_REPOSITORY_INVALID_RECORD", + "Records must contain SourceDocumentPersistenceRecord instances.", + $"Records[{index}]")); + } + } + + return new SourceDocumentRepositoryValidationResult(errors); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryValidationResult.cs new file mode 100644 index 00000000..d6bbb40d --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceDocumentRepositoryValidationResult.cs @@ -0,0 +1,14 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceDocumentRepositoryValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public SourceDocumentRepositoryValidationResult( + IEnumerable? issues = null) + { + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyDetailRecord.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyDetailRecord.cs new file mode 100644 index 00000000..78ea7999 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyDetailRecord.cs @@ -0,0 +1,13 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceFamilyDetailRecord( + SourceFamily SourceFamily, + string SourceFamilyDetailId, + string SourceFamilyMasterId, + string DetailExternalKey, + string FactorValue, + string FactorUnit, + string LifecycleStatus, + string RecordChecksumSha256, + string CreatedAt, + string UpdatedAt); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyMasterRecord.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyMasterRecord.cs new file mode 100644 index 00000000..83940de7 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyMasterRecord.cs @@ -0,0 +1,13 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceFamilyMasterRecord( + SourceFamily SourceFamily, + string SourceFamilyMasterId, + string SourceDocumentId, + string MasterExternalKey, + string LifecycleStatus, + string? EffectiveFrom, + string? EffectiveTo, + string RecordChecksumSha256, + string CreatedAt, + string UpdatedAt); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepository.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepository.cs new file mode 100644 index 00000000..47966e52 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepository.cs @@ -0,0 +1,10 @@ +namespace CarbonOps.Parser.Contracts; + +public interface ISourceFamilyRepository +{ + string ProviderName { get; } + + SourceFamilyRepositoryPersistResult PersistSourceFamilyRecords( + IEnumerable masterRecords, + IEnumerable detailRecords); +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryIssue.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryIssue.cs new file mode 100644 index 00000000..18e04c2b --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryIssue.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceFamilyRepositoryIssue( + string Code, + string Message, + string FieldName, + string Severity = "error"); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryPersistResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryPersistResult.cs new file mode 100644 index 00000000..4d508ee3 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryPersistResult.cs @@ -0,0 +1,28 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceFamilyRepositoryPersistResult +{ + public string ProviderName { get; } + + public SourceFamilyRepositoryPersistStatus Status { get; } + + public int PersistedMasterCount { get; } + + public int PersistedDetailCount { get; } + + public IReadOnlyList Issues { get; } + + public SourceFamilyRepositoryPersistResult( + string providerName, + SourceFamilyRepositoryPersistStatus status, + int persistedMasterCount, + int persistedDetailCount, + IEnumerable? issues = null) + { + ProviderName = providerName; + Status = status; + PersistedMasterCount = persistedMasterCount; + PersistedDetailCount = persistedDetailCount; + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryPersistStatus.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryPersistStatus.cs new file mode 100644 index 00000000..e369de27 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryPersistStatus.cs @@ -0,0 +1,7 @@ +namespace CarbonOps.Parser.Contracts; + +public enum SourceFamilyRepositoryPersistStatus +{ + Declared = 0, + FailedValidation = 1, +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryRegistry.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryRegistry.cs new file mode 100644 index 00000000..71058013 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryRegistry.cs @@ -0,0 +1,179 @@ +namespace CarbonOps.Parser.Contracts; + +public static class SourceFamilyRepositoryRegistry +{ + public static SourceFamilyRepositoryPersistResult CreatePersistResult( + string providerName, + IEnumerable masterRecords, + IEnumerable detailRecords, + IEnumerable? issues = null) + { + var masterSnapshot = masterRecords.ToArray(); + var detailSnapshot = detailRecords.ToArray(); + var collectedIssues = ValidateInputs(providerName, masterSnapshot, detailSnapshot).Issues + .Select(issue => new SourceFamilyRepositoryIssue( + issue.Code, + issue.Message, + issue.FieldName, + issue.Severity)) + .ToList(); + collectedIssues.AddRange(issues ?? []); + + var status = collectedIssues.Count == 0 + ? SourceFamilyRepositoryPersistStatus.Declared + : SourceFamilyRepositoryPersistStatus.FailedValidation; + + return new SourceFamilyRepositoryPersistResult( + providerName, + status, + status == SourceFamilyRepositoryPersistStatus.Declared ? masterSnapshot.Length : 0, + status == SourceFamilyRepositoryPersistStatus.Declared ? detailSnapshot.Length : 0, + collectedIssues); + } + + public static SourceFamilyRepositoryValidationResult ValidateInputs( + string providerName, + IEnumerable masterRecords, + IEnumerable detailRecords) + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(providerName)) + { + errors.Add(new SourceFamilyRepositoryIssue( + "SOURCE_FAMILY_REPOSITORY_MISSING_PROVIDER_NAME", + "ProviderName must be a non-empty string.", + "ProviderName")); + } + + var masterSnapshot = masterRecords.ToArray(); + var detailSnapshot = detailRecords.ToArray(); + var masterKeys = new HashSet<(SourceFamily SourceFamily, string SourceFamilyMasterId)>(); + + for (var index = 0; index < masterSnapshot.Length; index++) + { + var record = masterSnapshot[index]; + if (record is null) + { + errors.Add(new SourceFamilyRepositoryIssue( + "SOURCE_FAMILY_REPOSITORY_INVALID_MASTER_RECORD", + "MasterRecords must contain SourceFamilyMasterRecord instances.", + $"MasterRecords[{index}]")); + continue; + } + + ValidateSourceFamily(record.SourceFamily, $"MasterRecords[{index}].SourceFamily", errors); + ValidateMasterRecord(record, index, errors); + if (!string.IsNullOrWhiteSpace(record.SourceFamilyMasterId)) + { + masterKeys.Add((record.SourceFamily, record.SourceFamilyMasterId)); + } + } + + for (var index = 0; index < detailSnapshot.Length; index++) + { + var record = detailSnapshot[index]; + if (record is null) + { + errors.Add(new SourceFamilyRepositoryIssue( + "SOURCE_FAMILY_REPOSITORY_INVALID_DETAIL_RECORD", + "DetailRecords must contain SourceFamilyDetailRecord instances.", + $"DetailRecords[{index}]")); + continue; + } + + ValidateSourceFamily(record.SourceFamily, $"DetailRecords[{index}].SourceFamily", errors); + ValidateDetailRecord(record, index, masterKeys, errors); + } + + return new SourceFamilyRepositoryValidationResult(errors); + } + + public static SourceFamilyRepositoryTableNames GetTableNames(SourceFamily sourceFamily) + { + ValidateSourceFamily(sourceFamily, nameof(sourceFamily), []); + + var familyPrefix = sourceFamily switch + { + SourceFamily.GhgProtocol => "ghg", + SourceFamily.DefraDesnz => "defra", + SourceFamily.IpccEfdb => "ipcc", + _ => throw new ArgumentOutOfRangeException( + nameof(sourceFamily), + sourceFamily, + "Unknown source family."), + }; + + return new SourceFamilyRepositoryTableNames( + $"{familyPrefix}_emission_factor_masters", + $"{familyPrefix}_emission_factor_details"); + } + + private static void ValidateMasterRecord( + SourceFamilyMasterRecord record, + int index, + ICollection errors) + { + AppendRequiredStringIssue(errors, record.SourceFamilyMasterId, $"MasterRecords[{index}].SourceFamilyMasterId"); + AppendRequiredStringIssue(errors, record.SourceDocumentId, $"MasterRecords[{index}].SourceDocumentId"); + AppendRequiredStringIssue(errors, record.MasterExternalKey, $"MasterRecords[{index}].MasterExternalKey"); + AppendRequiredStringIssue(errors, record.LifecycleStatus, $"MasterRecords[{index}].LifecycleStatus"); + AppendRequiredStringIssue(errors, record.RecordChecksumSha256, $"MasterRecords[{index}].RecordChecksumSha256"); + AppendRequiredStringIssue(errors, record.CreatedAt, $"MasterRecords[{index}].CreatedAt"); + AppendRequiredStringIssue(errors, record.UpdatedAt, $"MasterRecords[{index}].UpdatedAt"); + } + + private static void ValidateDetailRecord( + SourceFamilyDetailRecord record, + int index, + ISet<(SourceFamily SourceFamily, string SourceFamilyMasterId)> masterKeys, + ICollection errors) + { + AppendRequiredStringIssue(errors, record.SourceFamilyDetailId, $"DetailRecords[{index}].SourceFamilyDetailId"); + AppendRequiredStringIssue(errors, record.SourceFamilyMasterId, $"DetailRecords[{index}].SourceFamilyMasterId"); + AppendRequiredStringIssue(errors, record.DetailExternalKey, $"DetailRecords[{index}].DetailExternalKey"); + AppendRequiredStringIssue(errors, record.FactorValue, $"DetailRecords[{index}].FactorValue"); + AppendRequiredStringIssue(errors, record.FactorUnit, $"DetailRecords[{index}].FactorUnit"); + AppendRequiredStringIssue(errors, record.LifecycleStatus, $"DetailRecords[{index}].LifecycleStatus"); + AppendRequiredStringIssue(errors, record.RecordChecksumSha256, $"DetailRecords[{index}].RecordChecksumSha256"); + AppendRequiredStringIssue(errors, record.CreatedAt, $"DetailRecords[{index}].CreatedAt"); + AppendRequiredStringIssue(errors, record.UpdatedAt, $"DetailRecords[{index}].UpdatedAt"); + + if (!string.IsNullOrWhiteSpace(record.SourceFamilyMasterId) && + !masterKeys.Contains((record.SourceFamily, record.SourceFamilyMasterId))) + { + errors.Add(new SourceFamilyRepositoryIssue( + "SOURCE_FAMILY_REPOSITORY_DETAIL_MASTER_NOT_DECLARED", + "Detail record SourceFamilyMasterId must reference a declared master record for the same source family.", + $"DetailRecords[{index}].SourceFamilyMasterId")); + } + } + + private static void ValidateSourceFamily( + SourceFamily sourceFamily, + string fieldName, + ICollection errors) + { + if (!Enum.IsDefined(sourceFamily)) + { + errors.Add(new SourceFamilyRepositoryIssue( + "SOURCE_FAMILY_REPOSITORY_INVALID_SOURCE_FAMILY", + "SourceFamily must be a supported Phase 1 source family.", + fieldName)); + } + } + + private static void AppendRequiredStringIssue( + ICollection errors, + string? value, + string fieldName) + { + if (string.IsNullOrWhiteSpace(value)) + { + errors.Add(new SourceFamilyRepositoryIssue( + "SOURCE_FAMILY_REPOSITORY_MISSING_REQUIRED_FIELD", + "Required fields must be non-empty strings.", + fieldName)); + } + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryTableNames.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryTableNames.cs new file mode 100644 index 00000000..4eb3f89d --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryTableNames.cs @@ -0,0 +1,5 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceFamilyRepositoryTableNames( + string MasterTableName, + string DetailTableName); diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryValidationResult.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryValidationResult.cs new file mode 100644 index 00000000..18758ce1 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceFamilyRepositoryValidationResult.cs @@ -0,0 +1,14 @@ +namespace CarbonOps.Parser.Contracts; + +public sealed record SourceFamilyRepositoryValidationResult +{ + public IReadOnlyList Issues { get; } + + public bool IsValid => Issues.Count == 0; + + public SourceFamilyRepositoryValidationResult( + IEnumerable? issues = null) + { + Issues = Array.AsReadOnly((issues ?? []).ToArray()); + } +} diff --git a/src/dotnet/CarbonOps.Parser.Contracts/SourceOnboardingRegistry.cs b/src/dotnet/CarbonOps.Parser.Contracts/SourceOnboardingRegistry.cs new file mode 100644 index 00000000..e7461d47 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Contracts/SourceOnboardingRegistry.cs @@ -0,0 +1,470 @@ +namespace CarbonOps.Parser.Contracts; + +public enum SourceOnboardingDiscoveryStrategy +{ + DeclaredReference = 0, + SourceSpecificDiscovery = 1, +} + +public enum SourceOnboardingUpdateCadence +{ + Unknown = 0, + Annual = 1, + Periodic = 2, +} + +public sealed record SourceOnboardingDocument +{ + public string DocumentId { get; } + + public string DisplayName { get; } + + public string SourceReference { get; } + + public string ExpectedFormat { get; } + + public bool Required { get; } + + public SourceOnboardingDocument( + string documentId, + string displayName, + string sourceReference, + string expectedFormat, + bool required = true) + { + DocumentId = documentId; + DisplayName = displayName; + SourceReference = sourceReference; + ExpectedFormat = expectedFormat; + Required = required; + } +} + +public sealed record SourceOnboardingParserCapability +{ + public ParserKey ParserKey { get; } + + public ParserSourceFormat ParserSourceFormat { get; } + + public bool SupportsParserExecution { get; } + + public string CapabilityNotes { get; } + + public SourceOnboardingParserCapability( + ParserKey parserKey, + ParserSourceFormat parserSourceFormat, + bool supportsParserExecution, + string capabilityNotes) + { + ParserKey = parserKey; + ParserSourceFormat = parserSourceFormat; + SupportsParserExecution = supportsParserExecution; + CapabilityNotes = capabilityNotes; + } +} + +public sealed record SourceOnboardingValidationExpectations +{ + public IReadOnlyList RequiredDocumentFields { get; } + + public bool ChecksumRequired { get; } + + public bool SchemaValidationRequired { get; } + + public string ValidationNotes { get; } + + public SourceOnboardingValidationExpectations( + IEnumerable requiredDocumentFields, + bool checksumRequired, + bool schemaValidationRequired, + string validationNotes) + { + RequiredDocumentFields = Array.AsReadOnly(requiredDocumentFields.ToArray()); + ChecksumRequired = checksumRequired; + SchemaValidationRequired = schemaValidationRequired; + ValidationNotes = validationNotes; + } +} + +public sealed record SourceOnboardingRuntimeSafety +{ + public bool AllowsNetworkCalls { get; } + + public bool AllowsFileReads { get; } + + public bool AllowsDatabaseWrites { get; } + + public bool RequiresCredentials { get; } + + public string SafetyNotes { get; } + + public SourceOnboardingRuntimeSafety( + bool allowsNetworkCalls, + bool allowsFileReads, + bool allowsDatabaseWrites, + bool requiresCredentials, + string safetyNotes) + { + AllowsNetworkCalls = allowsNetworkCalls; + AllowsFileReads = allowsFileReads; + AllowsDatabaseWrites = allowsDatabaseWrites; + RequiresCredentials = requiresCredentials; + SafetyNotes = safetyNotes; + } +} + +public sealed record SourceOnboardingRegistryEntry +{ + public string SourceId { get; } + + public string SourceFamily { get; } + + public string DisplayName { get; } + + public IReadOnlyList Documents { get; } + + public SourceOnboardingDiscoveryStrategy DiscoveryStrategy { get; } + + public SourceOnboardingParserCapability ParserCapability { get; } + + public SourceOnboardingValidationExpectations ValidationExpectations { get; } + + public SourceOnboardingUpdateCadence UpdateCadence { get; } + + public SourceOnboardingRuntimeSafety RuntimeSafety { get; } + + public bool Enabled { get; } + + public SourceOnboardingRegistryEntry( + string sourceId, + string sourceFamily, + string displayName, + IEnumerable documents, + SourceOnboardingDiscoveryStrategy discoveryStrategy, + SourceOnboardingParserCapability parserCapability, + SourceOnboardingValidationExpectations validationExpectations, + SourceOnboardingUpdateCadence updateCadence, + SourceOnboardingRuntimeSafety runtimeSafety, + bool enabled = true) + { + SourceId = sourceId; + SourceFamily = sourceFamily; + DisplayName = displayName; + Documents = Array.AsReadOnly(documents.ToArray()); + DiscoveryStrategy = discoveryStrategy; + ParserCapability = parserCapability; + ValidationExpectations = validationExpectations; + UpdateCadence = updateCadence; + RuntimeSafety = runtimeSafety; + Enabled = enabled; + } +} + +public sealed record SourceOnboardingRegistry +{ + public static IReadOnlyList Phase2OnboardingSourceFamilies { get; } = + Array.AsReadOnly(["ghg_protocol", "defra_desnz", "ipcc_efdb"]); + + public IReadOnlyList Entries { get; } + + public int EntryCount => Entries.Count; + + public SourceOnboardingRegistry(IEnumerable entries) + { + Entries = Array.AsReadOnly(entries.ToArray()); + } + + public static SourceOnboardingRegistry CreatePhase2SourceOnboardingRegistry() + { + var entries = SourceFamilyRegistry.SupportedFamilies.Select(sourceFamily => + { + var sourceId = sourceFamily.ToWireName(); + var displayName = GetDisplayName(sourceFamily); + + return new SourceOnboardingRegistryEntry( + sourceId, + sourceId, + displayName, + [ + new SourceOnboardingDocument( + $"{sourceId}_declared_reference", + $"{displayName} declared reference", + $"discovery://{sourceId}/onboarding", + "discovery"), + ], + SourceOnboardingDiscoveryStrategy.DeclaredReference, + new SourceOnboardingParserCapability( + ParserSelectionRegistry.GetParserKey(sourceFamily), + ParserSourceFormat.DiscoveryReference, + supportsParserExecution: false, + "Registry metadata only; parser execution is outside this onboarding contract."), + new SourceOnboardingValidationExpectations( + ["document_id", "display_name", "source_reference", "expected_format"], + checksumRequired: false, + schemaValidationRequired: false, + "Declared discovery references are validated for contract shape only."), + SourceOnboardingUpdateCadence.Unknown, + new SourceOnboardingRuntimeSafety( + allowsNetworkCalls: false, + allowsFileReads: false, + allowsDatabaseWrites: false, + requiresCredentials: false, + "Default onboarding registry is runtime-passive and local-only.")); + }); + + var registry = new SourceOnboardingRegistry(entries); + var validation = registry.Validate(); + + if (!validation.IsValid) + { + throw new InvalidOperationException( + $"Default source onboarding registry is invalid: {string.Join("; ", validation.Errors)}"); + } + + return registry; + } + + public static IReadOnlyList ListEntries(SourceOnboardingRegistry? registry = null) + { + var activeRegistry = registry ?? CreatePhase2SourceOnboardingRegistry(); + var validation = activeRegistry.Validate(); + + if (!validation.IsValid) + { + throw new ArgumentException( + $"Source onboarding registry is invalid: {string.Join("; ", validation.Errors)}", + nameof(registry)); + } + + return activeRegistry.Entries; + } + + public static bool TryGetBySourceFamily( + string sourceFamily, + out SourceOnboardingRegistryEntry? entry, + SourceOnboardingRegistry? registry = null) + { + entry = ListEntries(registry).SingleOrDefault(candidate => candidate.SourceFamily == sourceFamily); + + return entry is not null; + } + + public static bool TryGetBySourceFamily( + SourceFamily sourceFamily, + out SourceOnboardingRegistryEntry? entry, + SourceOnboardingRegistry? registry = null) => + TryGetBySourceFamily(sourceFamily.ToWireName(), out entry, registry); + + public ContractValidationResult Validate() + { + var errors = new List(); + var sourceIds = new HashSet(StringComparer.Ordinal); + var sourceFamilies = new HashSet(StringComparer.Ordinal); + var documentIds = new HashSet(StringComparer.Ordinal); + + for (var index = 0; index < Entries.Count; index++) + { + var entry = Entries[index]; + + if (entry is null) + { + errors.Add($"Entries[{index}] is required."); + continue; + } + + ValidateEntry(entry, errors, sourceIds, sourceFamilies, documentIds); + } + + ValidateOrdering(errors); + + return ContractValidationResult.FromErrors(errors); + } + + private static string GetDisplayName(SourceFamily sourceFamily) => + sourceFamily switch + { + SourceFamily.GhgProtocol => "GHG Protocol", + SourceFamily.DefraDesnz => "DEFRA/DESNZ", + SourceFamily.IpccEfdb => "IPCC EFDB", + _ => throw new ArgumentOutOfRangeException(nameof(sourceFamily), sourceFamily, "Unknown source family."), + }; + + private static void ValidateEntry( + SourceOnboardingRegistryEntry entry, + ICollection errors, + ISet sourceIds, + ISet sourceFamilies, + ISet documentIds) + { + AddRequiredError(errors, entry.SourceId, "SourceId"); + + AddRequiredError(errors, entry.SourceFamily, "SourceFamily"); + + AddRequiredError(errors, entry.DisplayName, "DisplayName"); + + if (!string.IsNullOrWhiteSpace(entry.SourceId) && !sourceIds.Add(entry.SourceId)) + { + errors.Add($"Duplicate SourceId found: {entry.SourceId}"); + } + + if (!string.IsNullOrWhiteSpace(entry.SourceFamily) && !sourceFamilies.Add(entry.SourceFamily)) + { + errors.Add($"Duplicate SourceFamily found: {entry.SourceFamily}"); + } + + if (entry.Documents.Count == 0) + { + errors.Add($"Documents must include at least one document for SourceId '{entry.SourceId}'."); + } + + ValidateDocumentOrdering(entry, errors); + + foreach (var document in entry.Documents) + { + ValidateDocument(entry, document, errors, documentIds); + } + + if (!Enum.IsDefined(entry.DiscoveryStrategy)) + { + errors.Add("DiscoveryStrategy must be a defined source onboarding discovery strategy."); + } + + if (!Enum.IsDefined(entry.UpdateCadence)) + { + errors.Add("UpdateCadence must be a defined source onboarding update cadence."); + } + + ValidateParserCapability(entry, errors); + ValidateValidationExpectations(entry, errors); + ValidateRuntimeSafety(entry, errors); + } + + private static void ValidateDocument( + SourceOnboardingRegistryEntry entry, + SourceOnboardingDocument document, + ICollection errors, + ISet documentIds) + { + if (document is null) + { + errors.Add($"Documents for SourceId '{entry.SourceId}' must not contain null entries."); + return; + } + + AddRequiredError(errors, document.DocumentId, "DocumentId"); + AddRequiredError(errors, document.DisplayName, "Document.DisplayName"); + AddRequiredError(errors, document.SourceReference, "SourceReference"); + AddRequiredError(errors, document.ExpectedFormat, "ExpectedFormat"); + + if (!string.IsNullOrWhiteSpace(document.DocumentId) && !documentIds.Add(document.DocumentId)) + { + errors.Add($"Duplicate DocumentId found: {document.DocumentId}"); + } + } + + private static void ValidateParserCapability( + SourceOnboardingRegistryEntry entry, + ICollection errors) + { + if (entry.ParserCapability is null) + { + errors.Add($"ParserCapability is required for SourceId '{entry.SourceId}'."); + return; + } + + if (entry.ParserCapability.ParserKey is null || string.IsNullOrWhiteSpace(entry.ParserCapability.ParserKey.Value)) + { + errors.Add("ParserCapability.ParserKey is required."); + } + + if (!Enum.IsDefined(entry.ParserCapability.ParserSourceFormat)) + { + errors.Add("ParserCapability.ParserSourceFormat must be a defined parser source format."); + } + + AddRequiredError(errors, entry.ParserCapability.CapabilityNotes, "ParserCapability.CapabilityNotes"); + } + + private static void ValidateValidationExpectations( + SourceOnboardingRegistryEntry entry, + ICollection errors) + { + if (entry.ValidationExpectations is null) + { + errors.Add($"ValidationExpectations is required for SourceId '{entry.SourceId}'."); + return; + } + + if (entry.ValidationExpectations.RequiredDocumentFields.Count == 0) + { + errors.Add($"RequiredDocumentFields must not be empty for SourceId '{entry.SourceId}'."); + } + + foreach (var fieldName in entry.ValidationExpectations.RequiredDocumentFields) + { + AddRequiredError(errors, fieldName, "RequiredDocumentFields"); + } + + AddRequiredError(errors, entry.ValidationExpectations.ValidationNotes, "ValidationNotes"); + } + + private static void ValidateRuntimeSafety( + SourceOnboardingRegistryEntry entry, + ICollection errors) + { + if (entry.RuntimeSafety is null) + { + errors.Add($"RuntimeSafety is required for SourceId '{entry.SourceId}'."); + return; + } + + AddRequiredError(errors, entry.RuntimeSafety.SafetyNotes, "SafetyNotes"); + } + + private static void ValidateDocumentOrdering( + SourceOnboardingRegistryEntry entry, + ICollection errors) + { + var documentIds = entry.Documents + .Where(document => document is not null) + .Select(document => document.DocumentId) + .ToArray(); + + if (!documentIds.SequenceEqual(documentIds.OrderBy(documentId => documentId, StringComparer.Ordinal))) + { + errors.Add($"Documents must be ordered by DocumentId for SourceId '{entry.SourceId}'."); + } + } + + private void ValidateOrdering(ICollection errors) + { + var expectedOrder = Entries + .Where(entry => entry is not null) + .OrderBy(entry => SourceOrderIndex(entry.SourceFamily)) + .ThenBy(entry => entry.SourceId, StringComparer.Ordinal) + .ToArray(); + + if (!Entries.Where(entry => entry is not null).SequenceEqual(expectedOrder)) + { + errors.Add("Entries must follow Phase 1 source order, then SourceId order."); + } + } + + private static int SourceOrderIndex(string sourceFamily) + { + var index = Phase2OnboardingSourceFamilies + .Select((candidate, position) => new { candidate, position }) + .SingleOrDefault(item => item.candidate == sourceFamily) + ?.position ?? -1; + + return index >= 0 ? index : Phase2OnboardingSourceFamilies.Count; + } + + private static void AddRequiredError(ICollection errors, string? value, string fieldName) + { + if (string.IsNullOrWhiteSpace(value)) + { + errors.Add($"{fieldName} is required."); + } + } +} diff --git a/src/dotnet/CarbonOps.Parser.Service/CarbonOps.Parser.Service.csproj b/src/dotnet/CarbonOps.Parser.Service/CarbonOps.Parser.Service.csproj new file mode 100644 index 00000000..d4d832c1 --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Service/CarbonOps.Parser.Service.csproj @@ -0,0 +1,14 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + diff --git a/src/dotnet/CarbonOps.Parser.Service/Program.cs b/src/dotnet/CarbonOps.Parser.Service/Program.cs new file mode 100644 index 00000000..1a4eea1f --- /dev/null +++ b/src/dotnet/CarbonOps.Parser.Service/Program.cs @@ -0,0 +1,360 @@ +using System.Collections; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Service; + +public static class Program +{ + public static int Main(string[] args) => + CarbonOpsParserServiceCommand.Run( + args, + Console.Out, + Console.Error, + CarbonOpsParserServiceCommand.ReadProcessEnvironment()); +} + +public static class CarbonOpsParserServiceCommand +{ + public const int SuccessExitCode = 0; + public const int ValidationFailedExitCode = 2; + public const int NotImplementedExitCode = 3; + public const string NotImplementedStatus = "not_implemented"; + + public static int Run( + string[] args, + TextWriter output, + TextWriter error, + IReadOnlyDictionary environment) + { + var command = args.Length == 0 ? "help" : args[0].Trim(); + + if (IsHelp(command)) + { + WriteHelp(output); + return SuccessExitCode; + } + + if (string.Equals(command, "validate-config", StringComparison.OrdinalIgnoreCase)) + { + return ValidateConfig(args.Skip(1).ToArray(), output, environment); + } + + if (string.Equals(command, "validate-postgresql-runtime", StringComparison.OrdinalIgnoreCase)) + { + return ValidatePostgreSQLRuntime(args.Skip(1).ToArray(), output, environment); + } + + if (string.Equals(command, "preview-source-cycle", StringComparison.OrdinalIgnoreCase) || + string.Equals(command, "validate-source-cycle", StringComparison.OrdinalIgnoreCase)) + { + return PreviewSourceCycle(args.Skip(1).ToArray(), output, environment); + } + + if (string.Equals(command, "run-once", StringComparison.OrdinalIgnoreCase)) + { + return RunOnce(args.Skip(1).ToArray(), output, environment); + } + + error.WriteLine($"Unknown command: {command}"); + WriteHelp(error); + return ValidationFailedExitCode; + } + + private static int ValidateConfig( + string[] args, + TextWriter output, + IReadOnlyDictionary environment) + { + var commandOptions = ParseCommandOptions(args); + var result = LoadAndValidate(commandOptions.ConfigPath, environment, commandOptions.Issues); + + output.WriteLine(result.Validation.IsValid ? "status=ready" : "status=blocked"); + output.WriteLine($"config_file_loaded={result.Load.ConfigFileLoaded}"); + output.WriteLine($"environment_loaded={result.Load.EnvironmentLoaded}"); + output.WriteLine("postgresql_connection_opened=False"); + output.WriteLine("secret_values_printed=False"); + + foreach (var required in ProductionConfigBoundary.RequiredEnvironmentVariables) + { + if (ProductionConfigBoundary.SecretEnvironmentVariables.Contains(required, StringComparer.Ordinal)) + { + output.WriteLine($"{required}_present={HasText(result.Load.Values[required])}"); + output.WriteLine("postgresql_password_configured=" + HasText(result.Load.Values[required])); + } + else + { + output.WriteLine($"{required}_present={HasText(result.Load.Values[required])}"); + } + } + + WriteIssues(output, result.Validation.Issues); + + return result.Validation.IsValid ? SuccessExitCode : ValidationFailedExitCode; + } + + private static int ValidatePostgreSQLRuntime( + string[] args, + TextWriter output, + IReadOnlyDictionary environment) + { + var commandOptions = ParseCommandOptions(args); + var result = LoadAndValidate(commandOptions.ConfigPath, environment, commandOptions.Issues); + + output.WriteLine(result.Validation.IsValid ? "status=ready" : "status=blocked"); + output.WriteLine(".net_runtime_production_ready=False"); + output.WriteLine("project_level_production_ready=False"); + output.WriteLine("postgresql_connection_opened=False"); + output.WriteLine("postgresql_sql_executed=False"); + output.WriteLine("schema_bootstrap_available=True"); + output.WriteLine("year_state_available=True"); + output.WriteLine("source_download_implemented=False"); + output.WriteLine("parser_orchestration_implemented=False"); + output.WriteLine("source_specific_master_detail_insert_baseline=True"); + output.WriteLine("master_detail_insert_e2e_validated=False"); + output.WriteLine("production_ingestion_ready=False"); + + if (result.Validation.IsValid && + PostgreSQLRuntimeConnectionBoundary.TryCreateFromProductionConfig( + result.Load.Values, + out var settings, + out var runtimeIssues) && + settings is not null) + { + foreach (var diagnostic in PostgreSQLRuntimeConnectionBoundary.BuildSafeDiagnostics(settings)) + { + output.WriteLine($"{diagnostic.Key}={diagnostic.Value}"); + } + + output.WriteLine($"required_table_count={PostgreSQLRuntimeSchemaCatalog.RequiredTableNames.Count}"); + output.WriteLine("required_tables=" + string.Join(",", PostgreSQLRuntimeSchemaCatalog.RequiredTableNames)); + } + else + { + if (result.Validation.IsValid) + { + PostgreSQLRuntimeConnectionBoundary.TryCreateFromProductionConfig( + result.Load.Values, + out _, + out var runtimeValidationIssues); + foreach (var issue in runtimeValidationIssues) + { + var safeMessage = Phase1OperationalDiagnostics.RedactDiagnosticValue("message", issue.Message); + output.WriteLine( + $"issue={issue.Code} field={issue.FieldName} severity={issue.Severity} message={safeMessage}"); + } + } + } + + WriteIssues(output, result.Validation.Issues); + + return result.Validation.IsValid ? SuccessExitCode : ValidationFailedExitCode; + } + + private static int PreviewSourceCycle( + string[] args, + TextWriter output, + IReadOnlyDictionary environment) + { + var commandOptions = ParseCommandOptions(args); + var result = LoadAndValidate(commandOptions.ConfigPath, environment, commandOptions.Issues); + var artifacts = SourceCycleConfiguration.LoadArtifacts(commandOptions.ConfigPath); + var enabledSourceFamilies = SourceCycleConfiguration.LoadEnabledSourceFamilies(commandOptions.ConfigPath); + var yearStateRepository = new PostgreSQLSourceFamilyYearStateRepository( + new PreviewSourceFamilyYearStateSession(), + PostgreSQLSourceFamilyYearStateRepository.DefaultInitialYear); + var orchestrator = new SourceCycleOrchestrator( + yearStateRepository, + enabledSourceFamilies, + artifacts); + var preview = orchestrator.PreviewAsync().GetAwaiter().GetResult(); + + output.WriteLine(result.Validation.IsValid ? "status=ready" : "status=blocked"); + output.WriteLine(".net_runtime_production_ready=False"); + output.WriteLine("project_level_production_ready=False"); + output.WriteLine("source_cycle_orchestration_available=True"); + output.WriteLine("source_discovery_load_behavior=configured_local_artifacts_only"); + output.WriteLine("live_network_access_supported=False"); + output.WriteLine("postgresql_connection_opened=False"); + output.WriteLine("postgresql_sql_executed=False"); + output.WriteLine("records_inserted=0"); + output.WriteLine("year_state_advanced=False"); + output.WriteLine("secret_values_printed=False"); + output.WriteLine("source_specific_master_detail_insert_baseline=True"); + output.WriteLine("master_detail_insert_e2e_validated=False"); + output.WriteLine("production_ingestion_ready=False"); + output.WriteLine("persistence_status=persistence_not_implemented"); + output.WriteLine($"source_family_run_count={preview.RunCount}"); + + foreach (var run in preview.Runs.OrderBy(item => item.SourceFamily.ToWireName(), StringComparer.Ordinal)) + { + var prefix = $"source_family={run.SourceFamily.ToWireName()}"; + output.WriteLine( + $"{prefix} latest_successful_year={FormatNullableYear(run.LatestSuccessfulYear)} target_year={run.TargetYear} " + + $"status={run.Status.ToWireName()} parser_status={run.ParserStatus.ToWireName()} " + + $"persistence_status={run.PersistenceStatus.ToWireName()} artifact_count={run.ArtifactCount} " + + $"parsed_row_count={run.ParsedRowCount} parser_issue_count={run.ParserIssueCount}"); + } + + if (!result.Validation.IsValid) + { + output.WriteLine("config_status=blocked"); + WriteIssues(output, result.Validation.Issues); + return ValidationFailedExitCode; + } + + output.WriteLine("config_status=ready"); + return SuccessExitCode; + } + + private static int RunOnce( + string[] args, + TextWriter output, + IReadOnlyDictionary environment) + { + var commandOptions = ParseCommandOptions(args); + var result = LoadAndValidate(commandOptions.ConfigPath, environment, commandOptions.Issues); + + output.WriteLine("status=blocked"); + output.WriteLine($"ingestion_status={NotImplementedStatus}"); + output.WriteLine("postgresql_connection_opened=False"); + output.WriteLine("records_inserted=0"); + output.WriteLine("message=.NET ingestion execution is not implemented in PROD-004."); + + if (!result.Validation.IsValid) + { + output.WriteLine("config_status=blocked"); + WriteIssues(output, result.Validation.Issues); + } + else + { + output.WriteLine("config_status=ready"); + } + + return NotImplementedExitCode; + } + + private static ProductionConfigCommandValidation LoadAndValidate( + string? configPath, + IReadOnlyDictionary environment, + IReadOnlyList commandIssues) + { + var load = ProductionConfigLoader.Load(configPath, environment); + var issues = new List(); + issues.AddRange(commandIssues); + issues.AddRange(load.Issues); + issues.AddRange(ProductionConfigBoundary.Validate(load.Values).Issues); + + return new ProductionConfigCommandValidation( + load, + new ProductionConfigValidationResult(issues)); + } + + private static ProductionConfigCommandOptions ParseCommandOptions(string[] args) + { + var issues = new List(); + string? configPath = null; + + for (var index = 0; index < args.Length; index++) + { + var arg = args[index]; + if (string.Equals(arg, "--config", StringComparison.OrdinalIgnoreCase)) + { + if (index + 1 >= args.Length || string.IsNullOrWhiteSpace(args[index + 1])) + { + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_COMMAND_MISSING_CONFIG_PATH", + "--config requires a file path.", + "config")); + continue; + } + + configPath = args[++index]; + continue; + } + + issues.Add(new ProductionConfigValidationIssue( + "PRODUCTION_CONFIG_COMMAND_UNKNOWN_ARGUMENT", + "Unknown command argument.", + "argument")); + } + + return new ProductionConfigCommandOptions( + configPath, + Array.AsReadOnly(issues.ToArray())); + } + + public static IReadOnlyDictionary ReadProcessEnvironment() + { + var values = new Dictionary(StringComparer.Ordinal); + + foreach (DictionaryEntry entry in Environment.GetEnvironmentVariables()) + { + if (entry.Key is string key) + { + values[key] = entry.Value?.ToString(); + } + } + + return values; + } + + private static void WriteIssues( + TextWriter output, + IReadOnlyList issues) + { + foreach (var issue in issues) + { + var safeMessage = Phase1OperationalDiagnostics.RedactDiagnosticValue("message", issue.Message); + output.WriteLine( + $"issue={issue.Code} field={issue.FieldName} severity={issue.Severity} message={safeMessage}"); + } + } + + private static bool IsHelp(string command) => + string.Equals(command, "help", StringComparison.OrdinalIgnoreCase) || + string.Equals(command, "--help", StringComparison.OrdinalIgnoreCase) || + string.Equals(command, "-h", StringComparison.OrdinalIgnoreCase); + + private static bool HasText(string? value) => !string.IsNullOrWhiteSpace(value); + + private static void WriteHelp(TextWriter writer) + { + writer.WriteLine("CarbonOps.Parser.Service"); + writer.WriteLine(); + writer.WriteLine("Usage:"); + writer.WriteLine(" dotnet run --project src/dotnet/CarbonOps.Parser.Service -- [--config ]"); + writer.WriteLine(); + writer.WriteLine("Commands:"); + writer.WriteLine(" help Show this command surface."); + writer.WriteLine(" validate-config Validate required .NET runtime configuration shape without opening PostgreSQL."); + writer.WriteLine(" validate-postgresql-runtime Report .NET PostgreSQL schema/year-state readiness without opening PostgreSQL."); + writer.WriteLine(" preview-source-cycle Preview source-family target years and local parser handoff without PostgreSQL writes."); + writer.WriteLine(" validate-source-cycle Alias for preview-source-cycle."); + writer.WriteLine(" run-once Run one scheduled-worker cycle placeholder; fails closed until .NET ingestion is implemented."); + } + + private static string FormatNullableYear(int? value) => + value?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "none"; + + private sealed record ProductionConfigCommandOptions( + string? ConfigPath, + IReadOnlyList Issues); + + private sealed record ProductionConfigCommandValidation( + ProductionConfigLoadResult Load, + ProductionConfigValidationResult Validation); + + private sealed class PreviewSourceFamilyYearStateSession : IPostgreSQLSourceFamilyYearStateSession + { + public Task LatestSuccessfulYearAsync( + SourceFamily sourceFamily, + CancellationToken cancellationToken = default) => + Task.FromResult(null); + + public Task RecordSuccessfulYearAsync( + SourceFamily sourceFamily, + int ingestedYear, + CancellationToken cancellationToken = default) => + throw new NotSupportedException("Source-cycle preview must not advance year-state."); + } +} diff --git a/src/dotnet/CarbonOps.Parser.sln b/src/dotnet/CarbonOps.Parser.sln index b3378bb4..dc94d72f 100644 --- a/src/dotnet/CarbonOps.Parser.sln +++ b/src/dotnet/CarbonOps.Parser.sln @@ -7,6 +7,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CarbonOps.Parser.Contracts" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CarbonOps.Parser.Contracts.Tests", "..\..\tests\dotnet\CarbonOps.Parser.Contracts.Tests\CarbonOps.Parser.Contracts.Tests.csproj", "{F3B9FC9B-48B1-489A-A042-8C97A02DFEE9}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CarbonOps.Parser.Service", "CarbonOps.Parser.Service\CarbonOps.Parser.Service.csproj", "{168E9BE0-30B8-442D-8FC2-C9F23DE4A969}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -24,5 +26,9 @@ Global {F3B9FC9B-48B1-489A-A042-8C97A02DFEE9}.Debug|Any CPU.Build.0 = Debug|Any CPU {F3B9FC9B-48B1-489A-A042-8C97A02DFEE9}.Release|Any CPU.ActiveCfg = Release|Any CPU {F3B9FC9B-48B1-489A-A042-8C97A02DFEE9}.Release|Any CPU.Build.0 = Release|Any CPU + {168E9BE0-30B8-442D-8FC2-C9F23DE4A969}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {168E9BE0-30B8-442D-8FC2-C9F23DE4A969}.Debug|Any CPU.Build.0 = Debug|Any CPU + {168E9BE0-30B8-442D-8FC2-C9F23DE4A969}.Release|Any CPU.ActiveCfg = Release|Any CPU + {168E9BE0-30B8-442D-8FC2-C9F23DE4A969}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/src/dotnet/README.md b/src/dotnet/README.md index 11cfc625..aaa92765 100644 --- a/src/dotnet/README.md +++ b/src/dotnet/README.md @@ -4,6 +4,131 @@ The .NET implementation is an independent Worker Service implementation option f It should follow the same conceptual ingestion workflow as the Python implementation while using .NET-appropriate project structure and typed application architecture. +## Current Entry Point + +`CarbonOps.Parser.Service` is the first scheduled-worker entrypoint baseline for +the .NET runtime. It is a console-style executable intended for direct operator +execution and cron/manual scheduling of a single cycle. + +Command shape: + +```bash +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- help +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- validate-config +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- validate-config --config /etc/carbonops-parser/dotnet.production.json +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- validate-postgresql-runtime --config /etc/carbonops-parser/dotnet.production.json +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- preview-source-cycle --config /etc/carbonops-parser/dotnet.production.json +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- validate-source-cycle --config /etc/carbonops-parser/dotnet.production.json +dotnet run --project src/dotnet/CarbonOps.Parser.Service -- run-once +``` + +`validate-config` now loads production configuration through the shared .NET +production config loader. It accepts an optional explicit JSON config file and +the process environment, then deterministically lets `CARBONOPS_PARSER_*` +environment values override file values. The command validates required key +presence and basic shape, reports secret presence, and redacts diagnostics. It +does not open PostgreSQL, run SQL, or print secret values. + +`validate-postgresql-runtime` validates the same explicit configuration and +reports the .NET PostgreSQL schema/year-state baseline without opening +PostgreSQL or running SQL. It reports that schema bootstrap and year-state +primitives exist and that the source-specific master/detail insert baseline is +available through `source_specific_master_detail_insert_baseline=True`. It also +reports `master_detail_insert_e2e_validated=False` and +`production_ingestion_ready=False` because this command is not the opt-in +Docker PostgreSQL E2E path or the Python/.NET persisted parity validation path. +Production source download and service run-once ingestion execution are still +false in this PostgreSQL validation command. + +`preview-source-cycle` and `validate-source-cycle` provide the PROD-006 safe +source-cycle orchestration baseline. They select enabled source families from +the optional JSON config, calculate each target year through the year-state +contract with default initial year `2024`, check configured local artifacts, and +hand supported local CSV/text artifacts to the existing normalized parser +contracts. The commands do not open PostgreSQL, run SQL, insert source-specific +master/detail records, update year-state, print secrets, or make uncontrolled +network calls. Missing configured target-year artifacts report +`no_available_source_year`; unsupported artifact shapes report +`parser_not_available`; completed parser handoff reports `parsed` and +`persistence_not_implemented`. + +The optional .NET source-cycle config extension is intentionally narrow: + +```json +{ + "enabled_source_families": ["ghg_protocol", "defra_desnz", "ipcc_efdb"], + "source_artifacts": { + "ghg_protocol": { + "2024": { + "path": "/var/lib/carbonops-parser/sources/ghg_protocol_2024.csv", + "content_type": "text/csv", + "extension": ".csv", + "version_label": "v1" + } + } + } +} +``` + +The .NET contracts project now includes an explicit Npgsql runtime boundary for +additive schema bootstrap and source-family year-state behavior. Construction +and validation do not connect to PostgreSQL. DB connections are opened only by +explicit runtime methods, and diagnostics redact passwords and connection +strings. The schema bootstrap DDL is limited to `CREATE TABLE IF NOT EXISTS` +and `CREATE INDEX IF NOT EXISTS` style statements for the shared/source-family +Phase 1 tables. + +The .NET contracts project also includes a PROD-007 source-specific +master/detail insert boundary. It maps normalized parser output into +`ghg_emission_factor_masters/details`, `defra_emission_factor_masters/details`, +and `ipcc_emission_factor_masters/details`, uses additive +`INSERT ... ON CONFLICT DO NOTHING` SQL, returns explicit inserted/skipped +duplicate/validation-failed counts, and updates source-family year-state only +inside the successful source-family persistence transaction. This is a baseline +only; the opt-in Docker PostgreSQL E2E path exercises it with local fixtures, +and Python/.NET persisted parity validation is covered by a separate opt-in +path. Service run-once ingestion execution remains separately scoped. + +PROD-009 extends the opt-in .NET Docker PostgreSQL E2E validation path in the +contract tests. The default .NET test suite does not open PostgreSQL. Enable the +E2E path only with `CARBONOPS_RUN_DOTNET_POSTGRESQL_INTEGRATION=1` and +externally supplied PostgreSQL settings: + +```bash +export CARBONOPS_RUN_DOTNET_POSTGRESQL_INTEGRATION=1 +export CARBONOPS_DOTNET_POSTGRESQL_TEST_DSN='' + +dotnet test tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj \ + --configuration Release \ + --no-restore \ + --filter "FullyQualifiedName~DotNetPostgreSQLIntegrationE2ETests" +``` + +The tests also accept split `CARBONOPS_PARSER_POSTGRES_*` settings through the +existing .NET production config boundary. They create a generated +PostgreSQL-safe schema name for deterministic idempotency checks and do not +print DSNs, passwords, or connection strings. The E2E path bootstraps schema +idempotently, parses the checked-in GHG Protocol, DEFRA/DESNZ, and IPCC EFDB +local CSV fixtures, writes source-specific master/detail rows, reruns the same +source-family/year/artifact to verify duplicate skips, verifies successful +2024 year-state progression to target year 2025, verifies +`no_available_source_year` does not advance state, and verifies transaction +rollback keeps year-state unchanged when persistence fails. + +This validation is still not a production ingestion command. `run-once` remains +fail-closed, no live network source access is enabled, Python/.NET persisted +parity validation is separate, and final project-level readiness is limited to +the supported scope in the verdict document. + +`run-once` is intentionally fail-closed. It may reuse the same config +validation boundary, but it reports +`ingestion_status=not_implemented`, opens no PostgreSQL connection, inserts no +records, and returns a non-zero exit code until later .NET parity tasks +implement PostgreSQL orchestration, source behavior, and inserts. + +This entrypoint by itself does not make `run-once` a production ingestion +command. + ## Role The .NET path should focus on: @@ -22,4 +147,9 @@ The .NET path should focus on: The .NET implementation should not depend on the Python implementation. -This documentation baseline does not add Worker Service code, source ingestion logic, database runtime behavior, or external dependencies. +PROD-004 adds only the .NET production config loader and redaction baseline on +top of the scheduled-worker command surface. PROD-005 adds only the .NET +PostgreSQL schema bootstrap/year-state item from the production parity map. +PROD-006 adds only the .NET source discovery/load/parsing orchestration item. +PROD-007 adds only the .NET source-specific master/detail insert runtime item. +It does not complete .NET production ingestion. diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj index fec2f8eb..4794d1c3 100644 --- a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOps.Parser.Contracts.Tests.csproj @@ -24,6 +24,7 @@ + diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOpsParserServiceCommandTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOpsParserServiceCommandTests.cs new file mode 100644 index 00000000..6702d6af --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/CarbonOpsParserServiceCommandTests.cs @@ -0,0 +1,372 @@ +using CarbonOps.Parser.Service; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class CarbonOpsParserServiceCommandTests +{ + [Fact] + public void HelpDocumentsScheduledWorkerCommandSurface() + { + var output = new StringWriter(); + var error = new StringWriter(); + + var exitCode = CarbonOpsParserServiceCommand.Run(["help"], output, error, ValidEnvironment()); + + Assert.Equal(0, exitCode); + var rendered = output.ToString(); + Assert.Contains("validate-config", rendered, StringComparison.Ordinal); + Assert.Contains("validate-postgresql-runtime", rendered, StringComparison.Ordinal); + Assert.Contains("preview-source-cycle", rendered, StringComparison.Ordinal); + Assert.Contains("validate-source-cycle", rendered, StringComparison.Ordinal); + Assert.Contains("run-once", rendered, StringComparison.Ordinal); + Assert.Contains("scheduled-worker", rendered, StringComparison.Ordinal); + Assert.Equal(string.Empty, error.ToString()); + } + + [Fact] + public void ValidateConfigReportsPresenceWithoutOpeningPostgreSqlOrPrintingSecrets() + { + var output = new StringWriter(); + var environment = ValidEnvironment(); + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["validate-config"], + output, + TextWriter.Null, + environment); + + Assert.Equal(0, exitCode); + var rendered = output.ToString(); + Assert.Contains("status=ready", rendered, StringComparison.Ordinal); + Assert.Contains("postgresql_connection_opened=False", rendered, StringComparison.Ordinal); + Assert.Contains("CARBONOPS_PARSER_POSTGRES_PASSWORD_present=True", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("runtime-secret-not-returned", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("Password=raw-secret", rendered, StringComparison.Ordinal); + } + + [Fact] + public void ValidateConfigAcceptsConfigFileOnlyValidShape() + { + var output = new StringWriter(); + var configPath = WriteConfigFile(ValidEnvironment()); + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["validate-config", "--config", configPath], + output, + TextWriter.Null, + new Dictionary()); + + Assert.Equal(0, exitCode); + var rendered = output.ToString(); + Assert.Contains("status=ready", rendered, StringComparison.Ordinal); + Assert.Contains("config_file_loaded=True", rendered, StringComparison.Ordinal); + Assert.Contains("environment_loaded=True", rendered, StringComparison.Ordinal); + Assert.Contains("postgresql_connection_opened=False", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("runtime-secret-not-returned", rendered, StringComparison.Ordinal); + } + + [Fact] + public void ValidatePostgreSqlRuntimeReportsReadinessWithoutClaimingProductionReady() + { + var output = new StringWriter(); + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["validate-postgresql-runtime"], + output, + TextWriter.Null, + ValidEnvironment()); + + Assert.Equal(0, exitCode); + var rendered = output.ToString(); + Assert.Contains("status=ready", rendered, StringComparison.Ordinal); + Assert.Contains(".net_runtime_production_ready=False", rendered, StringComparison.Ordinal); + Assert.Contains("project_level_production_ready=False", rendered, StringComparison.Ordinal); + Assert.Contains("postgresql_connection_opened=False", rendered, StringComparison.Ordinal); + Assert.Contains("schema_bootstrap_available=True", rendered, StringComparison.Ordinal); + Assert.Contains("year_state_available=True", rendered, StringComparison.Ordinal); + Assert.Contains("source_download_implemented=False", rendered, StringComparison.Ordinal); + Assert.Contains("parser_orchestration_implemented=False", rendered, StringComparison.Ordinal); + Assert.Contains("source_specific_master_detail_insert_baseline=True", rendered, StringComparison.Ordinal); + Assert.Contains("master_detail_insert_e2e_validated=False", rendered, StringComparison.Ordinal); + Assert.Contains("production_ingestion_ready=False", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("master_detail_inserts_implemented=True", rendered, StringComparison.Ordinal); + Assert.Contains("source_family_year_states", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("runtime-secret-not-returned", rendered, StringComparison.Ordinal); + } + + [Fact] + public void ValidatePostgreSqlRuntimeFailsClosedWhenConfigIsMissing() + { + var output = new StringWriter(); + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["validate-postgresql-runtime"], + output, + TextWriter.Null, + new Dictionary()); + + Assert.Equal(2, exitCode); + var rendered = output.ToString(); + Assert.Contains("status=blocked", rendered, StringComparison.Ordinal); + Assert.Contains(".net_runtime_production_ready=False", rendered, StringComparison.Ordinal); + Assert.Contains("postgresql_connection_opened=False", rendered, StringComparison.Ordinal); + Assert.Contains("PRODUCTION_CONFIG_MISSING_REQUIRED_ENV_VAR", rendered, StringComparison.Ordinal); + } + + [Fact] + public void PreviewSourceCycleReportsAllFamiliesWithoutOpeningPostgreSqlOrPrintingSecrets() + { + var output = new StringWriter(); + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["preview-source-cycle"], + output, + TextWriter.Null, + ValidEnvironment()); + + Assert.Equal(0, exitCode); + var rendered = output.ToString(); + Assert.Contains("status=ready", rendered, StringComparison.Ordinal); + Assert.Contains(".net_runtime_production_ready=False", rendered, StringComparison.Ordinal); + Assert.Contains("project_level_production_ready=False", rendered, StringComparison.Ordinal); + Assert.Contains("source_cycle_orchestration_available=True", rendered, StringComparison.Ordinal); + Assert.Contains("source_family_run_count=3", rendered, StringComparison.Ordinal); + Assert.Contains("source_family=ghg_protocol", rendered, StringComparison.Ordinal); + Assert.Contains("source_family=defra_desnz", rendered, StringComparison.Ordinal); + Assert.Contains("source_family=ipcc_efdb", rendered, StringComparison.Ordinal); + Assert.Contains("target_year=2024", rendered, StringComparison.Ordinal); + Assert.Contains("status=no_available_source_year", rendered, StringComparison.Ordinal); + Assert.Contains("postgresql_connection_opened=False", rendered, StringComparison.Ordinal); + Assert.Contains("postgresql_sql_executed=False", rendered, StringComparison.Ordinal); + Assert.Contains("records_inserted=0", rendered, StringComparison.Ordinal); + Assert.Contains("year_state_advanced=False", rendered, StringComparison.Ordinal); + Assert.Contains("secret_values_printed=False", rendered, StringComparison.Ordinal); + Assert.Contains("source_specific_master_detail_insert_baseline=True", rendered, StringComparison.Ordinal); + Assert.Contains("master_detail_insert_e2e_validated=False", rendered, StringComparison.Ordinal); + Assert.Contains("production_ingestion_ready=False", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("master_detail_inserts_implemented=True", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("runtime-secret-not-returned", rendered, StringComparison.Ordinal); + } + + [Fact] + public void PreviewSourceCycleCanRepresentConfiguredLocalParserHandoff() + { + var output = new StringWriter(); + var configPath = WriteSourceCycleConfig(ValidEnvironment(), FixturePath("ghg_protocol", "ghg_protocol_sample_factors.csv")); + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["validate-source-cycle", "--config", configPath], + output, + TextWriter.Null, + new Dictionary()); + + Assert.Equal(0, exitCode); + var rendered = output.ToString(); + Assert.Contains("source_family=ghg_protocol", rendered, StringComparison.Ordinal); + Assert.Contains("status=persistence_not_implemented", rendered, StringComparison.Ordinal); + Assert.Contains("parser_status=parsed", rendered, StringComparison.Ordinal); + Assert.Contains("parsed_row_count=2", rendered, StringComparison.Ordinal); + Assert.Contains("records_inserted=0", rendered, StringComparison.Ordinal); + Assert.Contains("year_state_advanced=False", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("runtime-secret-not-returned", rendered, StringComparison.Ordinal); + } + + [Fact] + public void ValidateConfigEnvironmentOverridesConfigFile() + { + var output = new StringWriter(); + var fileValues = ValidEnvironment(); + fileValues["CARBONOPS_PARSER_POSTGRES_PORT"] = "70000"; + var configPath = WriteConfigFile(fileValues); + var environment = new Dictionary(StringComparer.Ordinal) + { + ["CARBONOPS_PARSER_POSTGRES_PORT"] = "5432", + }; + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["validate-config", "--config", configPath], + output, + TextWriter.Null, + environment); + + Assert.Equal(0, exitCode); + var rendered = output.ToString(); + Assert.Contains("status=ready", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("PRODUCTION_CONFIG_INVALID_POSTGRES_PORT", rendered, StringComparison.Ordinal); + } + + [Fact] + public void ValidateConfigFailsClosedForMissingValuesWithoutPrintingSecrets() + { + var output = new StringWriter(); + var environment = ValidEnvironment(); + environment["CARBONOPS_PARSER_POSTGRES_PASSWORD"] = ""; + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["validate-config"], + output, + TextWriter.Null, + environment); + + Assert.Equal(2, exitCode); + var rendered = output.ToString(); + Assert.Contains("status=blocked", rendered, StringComparison.Ordinal); + Assert.Contains("PRODUCTION_CONFIG_MISSING_REQUIRED_ENV_VAR", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("runtime-secret-not-returned", rendered, StringComparison.Ordinal); + } + + [Fact] + public void ValidateConfigFailsClosedForInvalidPort() + { + var output = new StringWriter(); + var environment = ValidEnvironment(); + environment["CARBONOPS_PARSER_POSTGRES_PORT"] = "not-a-port"; + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["validate-config"], + output, + TextWriter.Null, + environment); + + Assert.Equal(2, exitCode); + var rendered = output.ToString(); + Assert.Contains("status=blocked", rendered, StringComparison.Ordinal); + Assert.Contains("issue=PRODUCTION_CONFIG_INVALID_POSTGRES_PORT", rendered, StringComparison.Ordinal); + Assert.Contains("field=CARBONOPS_PARSER_POSTGRES_PORT", rendered, StringComparison.Ordinal); + Assert.Contains("severity=error", rendered, StringComparison.Ordinal); + } + + [Fact] + public void ValidateConfigRedactsRawConnectionStringWithCredentials() + { + var output = new StringWriter(); + var environment = ValidEnvironment(); + environment["CARBONOPS_PARSER_POSTGRES_CONNECTION_STRING"] = + "postgresql://carbonops_runtime:raw-secret@db.internal.example/carbonops_parser"; + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["validate-config"], + output, + TextWriter.Null, + environment); + + Assert.Equal(2, exitCode); + var rendered = output.ToString(); + Assert.Contains("PRODUCTION_CONFIG_RAW_CONNECTION_STRING_NOT_ALLOWED", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("raw-secret", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("carbonops_runtime:raw-secret", rendered, StringComparison.Ordinal); + } + + [Fact] + public void RunOnceFailsClosedUntilDotNetIngestionIsImplemented() + { + var output = new StringWriter(); + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["run-once"], + output, + TextWriter.Null, + ValidEnvironment()); + + Assert.Equal(3, exitCode); + var rendered = output.ToString(); + Assert.Contains("status=blocked", rendered, StringComparison.Ordinal); + Assert.Contains("ingestion_status=not_implemented", rendered, StringComparison.Ordinal); + Assert.Contains("postgresql_connection_opened=False", rendered, StringComparison.Ordinal); + Assert.Contains("records_inserted=0", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("runtime-secret-not-returned", rendered, StringComparison.Ordinal); + } + + [Fact] + public void RunOnceAcceptsConfigOptionButRemainsNotImplementedAndNonZero() + { + var output = new StringWriter(); + var configPath = WriteConfigFile(ValidEnvironment()); + + var exitCode = CarbonOpsParserServiceCommand.Run( + ["run-once", "--config", configPath], + output, + TextWriter.Null, + new Dictionary()); + + Assert.Equal(3, exitCode); + var rendered = output.ToString(); + Assert.Contains("status=blocked", rendered, StringComparison.Ordinal); + Assert.Contains("ingestion_status=not_implemented", rendered, StringComparison.Ordinal); + Assert.Contains("config_status=ready", rendered, StringComparison.Ordinal); + Assert.Contains("postgresql_connection_opened=False", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("runtime-secret-not-returned", rendered, StringComparison.Ordinal); + } + + private static Dictionary ValidEnvironment() => new() + { + ["CARBONOPS_PARSER_ENV"] = "production", + ["CARBONOPS_PARSER_DATABASE_PROVIDER"] = "postgres", + ["CARBONOPS_PARSER_POSTGRES_HOST"] = "db.internal.example", + ["CARBONOPS_PARSER_POSTGRES_PORT"] = "5432", + ["CARBONOPS_PARSER_POSTGRES_DATABASE"] = "carbonops_parser", + ["CARBONOPS_PARSER_POSTGRES_USERNAME"] = "carbonops_runtime", + ["CARBONOPS_PARSER_POSTGRES_PASSWORD"] = "runtime-secret-not-returned", + ["CARBONOPS_PARSER_POSTGRES_SCHEMA"] = "carbonops", + ["CARBONOPS_PARSER_RAW_ARCHIVE_PATH"] = "/var/lib/carbonops/raw", + ["CARBONOPS_PARSER_LOG_LEVEL"] = "info", + }; + + private static string WriteConfigFile(IReadOnlyDictionary values) + { + var path = Path.Combine(Path.GetTempPath(), $"carbonops-service-config-{Guid.NewGuid():N}.json"); + var json = System.Text.Json.JsonSerializer.Serialize(values); + File.WriteAllText(path, json); + return path; + } + + private static string WriteSourceCycleConfig(IReadOnlyDictionary values, string artifactPath) + { + var merged = new Dictionary(StringComparer.Ordinal); + foreach (var item in values) + { + merged[item.Key] = item.Value; + } + + merged["enabled_source_families"] = new[] { "ghg_protocol" }; + merged["source_artifacts"] = new Dictionary + { + ["ghg_protocol"] = new Dictionary + { + ["2024"] = new Dictionary + { + ["path"] = artifactPath, + ["content_type"] = "text/csv", + ["extension"] = ".csv", + ["version_label"] = "v1", + }, + }, + }; + + var path = Path.Combine(Path.GetTempPath(), $"carbonops-service-source-cycle-{Guid.NewGuid():N}.json"); + File.WriteAllText(path, System.Text.Json.JsonSerializer.Serialize(merged)); + return path; + } + + private static string FixturePath(string familyDirectory, string fileName) + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixtureDirectory = Path.Combine( + directory.FullName, + "tests", + "fixtures", + "source_documents", + familyDirectory); + if (Directory.Exists(fixtureDirectory)) + { + return Path.Combine(fixtureDirectory, fileName); + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate source document fixture directory."); + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ContractEnumTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ContractEnumTests.cs index 9e9f9810..449e4867 100644 --- a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ContractEnumTests.cs +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ContractEnumTests.cs @@ -31,5 +31,9 @@ public void RuntimeStatusEnumsExposeExpectedContractValues() Assert.Equal( ["Pending", "Running", "Completed", "Failed"], Enum.GetNames()); + + Assert.Equal( + ["Completed", "CompletedWithFailures", "Failed", "NotExecutable"], + Enum.GetNames()); } } diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DataQualityValidationTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DataQualityValidationTests.cs new file mode 100644 index 00000000..02c454b6 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DataQualityValidationTests.cs @@ -0,0 +1,309 @@ +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class DataQualityValidationTests +{ + [Fact] + public void ValidationModelRepresentsBlockingWarningAndInfo() + { + var blocking = DataQualityValidation.CreateDiagnostic( + "BLOCKING", + "blocking diagnostic", + DataQualityValidationSeverity.BlockingError, + DataQualityValidationCheck.RequiredField); + var warning = DataQualityValidation.CreateDiagnostic( + "WARNING", + "warning diagnostic", + DataQualityValidationSeverity.Warning, + DataQualityValidationCheck.Unit); + var info = DataQualityValidation.CreateDiagnostic( + "INFO", + "info diagnostic", + DataQualityValidationSeverity.Info, + DataQualityValidationCheck.Structure); + + var result = new DataQualityValidationResult([blocking, warning, info]); + + Assert.False(result.IsValid); + Assert.True(result.HasBlockingErrors); + Assert.Equal(1, result.BlockingErrorCount); + Assert.Equal(1, result.WarningCount); + Assert.Equal(1, result.InfoCount); + } + + [Fact] + public void ValidationSeverityAndCheckNamesArePhase2WireNames() + { + Assert.Equal("blocking_error", DataQualityValidationSeverity.BlockingError.ToWireName()); + Assert.Equal("warning", DataQualityValidationSeverity.Warning.ToWireName()); + Assert.Equal("info", DataQualityValidationSeverity.Info.ToWireName()); + + Assert.Equal("required_field", DataQualityValidationCheck.RequiredField.ToWireName()); + Assert.Equal("numeric_value", DataQualityValidationCheck.NumericValue.ToWireName()); + Assert.Equal("unit", DataQualityValidationCheck.Unit.ToWireName()); + Assert.Equal( + "duplicate_factor_identity", + DataQualityValidationCheck.DuplicateFactorIdentity.ToWireName()); + Assert.Equal("provenance", DataQualityValidationCheck.Provenance.ToWireName()); + Assert.Equal("structure", DataQualityValidationCheck.Structure.ToWireName()); + + Assert.True(ContractWireNames.TryParseDataQualityValidationSeverityWireName( + "blocking_error", + out var severity)); + Assert.Equal(DataQualityValidationSeverity.BlockingError, severity); + Assert.True(ContractWireNames.TryParseDataQualityValidationCheckWireName( + "duplicate_factor_identity", + out var check)); + Assert.Equal(DataQualityValidationCheck.DuplicateFactorIdentity, check); + } + + [Fact] + public void MissingRequiredFieldsAreBlockingErrors() + { + var row = CreateRow( + [ + new("source_family", null), + new("source_id", null), + new("factor_id", " "), + new("factor_name", null), + new("factor_value", null), + ]); + + var result = DataQualityValidation.ValidateNormalizedFactorOutput(new ParserNormalizedOutputBatch([row])); + + Assert.False(result.IsValid); + Assert.Equal( + [ + "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD", + "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD", + "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD", + "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD", + "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD", + ], + DiagnosticCodes(result)); + Assert.Equal( + [ + "factor_id", + "factor_name", + "factor_value", + "source_family", + "source_id", + ], + result.Diagnostics.Select(diagnostic => diagnostic.FieldName)); + Assert.All( + result.Diagnostics, + diagnostic => Assert.Equal(DataQualityValidationSeverity.BlockingError, diagnostic.Severity)); + } + + [Fact] + public void InvalidNumericValuesAreBlockingErrorsWithoutValueLeakage() + { + var row = CreateRow([new("factor_value", "not-a-number")]); + + var result = DataQualityValidation.ValidateNormalizedFactorOutput(new ParserNormalizedOutputBatch([row])); + + Assert.False(result.IsValid); + var diagnostic = Assert.Single(result.Diagnostics); + Assert.Equal("NORMALIZED_FACTOR_INVALID_NUMERIC_VALUE", diagnostic.Code); + Assert.Equal("factor_value", diagnostic.FieldName); + Assert.Equal(DataQualityValidationCheck.NumericValue, diagnostic.Check); + Assert.DoesNotContain("not-a-number", diagnostic.Message, StringComparison.Ordinal); + Assert.DoesNotContain("not-a-number", diagnostic.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void UnsupportedUnitsAreWarnings() + { + var row = CreateRow([new("unit", "widgets per fortnight")]); + + var result = DataQualityValidation.ValidateNormalizedFactorOutput( + new ParserNormalizedOutputBatch([row]), + supportedUnits: ["kg CO2e/kWh"]); + + Assert.True(result.IsValid); + Assert.Equal(1, result.WarningCount); + var diagnostic = Assert.Single(result.Diagnostics); + Assert.Equal("NORMALIZED_FACTOR_UNSUPPORTED_UNIT", diagnostic.Code); + Assert.Equal(DataQualityValidationSeverity.Warning, diagnostic.Severity); + Assert.Equal(DataQualityValidationCheck.Unit, diagnostic.Check); + } + + [Fact] + public void DuplicateFactorIdentityIsBlockingError() + { + var first = CreateRow(rowIdentifier: "row-1", fields: [new("factor_id", "F1")]); + var duplicate = CreateRow(rowIdentifier: "row-2", fields: [new("factor_id", "F1")]); + + var result = DataQualityValidation.ValidateNormalizedFactorOutput( + new ParserNormalizedOutputBatch([first, duplicate])); + + Assert.False(result.IsValid); + var diagnostic = Assert.Single(result.Diagnostics); + Assert.Equal("NORMALIZED_FACTOR_DUPLICATE_IDENTITY", diagnostic.Code); + Assert.Equal(DataQualityValidationCheck.DuplicateFactorIdentity, diagnostic.Check); + Assert.Equal("1", ContextValue(diagnostic, "first_record_position")); + Assert.Equal("2", ContextValue(diagnostic, "record_position")); + } + + [Fact] + public void ProvenanceGapsAreWarningsWithSafeSourceContext() + { + var row = CreateRow(artifactReference: "", sourceRowNumber: null, fields: []); + + var result = DataQualityValidation.ValidateNormalizedFactorOutput(new ParserNormalizedOutputBatch([row])); + + Assert.True(result.IsValid); + var diagnostic = Assert.Single(result.Diagnostics); + Assert.Equal("NORMALIZED_FACTOR_PROVENANCE_GAP", diagnostic.Code); + Assert.Equal("defra_desnz", diagnostic.SourceFamily); + Assert.Equal(DataQualityValidationSeverity.Warning, diagnostic.Severity); + Assert.Equal( + new DataQualityProvenanceContext( + "row-1", + "defra_desnz", + "defra_desnz", + sourceReference: null, + rowNumber: null, + provenance: null, + documentId: null), + diagnostic.Provenance); + Assert.Equal("1", ContextValue(diagnostic, "record_position")); + } + + [Fact] + public void DiagnosticsAreDeterministicallyOrderedByRecordThenCode() + { + var first = CreateRow( + rowIdentifier: "row-1", + fields: + [ + new("factor_id", ""), + new("factor_value", "bad"), + new("unit", "bad-unit"), + ]); + var second = CreateRow( + rowIdentifier: "row-2", + fields: + [ + new("factor_id", ""), + new("unit", "bad-unit"), + ]); + + var result = DataQualityValidation.ValidateNormalizedFactorOutput( + new ParserNormalizedOutputBatch([second, first])); + + Assert.Equal( + [ + ("1", "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD"), + ("1", "NORMALIZED_FACTOR_UNSUPPORTED_UNIT"), + ("2", "NORMALIZED_FACTOR_INVALID_NUMERIC_VALUE"), + ("2", "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD"), + ("2", "NORMALIZED_FACTOR_UNSUPPORTED_UNIT"), + ], + result.Diagnostics.Select(diagnostic => (ContextValue(diagnostic, "record_position"), diagnostic.Code))); + } + + [Fact] + public void SensitiveValuesAreRedactedFromDiagnosticContext() + { + var diagnostic = DataQualityValidation.CreateDiagnostic( + "SAFE_CONTEXT", + "safe context diagnostic", + DataQualityValidationSeverity.Info, + DataQualityValidationCheck.Structure, + context: + [ + new("api_key", "abc123"), + new("password", "secret-value"), + new("source_reference", "https://user:pass@example.invalid/?token=abc123"), + new("visible", "ok"), + ]); + + Assert.Equal(DataQualityValidation.RedactedDiagnosticValue, ContextValue(diagnostic, "api_key")); + Assert.Equal(DataQualityValidation.RedactedDiagnosticValue, ContextValue(diagnostic, "password")); + Assert.Equal( + "https://[REDACTED]@example.invalid/?token=[REDACTED]", + ContextValue(diagnostic, "source_reference")); + Assert.Equal("ok", ContextValue(diagnostic, "visible")); + Assert.DoesNotContain("abc123", diagnostic.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("pass", diagnostic.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("secret-value", diagnostic.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void SensitiveValuesAreRedactedFromProvenanceContext() + { + var row = CreateRow( + artifactReference: "https://user:pass@example.invalid/factors.csv?token=abc123", + sourceRowNumber: null, + fields: [new("unit", "unsupported")]); + + var result = DataQualityValidation.ValidateNormalizedFactorOutput(new ParserNormalizedOutputBatch([row])); + + var diagnostic = Assert.Single(result.Diagnostics); + Assert.NotNull(diagnostic.Provenance); + Assert.Equal( + "https://[REDACTED]@example.invalid/factors.csv?token=[REDACTED]", + diagnostic.Provenance!.SourceReference); + Assert.DoesNotContain("pass", diagnostic.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("abc123", diagnostic.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void ValidFactorOutputHasNoDiagnostics() + { + var result = DataQualityValidation.ValidateNormalizedFactorOutput( + new ParserNormalizedOutputBatch([CreateRow()])); + + Assert.True(result.IsValid); + Assert.Empty(result.Diagnostics); + } + + private static ParserNormalizedOutputRow CreateRow( + IEnumerable? fields = null, + string rowIdentifier = "row-1", + string artifactReference = "memory://defra", + int? sourceRowNumber = 2) + { + var values = new Dictionary(StringComparer.Ordinal) + { + ["source_family"] = "defra_desnz", + ["source_id"] = "defra_desnz", + ["source_year"] = "2024", + ["source_version"] = "v1", + ["row_number"] = sourceRowNumber?.ToString(), + ["factor_id"] = "F1", + ["factor_name"] = "Electricity", + ["factor_value"] = "0.233", + ["unit"] = "kg CO2e/kWh", + }; + + foreach (var field in fields ?? []) + { + if (field.Value is null) + { + values.Remove(field.Key); + } + else + { + values[field.Key] = field.Value; + } + } + + return new ParserNormalizedOutputRow( + SourceFamily.DefraDesnz, + "defra_desnz", + new ParserKey("defra_desnz"), + artifactReference, + rowIdentifier, + sourceRowNumber, + values.Select(pair => new ParserNormalizedField(pair.Key, pair.Value))); + } + + private static string[] DiagnosticCodes(DataQualityValidationResult result) => + result.Diagnostics.Select(diagnostic => diagnostic.Code).ToArray(); + + private static string? ContextValue(DataQualityDiagnostic diagnostic, string key) => + diagnostic.Context.FirstOrDefault(item => item.Key == key)?.Value; +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraDesnzNormalizedContentParserTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraDesnzNormalizedContentParserTests.cs new file mode 100644 index 00000000..55cb9ee1 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraDesnzNormalizedContentParserTests.cs @@ -0,0 +1,281 @@ +using System.Text.Json; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class DefraDesnzNormalizedContentParserTests +{ + [Fact] + public void DefraDesnzHeaderIsDeterministic() + { + using var expectations = LoadParityExpectations(); + + Assert.Equal( + JsonStringArray(expectations.RootElement.GetProperty("header")), + DefraDesnzNormalizedContentParser.Header); + } + + [Fact] + public void ValidDefraDesnzContentReturnsNormalizedRows() + { + var request = CreateRequest(); + var result = DefraDesnzNormalizedContentParser.Parse( + request, + CreateContentMap("defra_desnz_normalized_factors.csv")); + + Assert.Equal(ParserRunStatus.Completed, result.Status); + Assert.Equal(2, result.RowCount); + Assert.Empty(result.ValidationIssues); + Assert.True(result.Validate().IsValid); + Assert.All(result.Rows, row => Assert.True(row.Validate().IsValid)); + + var first = result.Rows[0]; + Assert.Equal(SourceFamily.DefraDesnz, first.SourceFamily); + Assert.Equal("defra_desnz", first.SourceKey); + Assert.Equal("defra_desnz_2024_conversion-factors-2024_DEFRA-2024-ELEC_row_2", first.RowIdentifier); + Assert.Equal(2, first.SourceRowNumber); + Assert.Equal(2024, first.ReportingYear); + Assert.Equal( + [ + new ParserNormalizedField("source_family", "defra_desnz"), + new ParserNormalizedField("source_year", "2024"), + new ParserNormalizedField("source_version", "conversion-factors-2024"), + new ParserNormalizedField("factor_id", "DEFRA-2024-ELEC"), + new ParserNormalizedField("factor_name", "Electricity generated"), + new ParserNormalizedField("factor_value", "0.20705"), + new ParserNormalizedField("unit", "kWh"), + new ParserNormalizedField("category", "Energy"), + new ParserNormalizedField("subcategory", "Electricity"), + new ParserNormalizedField("activity", "Generated"), + new ParserNormalizedField("greenhouse_gas", "CO2e"), + new ParserNormalizedField("provenance_artifact_reference", ArtifactReference), + new ParserNormalizedField("provenance_checksum_algorithm", "sha256"), + new ParserNormalizedField("provenance_checksum_value", ChecksumValue), + new ParserNormalizedField("provenance_row_number", "2"), + new ParserNormalizedField("provenance", "worksheet:UK electricity row 10"), + new ParserNormalizedField("source_family_master_id", "defra_master_2024_conversion-factors-2024_DEFRA-2024-ELEC"), + new ParserNormalizedField("source_family_detail_id", "defra_detail_2024_conversion-factors-2024_DEFRA-2024-ELEC"), + new ParserNormalizedField("master_external_key", "2024:conversion-factors-2024:DEFRA-2024-ELEC"), + new ParserNormalizedField("detail_external_key", "DEFRA-2024-ELEC:kWh:CO2e"), + ], + first.Fields); + } + + [Fact] + public void ValidDefraDesnzContentMatchesSharedParityExpectations() + { + using var expectations = LoadParityExpectations(); + var root = expectations.RootElement; + var request = CreateRequest(); + var result = DefraDesnzNormalizedContentParser.Parse( + request, + CreateContentMap("defra_desnz_normalized_factors.csv")); + var expectedRows = root.GetProperty("sample_rows").EnumerateArray().ToArray(); + + Assert.Equal(root.GetProperty("sample_status").GetProperty("dotnet").GetString(), result.Status.ToString()); + Assert.Equal( + JsonStringArray(root.GetProperty("sample_issue_codes")), + result.ValidationIssues.Select(issue => issue.Code)); + Assert.Equal(expectedRows.Length, result.RowCount); + + for (var index = 0; index < expectedRows.Length; index++) + { + var expected = expectedRows[index]; + var actual = result.Rows[index]; + + Assert.Equal(expected.GetProperty("row_identifier").GetString(), actual.RowIdentifier); + Assert.Equal(expected.GetProperty("source_row_number").GetInt32(), actual.SourceRowNumber); + Assert.Equal(expected.GetProperty("reporting_year").GetInt32(), actual.ReportingYear); + Assert.Equal(JsonFieldArray(expected.GetProperty("fields")), actual.Fields); + } + } + + [Fact] + public void DefraDesnzParserIsDeterministicForFixtureInput() + { + var request = CreateRequest(); + var content = CreateContentMap("defra_desnz_normalized_factors.csv"); + + var first = DefraDesnzNormalizedContentParser.Parse(request, content); + var second = DefraDesnzNormalizedContentParser.Parse(request, content); + + Assert.Equal(first, second); + Assert.Equal(2, first.RowCount); + } + + [Fact] + public void MalformedDefraDesnzRowsReturnStructuredErrors() + { + using var expectations = LoadParityExpectations(); + var root = expectations.RootElement; + var result = DefraDesnzNormalizedContentParser.Parse( + CreateRequest(), + CreateContentMap("defra_desnz_malformed_factors.csv")); + + Assert.Equal(root.GetProperty("malformed_status").GetProperty("dotnet").GetString(), result.Status.ToString()); + Assert.Equal(0, result.RowCount); + Assert.Equal( + root.GetProperty("malformed_issues").EnumerateArray().Select(issue => issue.GetProperty("code").GetString()), + result.ValidationIssues.Select(issue => issue.Code)); + Assert.Equal( + root.GetProperty("malformed_issues").EnumerateArray().Select(issue => issue.GetProperty("field_key").GetString()), + result.ValidationIssues.Select(issue => issue.FieldKey)); + Assert.Equal( + root.GetProperty("malformed_issues").EnumerateArray().Select(issue => (int?)issue.GetProperty("source_row_number").GetInt32()), + result.ValidationIssues.Select(issue => issue.SourceRowNumber)); + Assert.Equal("not-a-number", result.ValidationIssues[0].Context.Single(context => context.Key == "raw_value").Value); + } + + [Fact] + public void EmptyDefraDesnzContentMatchesDocumentedParityStatus() + { + using var expectations = LoadParityExpectations(); + var root = expectations.RootElement; + var content = string.Join("\n", [string.Join(",", DefraDesnzNormalizedContentParser.Header), string.Empty]); + + var result = DefraDesnzNormalizedContentParser.Parse( + CreateRequest(), + new Dictionary { [ArtifactReference] = content }); + + Assert.Equal(root.GetProperty("empty_status").GetProperty("dotnet").GetString(), result.Status.ToString()); + Assert.Equal(0, result.RowCount); + Assert.Equal( + JsonStringArray(root.GetProperty("empty_issue_codes")), + result.ValidationIssues.Select(issue => issue.Code)); + Assert.Equal(ParserValidationIssueSeverity.Warning, result.ValidationIssues[0].Severity); + } + + [Fact] + public void InvalidDefraDesnzHeaderReturnsFailedIssue() + { + var result = DefraDesnzNormalizedContentParser.Parse( + CreateRequest(), + new Dictionary { [ArtifactReference] = "source_year,wrong\n2024,value\n" }); + + Assert.Equal(ParserRunStatus.Failed, result.Status); + Assert.Equal("DEFRA_DESNZ_CONTENT_INVALID_HEADER", result.ValidationIssues[0].Code); + Assert.Equal("header", result.ValidationIssues[0].FieldKey); + } + + [Fact] + public void NonDefraSourceFamilyReturnsFailedIssue() + { + var parserKey = ParserSelectionRegistry.GetParserKey(SourceFamily.GhgProtocol); + var artifact = new ParserInputArtifact( + SourceFamily.GhgProtocol, + SourceFamily.GhgProtocol.ToWireName(), + parserKey, + ParserSourceFormat.DiscoveryReference, + ArtifactReference, + "defra_desnz_normalized_factors.csv", + "sha256", + ChecksumValue, + isDryRunChecksum: false, + "text/csv", + ".csv", + 2024); + var request = new ParserAdapterRunRequest( + SourceFamily.GhgProtocol, + SourceFamily.GhgProtocol.ToWireName(), + parserKey, + [artifact], + requestedReportingYear: 2024); + + var result = DefraDesnzNormalizedContentParser.Parse( + request, + CreateContentMap("defra_desnz_normalized_factors.csv")); + + Assert.Equal(ParserRunStatus.Failed, result.Status); + Assert.Equal("DEFRA_DESNZ_CONTENT_SOURCE_FAMILY_MISMATCH", result.ValidationIssues[0].Code); + Assert.Equal("source_family", result.ValidationIssues[0].FieldKey); + } + + private const string ArtifactReference = "tests/fixtures/source_documents/defra_desnz/defra_desnz_normalized_factors.csv"; + private const string ChecksumValue = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; + + private static ParserAdapterRunRequest CreateRequest() + { + var parserKey = ParserSelectionRegistry.GetParserKey(SourceFamily.DefraDesnz); + var artifact = new ParserInputArtifact( + SourceFamily.DefraDesnz, + SourceFamily.DefraDesnz.ToWireName(), + parserKey, + ParserSourceFormat.DiscoveryReference, + ArtifactReference, + "defra_desnz_normalized_factors.csv", + "sha256", + ChecksumValue, + isDryRunChecksum: false, + "text/csv", + ".csv", + 2024); + + return new ParserAdapterRunRequest( + SourceFamily.DefraDesnz, + SourceFamily.DefraDesnz.ToWireName(), + parserKey, + [artifact], + requestedReportingYear: 2024); + } + + private static IReadOnlyDictionary CreateContentMap(string fixtureName) => + new Dictionary + { + [ArtifactReference] = File.ReadAllText(Path.Combine(FixtureDirectory(), fixtureName)), + }; + + private static JsonDocument LoadParityExpectations() => + JsonDocument.Parse(File.ReadAllText(Path.Combine(ParityFixtureDirectory(), "defra_desnz_normalized_output_expectations.json"))); + + private static IReadOnlyList JsonStringArray(JsonElement array) => + array.EnumerateArray().Select(item => item.GetString() ?? string.Empty).ToArray(); + + private static IReadOnlyList JsonFieldArray(JsonElement array) => + array + .EnumerateArray() + .Select(field => + { + var values = field.EnumerateArray().ToArray(); + return new ParserNormalizedField(values[0].GetString() ?? string.Empty, values[1].GetString()); + }) + .ToArray(); + + private static string FixtureDirectory() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixtureDirectory = Path.Combine( + directory.FullName, + "tests", + "fixtures", + "source_documents", + "defra_desnz"); + if (Directory.Exists(fixtureDirectory)) + { + return fixtureDirectory; + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate DEFRA/DESNZ fixture directory."); + } + + private static string ParityFixtureDirectory() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixtureDirectory = Path.Combine(directory.FullName, "tests", "fixtures", "parity"); + if (Directory.Exists(fixtureDirectory)) + { + return fixtureDirectory; + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate parity fixture directory."); + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraSourceDiscoveryBoundaryTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraSourceDiscoveryBoundaryTests.cs new file mode 100644 index 00000000..e3bc0da6 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraSourceDiscoveryBoundaryTests.cs @@ -0,0 +1,305 @@ +using System.Reflection; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class DefraSourceDiscoveryBoundaryTests +{ + [Fact] + public void RequestIsDeterministicAndRuntimePassive() + { + var first = DefraSourceDiscoveryBoundary.CreateRequest(); + var second = DefraSourceDiscoveryBoundary.CreateRequest(); + + Assert.Equal(first, second); + Assert.Equal(SourceFamily.DefraDesnz, first.SourceFamily); + Assert.Equal("defra_desnz", first.SourceKey); + Assert.Equal("discovery://defra_desnz/homepage", first.DiscoveryReferenceUri); + Assert.Equal(DefraSourceDiscoveryMode.RuntimePassive, first.Mode); + Assert.False(first.AllowNetwork); + Assert.False(first.AllowDownload); + Assert.False(first.AllowParse); + Assert.False(first.AllowDatabaseWrites); + Assert.False(first.AllowScheduler); + Assert.True(DefraSourceDiscoveryBoundary.Validate(first).IsValid); + } + + [Fact] + public void ResultDeclaresDefraCandidateWithoutRuntimeWork() + { + var result = DefraSourceDiscoveryBoundary.CreateResult(); + + Assert.Equal(DefraSourceDiscoveryStatus.Declared, result.Status); + Assert.Equal(1, result.CandidateCount); + Assert.Equal(["defra_source_discovery_candidate_001_defra_desnz"], result.CandidateIds); + Assert.True(result.NoNetwork); + Assert.True(result.NoDownload); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.True(result.NoSql); + Assert.True(result.NoScheduler); + Assert.True(DefraSourceDiscoveryBoundary.Validate(result).IsValid); + + var candidate = result.Candidates[0]; + Assert.Equal(SourceFamily.DefraDesnz, candidate.SourceFamily); + Assert.Equal("defra_desnz", candidate.SourceKey); + Assert.Equal("DEFRA/DESNZ", candidate.Title); + Assert.Equal("discovery://defra_desnz/homepage", candidate.ReferenceUri); + Assert.Equal("discovery", candidate.ArtifactKind); + Assert.Equal(DefraSourceDiscoveryStatus.Declared, candidate.Status); + Assert.Equal("dn047_defra_discovery_boundary", candidate.VersionLabel); + Assert.Equal("runtime_passive_discovery_unavailable", candidate.DiscoveredAtLabel); + Assert.False(candidate.DownloadAllowed); + } + + [Fact] + public void BoundaryIsDefraOnly() + { + var result = DefraSourceDiscoveryBoundary.CreateResult(); + + Assert.Equal([SourceFamily.DefraDesnz], result.Candidates.Select(candidate => candidate.SourceFamily)); + Assert.Equal(["defra_desnz"], result.Candidates.Select(candidate => candidate.SourceKey)); + Assert.DoesNotContain("ghg_protocol", result.CandidateIds); + Assert.DoesNotContain("ipcc_efdb", result.CandidateIds); + } + + [Fact] + public void InvalidRequestFailsClosedWithNoCandidates() + { + var request = new DefraSourceDiscoveryRequest( + SourceFamily.DefraDesnz, + "ghg_protocol", + "discovery://defra_desnz/homepage", + allowNetwork: true, + allowDownload: true, + allowParse: true, + allowDatabaseWrites: true, + allowScheduler: true); + + var result = DefraSourceDiscoveryBoundary.CreateResult(request); + + Assert.Equal(DefraSourceDiscoveryStatus.Invalid, result.Status); + Assert.Empty(result.Candidates); + Assert.True(result.NoNetwork); + Assert.True(result.NoDownload); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.Equal( + [ + "DEFRA_SOURCE_DISCOVERY_SOURCE_KEY_MISMATCH", + "DEFRA_SOURCE_DISCOVERY_NETWORK_NOT_ALLOWED", + "DEFRA_SOURCE_DISCOVERY_DOWNLOAD_NOT_ALLOWED", + "DEFRA_SOURCE_DISCOVERY_PARSE_NOT_ALLOWED", + "DEFRA_SOURCE_DISCOVERY_DATABASE_WRITES_NOT_ALLOWED", + "DEFRA_SOURCE_DISCOVERY_SCHEDULER_NOT_ALLOWED", + ], + result.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void CandidateInvalidInputsFailClosed() + { + var candidate = new DefraSourceDocumentCandidate( + SourceFamily.GhgProtocol, + "ghg_protocol", + "candidate-1", + "", + "discovery://defra_desnz/homepage", + "xlsx", + DefraSourceDiscoveryStatus.Invalid, + documentYear: 0, + reportingYear: -1, + downloadAllowed: true); + + var result = DefraSourceDiscoveryBoundary.Validate(candidate); + + Assert.False(result.IsValid); + Assert.Equal( + [ + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_MISSING_TITLE", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_INVALID_DOCUMENT_YEAR", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_INVALID_REPORTING_YEAR", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_SOURCE_FAMILY_MISMATCH", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_SOURCE_KEY_MISMATCH", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_ARTIFACT_KIND_MISMATCH", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_UNSUPPORTED_STATUS", + "DEFRA_SOURCE_DISCOVERY_CANDIDATE_DOWNLOAD_NOT_ALLOWED", + ], + result.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void CandidateReferenceIsMetadataOnly() + { + var candidate = new DefraSourceDocumentCandidate( + SourceFamily.DefraDesnz, + "defra_desnz", + "defra-source-remote-candidate", + "DEFRA/DESNZ remote metadata", + "https://example.invalid/not-fetched.xlsx", + "discovery"); + + var result = DefraSourceDiscoveryBoundary.Validate(candidate); + + Assert.True(result.IsValid); + Assert.Empty(result.Issues); + } + + [Fact] + public void ValidationDoesNotRequireNetworkFileDatabaseParserDownloaderOrSchedulerRuntime() + { + var candidate = new DefraSourceDocumentCandidate( + SourceFamily.DefraDesnz, + "defra_desnz", + "defra-source-local-reference-candidate", + "DEFRA/DESNZ local metadata", + "/definitely/not-present/defra-desnz-factors.xlsx", + "discovery"); + var result = new DefraSourceDiscoveryResult( + DefraSourceDiscoveryStatus.Declared, + DefraSourceDiscoveryBoundary.CreateRequest(), + [candidate]); + + Assert.True(DefraSourceDiscoveryBoundary.Validate(candidate).IsValid); + Assert.True(DefraSourceDiscoveryBoundary.Validate(result).IsValid); + Assert.True(result.NoNetwork); + Assert.True(result.NoDownload); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.True(result.NoSql); + Assert.True(result.NoScheduler); + } + + [Fact] + public void ResultValidationRejectsSideEffectFlags() + { + var valid = DefraSourceDiscoveryBoundary.CreateResult(); + var result = new DefraSourceDiscoveryResult( + valid.Status, + valid.Request, + valid.Candidates, + valid.Issues, + noNetwork: false, + noSql: false); + + var validation = DefraSourceDiscoveryBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "DEFRA_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED", + "DEFRA_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED", + "DEFRA_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + ], + validation.Issues.Select(issue => issue.Code)); + Assert.Equal(["no_network", "no_sql"], validation.Issues.Take(2).Select(issue => issue.FieldName)); + } + + [Fact] + public void ResultValidationRejectsDeclaredResultsWithIssueMetadata() + { + var valid = DefraSourceDiscoveryBoundary.CreateResult(); + var result = new DefraSourceDiscoveryResult( + DefraSourceDiscoveryStatus.Declared, + valid.Request, + valid.Candidates, + [ + new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_TEST_ISSUE", + "test issue", + "test"), + ]); + + var validation = DefraSourceDiscoveryBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "DEFRA_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + "DEFRA_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + ], + validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void ResultValidationRejectsUndefinedStatus() + { + var valid = DefraSourceDiscoveryBoundary.CreateResult(); + var result = new DefraSourceDiscoveryResult( + (DefraSourceDiscoveryStatus)999, + valid.Request, + valid.Candidates, + [ + new DefraSourceDiscoveryIssue( + "DEFRA_SOURCE_DISCOVERY_TEST_ISSUE", + "test issue", + "test"), + ]); + + var validation = DefraSourceDiscoveryBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + ["DEFRA_SOURCE_DISCOVERY_RESULT_INVALID_STATUS"], + validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void BoundaryPublicSurfaceDoesNotExposeRuntimeExecutionMethods() + { + var publicMethodNames = typeof(DefraSourceDiscoveryBoundary) + .GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Select(method => method.Name) + .ToArray(); + + Assert.Contains("CreateRequest", publicMethodNames); + Assert.Contains("CreateResult", publicMethodNames); + Assert.Equal(3, publicMethodNames.Count(methodName => methodName == "Validate")); + Assert.DoesNotContain("Discover", publicMethodNames); + Assert.DoesNotContain("Fetch", publicMethodNames); + Assert.DoesNotContain("Parse", publicMethodNames); + Assert.DoesNotContain("Execute", publicMethodNames); + } + + [Fact] + public void BoundaryTypesDoNotExposeRuntimeExecutionMethods() + { + var publicMethodNames = new[] + { + typeof(DefraSourceDiscoveryRequest), + typeof(DefraSourceDocumentCandidate), + typeof(DefraSourceDiscoveryResult), + typeof(DefraSourceDiscoveryIssue), + typeof(DefraSourceDiscoveryValidationResult), + } + .SelectMany(type => type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + .Where(method => !method.Name.StartsWith("get_", StringComparison.Ordinal)) + .Select(method => method.Name) + .ToArray(); + + Assert.DoesNotContain("Discover", publicMethodNames); + Assert.DoesNotContain("Fetch", publicMethodNames); + Assert.DoesNotContain("Parse", publicMethodNames); + Assert.DoesNotContain("Execute", publicMethodNames); + Assert.DoesNotContain("Schedule", publicMethodNames); + Assert.DoesNotContain("Persist", publicMethodNames); + Assert.DoesNotContain("Open", publicMethodNames); + Assert.DoesNotContain("Read", publicMethodNames); + Assert.DoesNotContain("Write", publicMethodNames); + } + + [Fact] + public void DefraDiscoveryWireNamesArePythonAligned() + { + Assert.Equal("runtime_passive", DefraSourceDiscoveryMode.RuntimePassive.ToWireName()); + Assert.Equal("declared", DefraSourceDiscoveryStatus.Declared.ToWireName()); + Assert.Equal("invalid", DefraSourceDiscoveryStatus.Invalid.ToWireName()); + Assert.True(ContractWireNames.TryParseDefraSourceDiscoveryModeWireName("runtime_passive", out var parsedMode)); + Assert.Equal(DefraSourceDiscoveryMode.RuntimePassive, parsedMode); + Assert.True(ContractWireNames.TryParseDefraSourceDiscoveryStatusWireName("declared", out var parsedStatus)); + Assert.Equal(DefraSourceDiscoveryStatus.Declared, parsedStatus); + Assert.False(ContractWireNames.TryParseDefraSourceDiscoveryStatusWireName("unknown", out _)); + Assert.Throws(() => ((DefraSourceDiscoveryMode)999).ToWireName()); + Assert.Throws(() => ((DefraSourceDiscoveryStatus)999).ToWireName()); + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraSourceDownloadExecutionBoundaryTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraSourceDownloadExecutionBoundaryTests.cs new file mode 100644 index 00000000..e5332a78 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DefraSourceDownloadExecutionBoundaryTests.cs @@ -0,0 +1,438 @@ +using System.Reflection; +using System.Security.Cryptography; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class DefraSourceDownloadExecutionBoundaryTests +{ + [Fact] + public void RequestFromCandidateIsExplicitOptIn() + { + var candidate = DownloadableCandidate(); + using var temp = new TemporaryDirectory(); + + var request = DefraSourceDownloadExecutionBoundary.CreateRequest( + candidate, + temp.Path, + "defra/conversion-factors.xlsx"); + + Assert.Equal(SourceFamily.DefraDesnz, request.SourceFamily); + Assert.Equal("defra_desnz", request.SourceKey); + Assert.Equal("defra_source_discovery_candidate_001_defra_desnz", request.CandidateId); + Assert.Equal("DEFRA/DESNZ", request.CandidateTitle); + Assert.Equal("mock://defra_desnz/conversion-factors.xlsx", request.SourceReferenceUri); + Assert.Equal("xlsx", request.ArtifactKind); + Assert.True(request.CandidateDownloadAllowed); + Assert.False(request.AllowDownloadExecution); + Assert.False(request.AllowFileWrite); + Assert.Equal("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", request.ContentType); + Assert.Equal(".xlsx", request.Extension); + Assert.Equal("dn048_mock_download", request.VersionLabel); + + var validation = DefraSourceDownloadExecutionBoundary.Validate(request); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "DEFRA_SOURCE_DOWNLOAD_EXECUTION_NOT_ALLOWED", + "DEFRA_SOURCE_DOWNLOAD_FILE_WRITE_NOT_ALLOWED", + ], + validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void DefaultDiscoveryCandidateIsNotDownloadable() + { + using var temp = new TemporaryDirectory(); + var candidate = DefraSourceDiscoveryBoundary.CreateResult().Candidates[0]; + var request = DefraSourceDownloadExecutionBoundary.CreateRequest( + candidate, + temp.Path, + "defra/source.discovery", + allowDownloadExecution: true, + allowFileWrite: true); + + var result = DefraSourceDownloadExecutionBoundary.Execute(request, UnexpectedTransport); + + Assert.Equal(DefraSourceDownloadExecutionStatus.Blocked, result.Status); + Assert.False(result.Downloaded); + Assert.Null(result.Artifact); + Assert.Equal( + [ + "DEFRA_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE", + "DEFRA_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE", + ], + result.Issues.Select(issue => issue.Code)); + Assert.False(File.Exists(Path.Combine(temp.Path, "defra/source.discovery"))); + } + + [Fact] + public void InvalidDownloadRequestFailsClosedBeforeTransport() + { + using var temp = new TemporaryDirectory(); + var request = ValidRequest(temp.Path) with + { + SourceFamily = SourceFamily.GhgProtocol, + SourceKey = "ghg_protocol", + AllowParse = true, + AllowDatabaseWrites = true, + AllowScheduler = true, + }; + + var result = DefraSourceDownloadExecutionBoundary.Execute(request, UnexpectedTransport); + + Assert.Equal(DefraSourceDownloadExecutionStatus.Blocked, result.Status); + Assert.False(result.Downloaded); + Assert.Null(result.Artifact); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.True(result.NoSql); + Assert.True(result.NoScheduler); + Assert.Equal( + [ + "DEFRA_SOURCE_DOWNLOAD_SOURCE_FAMILY_MISMATCH", + "DEFRA_SOURCE_DOWNLOAD_SOURCE_KEY_MISMATCH", + "DEFRA_SOURCE_DOWNLOAD_PARSE_NOT_ALLOWED", + "DEFRA_SOURCE_DOWNLOAD_DATABASE_WRITES_NOT_ALLOWED", + "DEFRA_SOURCE_DOWNLOAD_SCHEDULER_NOT_ALLOWED", + ], + result.Issues.Select(issue => issue.Code)); + } + + [Theory] + [InlineData("source_reference_uri", "https://example.invalid/defra.xlsx", "DEFRA_SOURCE_DOWNLOAD_NETWORK_NOT_ALLOWED")] + [InlineData("source_reference_uri", "http://example.invalid/defra.xlsx", "DEFRA_SOURCE_DOWNLOAD_INSECURE_HTTP_NOT_ALLOWED")] + [InlineData("source_reference_uri", "file:///tmp/defra.xlsx", "DEFRA_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI")] + [InlineData("source_reference_uri", "s3://bucket/defra.xlsx", "DEFRA_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI")] + [InlineData("source_reference_uri", "defra/conversion-factors.xlsx", "DEFRA_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME")] + [InlineData("source_reference_uri", "://defra/conversion-factors.xlsx", "DEFRA_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI")] + [InlineData("source_reference_uri", "https:///defra.xlsx", "DEFRA_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI")] + [InlineData("target_root", "relative/root", "DEFRA_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_ABSOLUTE")] + [InlineData("target_relative_path", "../outside.xlsx", "DEFRA_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE")] + [InlineData("target_relative_path", "/absolute.xlsx", "DEFRA_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_ABSOLUTE")] + [InlineData("target_relative_path", "download://defra/source.xlsx", "DEFRA_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_URI")] + public void UnsafeRequestInputsFailClosed(string fieldName, string value, string expectedCode) + { + using var temp = new TemporaryDirectory(); + var request = WithField(ValidRequest(temp.Path), fieldName, value); + + var validation = DefraSourceDownloadExecutionBoundary.Validate(request); + + Assert.False(validation.IsValid); + Assert.Contains(expectedCode, validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void SuccessfulDownloadIsExplicitAndUsesInjectedTransport() + { + using var temp = new TemporaryDirectory(); + var payload = "deterministic defra source bytes"u8.ToArray(); + var calls = new List(); + var request = ValidRequest(temp.Path); + + var result = DefraSourceDownloadExecutionBoundary.Execute( + request, + sourceReferenceUri => + { + calls.Add(sourceReferenceUri); + return new DefraSourceDownloadTransportResponse( + payload, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "mock://defra_desnz/final.xlsx"); + }); + + var targetPath = Path.Combine(temp.Path, "defra/conversion-factors.xlsx"); + var checksum = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); + Assert.Equal(["mock://defra_desnz/conversion-factors.xlsx"], calls); + Assert.Equal(payload, File.ReadAllBytes(targetPath)); + Assert.Equal(DefraSourceDownloadExecutionStatus.Downloaded, result.Status); + Assert.True(result.Downloaded); + Assert.Equal( + new DefraSourceDownloadedArtifact( + SourceFamily.DefraDesnz, + "defra_desnz", + "defra_source_discovery_candidate_001_defra_desnz", + "defra_source_download_artifact_defra_source_discovery_candidate_001_defra_desnz", + "xlsx", + "mock://defra_desnz/conversion-factors.xlsx", + targetPath, + "conversion-factors.xlsx", + checksum, + payload.LongLength, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".xlsx", + "mock://defra_desnz/final.xlsx", + VersionLabel: "dn048_mock_download"), + result.Artifact); + Assert.True(DefraSourceDownloadExecutionBoundary.Validate(result).IsValid); + } + + [Fact] + public void TargetExistsBlocksBeforeTransportByDefault() + { + using var temp = new TemporaryDirectory(); + var request = ValidRequest(temp.Path); + var targetPath = Path.Combine(temp.Path, request.TargetRelativePath); + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + File.WriteAllBytes(targetPath, "existing"u8.ToArray()); + + var result = DefraSourceDownloadExecutionBoundary.Execute(request, UnexpectedTransport); + + Assert.Equal(DefraSourceDownloadExecutionStatus.Blocked, result.Status); + Assert.Equal(["DEFRA_SOURCE_DOWNLOAD_TARGET_EXISTS"], result.Issues.Select(issue => issue.Code)); + Assert.Equal("existing"u8.ToArray(), File.ReadAllBytes(targetPath)); + } + + [Fact] + public void ExistingFinalTargetSymlinkIsRejected() + { + if (OperatingSystem.IsWindows()) + { + return; + } + + using var temp = new TemporaryDirectory(); + var outside = Path.Combine(temp.Path, "outside"); + var targetParent = Path.Combine(temp.Path, "target-root", "defra"); + Directory.CreateDirectory(outside); + Directory.CreateDirectory(targetParent); + var targetPath = Path.Combine(targetParent, "escape.xlsx"); + File.CreateSymbolicLink(targetPath, Path.Combine(outside, "escape.xlsx")); + var request = ValidRequest(Path.Combine(temp.Path, "target-root")) with + { + TargetRelativePath = "defra/escape.xlsx", + AllowOverwrite = true, + }; + + var result = DefraSourceDownloadExecutionBoundary.Execute( + request, + _ => new DefraSourceDownloadTransportResponse("escape"u8.ToArray())); + + Assert.Equal(DefraSourceDownloadExecutionStatus.Blocked, result.Status); + Assert.Null(result.Artifact); + Assert.Equal(["DEFRA_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE"], result.Issues.Select(issue => issue.Code)); + Assert.False(File.Exists(Path.Combine(outside, "escape.xlsx"))); + } + + [Fact] + public void ParentSymlinkSwapDuringTransportCannotEscapeTargetRoot() + { + if (OperatingSystem.IsWindows()) + { + return; + } + + using var temp = new TemporaryDirectory(); + var targetRoot = Path.Combine(temp.Path, "target-root"); + var outside = Path.Combine(temp.Path, "outside"); + var targetParent = Path.Combine(targetRoot, "defra"); + Directory.CreateDirectory(targetParent); + Directory.CreateDirectory(outside); + var request = ValidRequest(targetRoot) with { TargetRelativePath = "defra/escape.xlsx" }; + + var result = DefraSourceDownloadExecutionBoundary.Execute( + request, + _ => + { + Directory.Delete(targetParent, recursive: true); + Directory.CreateSymbolicLink(targetParent, outside); + return new DefraSourceDownloadTransportResponse("escape"u8.ToArray()); + }); + + Assert.NotEqual(DefraSourceDownloadExecutionStatus.Downloaded, result.Status); + Assert.False(result.Downloaded); + Assert.Null(result.Artifact); + Assert.False(File.Exists(Path.Combine(outside, "escape.xlsx"))); + Assert.True(Directory.Exists(targetParent)); + } + + [Fact] + public void ChecksumMismatchFailsWithoutWritingFile() + { + using var temp = new TemporaryDirectory(); + var request = ValidRequest(temp.Path) with { ExpectedChecksumSha256 = new string('a', 64) }; + + var result = DefraSourceDownloadExecutionBoundary.Execute( + request, + _ => new DefraSourceDownloadTransportResponse("unexpected"u8.ToArray())); + + Assert.Equal(DefraSourceDownloadExecutionStatus.Failed, result.Status); + Assert.Null(result.Artifact); + Assert.Equal(["DEFRA_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH"], result.Issues.Select(issue => issue.Code)); + Assert.False(File.Exists(Path.Combine(temp.Path, request.TargetRelativePath))); + } + + [Fact] + public void TransportErrorsAndEmptyContentAreFailedResults() + { + using var temp = new TemporaryDirectory(); + using var other = new TemporaryDirectory(); + + var failed = DefraSourceDownloadExecutionBoundary.Execute( + ValidRequest(temp.Path), + _ => throw new InvalidOperationException("offline")); + var empty = DefraSourceDownloadExecutionBoundary.Execute( + ValidRequest(other.Path), + _ => new DefraSourceDownloadTransportResponse(Array.Empty())); + + Assert.Equal(DefraSourceDownloadExecutionStatus.Failed, failed.Status); + Assert.Equal(["DEFRA_SOURCE_DOWNLOAD_TRANSPORT_FAILED"], failed.Issues.Select(issue => issue.Code)); + Assert.Equal(DefraSourceDownloadExecutionStatus.Failed, empty.Status); + Assert.Equal(["DEFRA_SOURCE_DOWNLOAD_RESPONSE_EMPTY_CONTENT"], empty.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void TransportResponseValidationFailsClosed() + { + using var missing = new TemporaryDirectory(); + using var missingContent = new TemporaryDirectory(); + using var blankMetadata = new TemporaryDirectory(); + + var missingResponse = DefraSourceDownloadExecutionBoundary.Execute( + ValidRequest(missing.Path), + _ => null!); + var missingContentResponse = DefraSourceDownloadExecutionBoundary.Execute( + ValidRequest(missingContent.Path), + _ => new DefraSourceDownloadTransportResponse(null!)); + var blankMetadataResponse = DefraSourceDownloadExecutionBoundary.Execute( + ValidRequest(blankMetadata.Path), + _ => new DefraSourceDownloadTransportResponse("content"u8.ToArray(), " ", " ")); + + Assert.Equal(DefraSourceDownloadExecutionStatus.Failed, missingResponse.Status); + Assert.Equal(["DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING"], missingResponse.Issues.Select(issue => issue.Code)); + Assert.Equal(DefraSourceDownloadExecutionStatus.Failed, missingContentResponse.Status); + Assert.Equal( + ["DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT"], + missingContentResponse.Issues.Select(issue => issue.Code)); + Assert.Equal(DefraSourceDownloadExecutionStatus.Failed, blankMetadataResponse.Status); + Assert.Equal( + [ + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + ], + blankMetadataResponse.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void ResultValidationRejectsSideEffectFlags() + { + using var temp = new TemporaryDirectory(); + var result = DefraSourceDownloadExecutionBoundary.Execute( + ValidRequest(temp.Path), + _ => new DefraSourceDownloadTransportResponse("content"u8.ToArray())) with + { + NoDatabaseWrites = false, + NoSql = false, + }; + + var validation = DefraSourceDownloadExecutionBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "DEFRA_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED", + "DEFRA_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED", + ], + validation.Issues.Select(issue => issue.Code)); + Assert.Equal(["no_database_writes", "no_sql"], validation.Issues.Select(issue => issue.FieldName)); + } + + [Theory] + [InlineData(DefraSourceDownloadExecutionStatus.Blocked)] + [InlineData(DefraSourceDownloadExecutionStatus.Failed)] + public void ResultValidationRejectsBlockedOrFailedResultsWithoutIssues(DefraSourceDownloadExecutionStatus status) + { + using var temp = new TemporaryDirectory(); + var result = new DefraSourceDownloadExecutionResult(status, ValidRequest(temp.Path)); + + var validation = DefraSourceDownloadExecutionBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal(["DEFRA_SOURCE_DOWNLOAD_RESULT_MISSING_ISSUES"], validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void BoundaryPublicSurfaceOnlyExposesExplicitExecutionMethods() + { + var publicMethodNames = typeof(DefraSourceDownloadExecutionBoundary) + .GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Select(method => method.Name) + .ToArray(); + + Assert.Contains("CreateRequest", publicMethodNames); + Assert.Contains("Execute", publicMethodNames); + Assert.Equal(2, publicMethodNames.Count(methodName => methodName == "Validate")); + Assert.DoesNotContain("Parse", publicMethodNames); + Assert.DoesNotContain("Persist", publicMethodNames); + Assert.DoesNotContain("Schedule", publicMethodNames); + } + + [Fact] + public void DefraDownloadExecutionWireNamesArePythonAligned() + { + Assert.Equal("blocked", DefraSourceDownloadExecutionStatus.Blocked.ToWireName()); + Assert.Equal("downloaded", DefraSourceDownloadExecutionStatus.Downloaded.ToWireName()); + Assert.Equal("failed", DefraSourceDownloadExecutionStatus.Failed.ToWireName()); + Assert.True(ContractWireNames.TryParseDefraSourceDownloadExecutionStatusWireName("downloaded", out var parsed)); + Assert.Equal(DefraSourceDownloadExecutionStatus.Downloaded, parsed); + Assert.False(ContractWireNames.TryParseDefraSourceDownloadExecutionStatusWireName("unknown", out _)); + Assert.Throws(() => ((DefraSourceDownloadExecutionStatus)999).ToWireName()); + } + + private static DefraSourceDownloadExecutionRequest ValidRequest(string targetRoot) => + DefraSourceDownloadExecutionBoundary.CreateRequest( + DownloadableCandidate(), + targetRoot, + "defra/conversion-factors.xlsx", + allowDownloadExecution: true, + allowFileWrite: true); + + private static DefraSourceDocumentCandidate DownloadableCandidate() => + new( + SourceFamily.DefraDesnz, + "defra_desnz", + "defra_source_discovery_candidate_001_defra_desnz", + "DEFRA/DESNZ", + "mock://defra_desnz/conversion-factors.xlsx", + "xlsx", + contentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + extension: ".xlsx", + versionLabel: "dn048_mock_download", + downloadAllowed: true); + + private static DefraSourceDownloadExecutionRequest WithField( + DefraSourceDownloadExecutionRequest request, + string fieldName, + string value) => + fieldName switch + { + "source_reference_uri" => request with { SourceReferenceUri = value }, + "target_root" => request with { TargetRoot = value }, + "target_relative_path" => request with { TargetRelativePath = value }, + _ => throw new ArgumentOutOfRangeException(nameof(fieldName), fieldName, "Unknown test field."), + }; + + private static DefraSourceDownloadTransportResponse UnexpectedTransport(string sourceReferenceUri) => + throw new InvalidOperationException($"transport should not be called for {sourceReferenceUri}"); + + private sealed class TemporaryDirectory : IDisposable + { + public string Path { get; } = System.IO.Path.Combine( + System.IO.Path.GetTempPath(), + $"carbonops-defra-download-{Guid.NewGuid():N}"); + + public TemporaryDirectory() + { + Directory.CreateDirectory(Path); + } + + public void Dispose() + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DotNetPostgreSQLIntegrationE2ETests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DotNetPostgreSQLIntegrationE2ETests.cs new file mode 100644 index 00000000..4df1b0c7 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/DotNetPostgreSQLIntegrationE2ETests.cs @@ -0,0 +1,565 @@ +using System.Security.Cryptography; +using Npgsql; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class DotNetPostgreSQLIntegrationE2ETests +{ + private const string OptInEnvironmentVariable = "CARBONOPS_RUN_DOTNET_POSTGRESQL_INTEGRATION"; + private const string DotNetTestDsnEnvironmentVariable = "CARBONOPS_DOTNET_POSTGRESQL_TEST_DSN"; + private const string SharedTestDsnEnvironmentVariable = "CARBONOPS_POSTGRESQL_TEST_DSN"; + + [Fact] + public void DefaultIntegrationGuardDoesNotConnectWithoutOptIn() + { + var gate = ResolveIntegrationSettings( + new Dictionary(StringComparer.Ordinal), + allowGeneratedSchema: true); + + Assert.False(gate.Enabled); + Assert.False(gate.ConnectionAttempted); + Assert.Contains(OptInEnvironmentVariable, gate.Reason, StringComparison.Ordinal); + } + + [Fact] + public void MissingPostgreSQLConfigFailsClosedWhenOptedIn() + { + var gate = ResolveIntegrationSettings( + new Dictionary(StringComparer.Ordinal) + { + [OptInEnvironmentVariable] = "1", + }, + allowGeneratedSchema: true); + + Assert.True(gate.Enabled); + Assert.False(gate.ConnectionAttempted); + Assert.False(gate.HasSettings); + Assert.Contains("POSTGRESQL_RUNTIME_MISSING_PASSWORD", gate.Reason, StringComparison.Ordinal); + Assert.DoesNotContain("Password=", gate.Reason, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("postgresql://", gate.Reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task DockerPostgreSQLBootstrapIsIdempotentWhenEnabled() + { + var gate = ResolveIntegrationSettingsFromEnvironment(); + if (!gate.Enabled) + { + return; + } + + Assert.True(gate.HasSettings, gate.Reason); + var bootstrapper = new PostgreSQLRuntimeSchemaBootstrapper(); + + var first = await bootstrapper.BootstrapAsync(gate.Settings!); + var second = await bootstrapper.BootstrapAsync(gate.Settings!); + + Assert.Empty(first.MissingTableNames); + Assert.Empty(second.MissingTableNames); + Assert.Equal(PostgreSQLRuntimeSchemaCatalog.RequiredTableNames, first.PresentTableNames); + Assert.Equal(PostgreSQLRuntimeSchemaCatalog.RequiredTableNames, second.PresentTableNames); + Assert.Empty(second.CreatedTableNames); + } + + [Theory] + [MemberData(nameof(SourceFamilyFixtureCases))] + public async Task SourceFamilyFixtureInsertRerunAndYearStateAreIdempotentWhenEnabled( + SourceFamily sourceFamily) + { + var gate = ResolveIntegrationSettingsFromEnvironment(); + if (!gate.Enabled) + { + return; + } + + Assert.True(gate.HasSettings, gate.Reason); + var settings = gate.Settings!; + var bootstrapper = new PostgreSQLRuntimeSchemaBootstrapper(); + var firstBootstrap = await bootstrapper.BootstrapAsync(settings); + var secondBootstrap = await bootstrapper.BootstrapAsync(settings); + Assert.Empty(firstBootstrap.MissingTableNames); + Assert.Empty(secondBootstrap.MissingTableNames); + Assert.Empty(secondBootstrap.CreatedTableNames); + + await using var dataSource = NpgsqlDataSource.Create( + PostgreSQLRuntimeConnectionBoundary.BuildConnectionString(settings)); + + var parsed = ParseFixture(sourceFamily, settings.Schema, targetYear: 2024); + var repository = new PostgreSQLSourceSpecificFactorPersistenceRepository( + new NpgsqlSourceSpecificFactorPersistenceSession(dataSource)); + var yearState = new PostgreSQLSourceFamilyYearStateRepository( + new NpgsqlSourceFamilyYearStateSession(dataSource)); + + var first = await repository.PersistAsync(parsed); + var second = await repository.PersistAsync(parsed); + var state = await yearState.GetYearStateAsync(sourceFamily); + var counts = await CountSourceFamilyRowsAsync(dataSource, sourceFamily, "prod009-" + settings.Schema); + + Assert.Equal(PostgreSQLSourceSpecificFactorPersistenceStatus.Inserted, first.Status); + Assert.True(first.MasterInserted > 0); + Assert.True(first.DetailInserted > 0); + Assert.Equal(0, first.MasterSkippedDuplicate); + Assert.Equal(0, first.DetailSkippedDuplicate); + + Assert.Equal(PostgreSQLSourceSpecificFactorPersistenceStatus.Inserted, second.Status); + Assert.Equal(0, second.MasterInserted); + Assert.Equal(0, second.DetailInserted); + Assert.True(second.MasterSkippedDuplicate > 0); + Assert.True(second.DetailSkippedDuplicate > 0); + + Assert.Equal(first.MasterInserted, counts.MasterRows); + Assert.Equal(first.DetailInserted, counts.DetailRows); + Assert.Equal(2024, state.LatestYear); + Assert.Equal(2025, state.NextYear); + Assert.Equal(1, await CountYearStateRowsAsync(dataSource, sourceFamily, 2024)); + } + + [Fact] + public async Task PersistedParityFixtureBaselineWhenEnabled() + { + var gate = ResolveIntegrationSettingsFromEnvironment(); + if (!gate.Enabled) + { + return; + } + + Assert.True(gate.HasSettings, gate.Reason); + var settings = gate.Settings!; + await new PostgreSQLRuntimeSchemaBootstrapper().BootstrapAsync(settings); + await using var dataSource = NpgsqlDataSource.Create( + PostgreSQLRuntimeConnectionBoundary.BuildConnectionString(settings)); + var repository = new PostgreSQLSourceSpecificFactorPersistenceRepository( + new NpgsqlSourceSpecificFactorPersistenceSession(dataSource)); + + foreach (var sourceFamily in SourceFamilyFixtureCases().Select(item => (SourceFamily)item[0])) + { + var parsed = ParseFixture(sourceFamily, "prod010-parity", targetYear: 2024); + var first = await repository.PersistAsync(parsed); + var second = await repository.PersistAsync(parsed); + + Assert.Equal(PostgreSQLSourceSpecificFactorPersistenceStatus.Inserted, first.Status); + Assert.True(first.MasterInserted > 0); + Assert.True(first.DetailInserted > 0); + Assert.Equal(PostgreSQLSourceSpecificFactorPersistenceStatus.Inserted, second.Status); + Assert.Equal(0, second.MasterInserted); + Assert.Equal(0, second.DetailInserted); + Assert.True(second.MasterSkippedDuplicate > 0); + Assert.True(second.DetailSkippedDuplicate > 0); + Assert.Equal(1, await CountYearStateRowsAsync(dataSource, sourceFamily, 2024)); + } + } + + [Theory] + [MemberData(nameof(SourceFamilyFixtureCases))] + public async Task NoAvailableSourceYearDoesNotAdvanceYearStateWhenEnabled( + SourceFamily sourceFamily) + { + var gate = ResolveIntegrationSettingsFromEnvironment(); + if (!gate.Enabled) + { + return; + } + + Assert.True(gate.HasSettings, gate.Reason); + await new PostgreSQLRuntimeSchemaBootstrapper().BootstrapAsync(gate.Settings!); + await using var dataSource = NpgsqlDataSource.Create( + PostgreSQLRuntimeConnectionBoundary.BuildConnectionString(gate.Settings!)); + var yearState = new PostgreSQLSourceFamilyYearStateRepository( + new NpgsqlSourceFamilyYearStateSession(dataSource)); + await yearState.RecordSuccessfulYearAsync(sourceFamily, 2024); + var orchestrator = new SourceCycleOrchestrator( + yearState, + [sourceFamily], + artifacts: []); + + var result = await orchestrator.PreviewAsync(); + var run = Assert.Single(result.Runs); + var after = await yearState.GetYearStateAsync(sourceFamily); + + Assert.Equal(SourceCycleRunStatus.NoAvailableSourceYear, run.Status); + Assert.Equal(2024, run.LatestSuccessfulYear); + Assert.Equal(2025, run.TargetYear); + Assert.Equal(2024, after.LatestYear); + Assert.Equal(2025, after.NextYear); + Assert.Equal(1, await CountYearStateRowsAsync(dataSource, sourceFamily, 2024)); + } + + [Fact] + public async Task FailedPersistenceRollsBackAndDoesNotAdvanceYearStateWhenEnabled() + { + var gate = ResolveIntegrationSettingsFromEnvironment(); + if (!gate.Enabled) + { + return; + } + + Assert.True(gate.HasSettings, gate.Reason); + var settings = gate.Settings!; + await new PostgreSQLRuntimeSchemaBootstrapper().BootstrapAsync(settings); + await using var dataSource = NpgsqlDataSource.Create( + PostgreSQLRuntimeConnectionBoundary.BuildConnectionString(settings)); + var parsed = ParseFixture(SourceFamily.DefraDesnz, settings.Schema, targetYear: 2024); + var mapped = PostgreSQLSourceSpecificFactorPersistenceMapper.Map(parsed); + var validBatch = Assert.Single(mapped.Batches); + var invalidDetail = validBatch.DetailRecords[0] with + { + SourceFamilyMasterId = Guid.NewGuid(), + DetailExternalKey = "invalid-master-reference-" + settings.Schema, + }; + var invalidBatch = validBatch with + { + DetailRecords = [invalidDetail], + }; + var session = new NpgsqlSourceSpecificFactorPersistenceSession(dataSource); + + await Assert.ThrowsAnyAsync( + () => session.PersistSourceFamilyYearAsync(invalidBatch)); + + Assert.Equal(0, await CountYearStateRowsAsync(dataSource, SourceFamily.DefraDesnz, 2024)); + } + + [Fact] + public void IntegrationDiagnosticsRedactSecrets() + { + var secret = "secret-not-returned"; + var gate = ResolveIntegrationSettings( + new Dictionary(StringComparer.Ordinal) + { + [OptInEnvironmentVariable] = "1", + ["CARBONOPS_PARSER_ENV"] = "production", + ["CARBONOPS_PARSER_DATABASE_PROVIDER"] = "postgres", + ["CARBONOPS_PARSER_POSTGRES_HOST"] = "localhost", + ["CARBONOPS_PARSER_POSTGRES_PORT"] = "5432", + ["CARBONOPS_PARSER_POSTGRES_DATABASE"] = "carbonops_parser", + ["CARBONOPS_PARSER_POSTGRES_USERNAME"] = "carbonops_runtime", + ["CARBONOPS_PARSER_POSTGRES_PASSWORD"] = secret, + ["CARBONOPS_PARSER_POSTGRES_SCHEMA"] = "carbonops_prod009_redaction", + ["CARBONOPS_PARSER_RAW_ARCHIVE_PATH"] = "/tmp/carbonops-parser", + ["CARBONOPS_PARSER_LOG_LEVEL"] = "info", + }, + allowGeneratedSchema: false); + + Assert.True(gate.HasSettings, gate.Reason); + var rendered = string.Join( + "\n", + PostgreSQLRuntimeConnectionBoundary.BuildSafeDiagnostics(gate.Settings!) + .Select(item => $"{item.Key}={item.Value}")); + + Assert.Contains("postgresql_password=[redacted]", rendered, StringComparison.Ordinal); + Assert.Contains("postgresql_connection_string=[redacted]", rendered, StringComparison.Ordinal); + Assert.DoesNotContain(secret, rendered, StringComparison.Ordinal); + } + + private static IntegrationGate ResolveIntegrationSettingsFromEnvironment() => + ResolveIntegrationSettings( + Environment.GetEnvironmentVariables() + .Cast() + .ToDictionary( + entry => (string)entry.Key, + entry => entry.Value?.ToString(), + StringComparer.Ordinal), + allowGeneratedSchema: true); + + private static IntegrationGate ResolveIntegrationSettings( + IReadOnlyDictionary environment, + bool allowGeneratedSchema) + { + if (!string.Equals(Get(environment, OptInEnvironmentVariable), "1", StringComparison.Ordinal)) + { + return new IntegrationGate( + Enabled: false, + ConnectionAttempted: false, + HasSettings: false, + Settings: null, + Reason: $"{OptInEnvironmentVariable}=1 is required for .NET PostgreSQL integration tests."); + } + + var schema = Get(environment, "CARBONOPS_DOTNET_POSTGRESQL_TEST_SCHEMA"); + if (string.IsNullOrWhiteSpace(schema) && allowGeneratedSchema) + { + schema = "carbonops_prod009_" + Guid.NewGuid().ToString("N"); + } + + var dsn = Get(environment, DotNetTestDsnEnvironmentVariable) ?? Get(environment, SharedTestDsnEnvironmentVariable); + if (!string.IsNullOrWhiteSpace(dsn)) + { + return ResolveFromDsn(dsn, schema); + } + + var values = ProductionConfigBoundary.KnownConfigurationKeys.ToDictionary( + key => key, + key => Get(environment, key), + StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(schema)) + { + values["CARBONOPS_PARSER_POSTGRES_SCHEMA"] = schema; + } + + var created = PostgreSQLRuntimeConnectionBoundary.TryCreateFromProductionConfig( + values, + out var settings, + out var issues); + return created && settings is not null + ? new IntegrationGate(true, false, true, settings, "ready") + : new IntegrationGate(true, false, false, null, string.Join(",", issues.Select(issue => issue.Code))); + } + + private static IntegrationGate ResolveFromDsn(string dsn, string? schema) + { + try + { + if (Uri.TryCreate(dsn, UriKind.Absolute, out var uri) && + (string.Equals(uri.Scheme, "postgresql", StringComparison.OrdinalIgnoreCase) || + string.Equals(uri.Scheme, "postgres", StringComparison.OrdinalIgnoreCase))) + { + return ResolveFromPostgreSQLUri(uri, schema); + } + + var builder = new NpgsqlConnectionStringBuilder(dsn); + var resolvedSchema = string.IsNullOrWhiteSpace(schema) + ? FirstSearchPathSchema(builder.SearchPath) + : schema; + var settings = new PostgreSQLRuntimeConnectionSettings( + builder.Host ?? string.Empty, + builder.Port, + builder.Database ?? string.Empty, + builder.Username ?? string.Empty, + builder.Password ?? string.Empty, + resolvedSchema ?? string.Empty, + string.IsNullOrWhiteSpace(builder.ApplicationName) + ? "carbonops-parser-dotnet-prod009-tests" + : builder.ApplicationName, + builder.Timeout <= 0 ? 15 : builder.Timeout); + var validation = PostgreSQLRuntimeConnectionBoundary.Validate(settings); + return validation.IsValid + ? new IntegrationGate(true, false, true, settings, "ready") + : new IntegrationGate( + true, + false, + false, + null, + string.Join(",", validation.Issues.Select(issue => issue.Code))); + } + catch (ArgumentException ex) + { + return new IntegrationGate(true, false, false, null, ex.GetType().Name); + } + } + + private static IntegrationGate ResolveFromPostgreSQLUri(Uri uri, string? schema) + { + var userInfo = uri.UserInfo.Split(':', 2); + var settings = new PostgreSQLRuntimeConnectionSettings( + uri.Host, + uri.Port > 0 ? uri.Port : 5432, + Uri.UnescapeDataString(uri.AbsolutePath.TrimStart('/')), + userInfo.Length > 0 ? Uri.UnescapeDataString(userInfo[0]) : string.Empty, + userInfo.Length > 1 ? Uri.UnescapeDataString(userInfo[1]) : string.Empty, + schema ?? string.Empty, + "carbonops-parser-dotnet-prod009-tests"); + var validation = PostgreSQLRuntimeConnectionBoundary.Validate(settings); + return validation.IsValid + ? new IntegrationGate(true, false, true, settings, "ready") + : new IntegrationGate( + true, + false, + false, + null, + string.Join(",", validation.Issues.Select(issue => issue.Code))); + } + + public static IEnumerable SourceFamilyFixtureCases() + { + yield return [SourceFamily.GhgProtocol]; + yield return [SourceFamily.DefraDesnz]; + yield return [SourceFamily.IpccEfdb]; + } + + private static ParserNormalizedOutputBatch ParseFixture( + SourceFamily sourceFamily, + string uniqueLabel, + int targetYear) + { + var fixture = Fixture(sourceFamily); + var artifactPath = FixturePath(fixture.FamilyDirectory, fixture.FileName); + var content = File.ReadAllText(artifactPath); + var artifactChecksum = Sha256Hex(content); + var parserKey = ParserSelectionRegistry.GetParserKey(sourceFamily); + var artifact = new ParserInputArtifact( + sourceFamily, + sourceFamily.ToWireName(), + parserKey, + ParserSourceFormat.DiscoveryReference, + artifactPath, + Path.GetFileName(artifactPath), + "sha256", + artifactChecksum, + isDryRunChecksum: false, + "text/csv", + ".csv", + reportingYear: targetYear); + var request = new ParserAdapterRunRequest( + sourceFamily, + sourceFamily.ToWireName(), + parserKey, + [artifact], + runId: "prod009-" + uniqueLabel, + requestedReportingYear: targetYear); + var contentByArtifactReference = new Dictionary(StringComparer.Ordinal) + { + [artifactPath] = content, + }; + var parsed = sourceFamily switch + { + SourceFamily.GhgProtocol => GhgProtocolNormalizedContentParser.Parse(request, contentByArtifactReference), + SourceFamily.DefraDesnz => DefraDesnzNormalizedContentParser.Parse(request, contentByArtifactReference), + SourceFamily.IpccEfdb => IpccEfdbNormalizedContentParser.Parse(request, contentByArtifactReference), + _ => throw new ArgumentOutOfRangeException(nameof(sourceFamily), sourceFamily, "Unknown source family."), + }; + + Assert.Equal(ParserRunStatus.Completed, parsed.Status); + Assert.True(parsed.RowCount > 0); + + return new ParserNormalizedOutputBatch( + parsed.Rows.Select(row => RewriteForIsolatedE2E(row, uniqueLabel, artifactChecksum, targetYear))); + } + + private static ParserNormalizedOutputRow RewriteForIsolatedE2E( + ParserNormalizedOutputRow row, + string uniqueLabel, + string artifactChecksum, + int targetYear) + { + var sourceVersion = "prod009-" + uniqueLabel; + var factorId = row.Fields + .Where(field => field.Key == "factor_id") + .Select(field => field.Value) + .FirstOrDefault() ?? row.RowIdentifier; + var replacementFields = row.Fields + .Where(field => field.Key is not ( + "source_year" or + "source_version" or + "run_id" or + "provenance_checksum_value" or + "master_external_key" or + "source_family_master_id" or + "source_family_detail_id")) + .Concat( + [ + new ParserNormalizedField("source_year", targetYear.ToString(System.Globalization.CultureInfo.InvariantCulture)), + new ParserNormalizedField("source_version", sourceVersion), + new ParserNormalizedField("run_id", "prod009-" + uniqueLabel), + new ParserNormalizedField("provenance_checksum_value", artifactChecksum), + new ParserNormalizedField("master_external_key", $"{targetYear}:{sourceVersion}:{factorId}"), + ]); + + return new ParserNormalizedOutputRow( + row.SourceFamily, + row.SourceKey, + row.ParserKey, + row.ArtifactReference, + row.RowIdentifier, + row.SourceRowNumber, + replacementFields, + row.Issues, + targetYear); + } + + private static async Task<(int MasterRows, int DetailRows)> CountSourceFamilyRowsAsync( + NpgsqlDataSource dataSource, + SourceFamily sourceFamily, + string sourceVersion) + { + var names = SourceFamilyRepositoryRegistry.GetTableNames(sourceFamily); + var masterIdColumn = SourceFamilyMasterIdColumn(sourceFamily); + await using var command = dataSource.CreateCommand($$""" + SELECT + (SELECT count(*) FROM {{names.MasterTableName}} WHERE source_version = $1), + (SELECT count(*) FROM {{names.DetailTableName}} d + JOIN {{names.MasterTableName}} m + ON m.{{masterIdColumn}} = d.{{masterIdColumn}} + WHERE m.source_version = $1) + """); + command.Parameters.AddWithValue(sourceVersion); + + await using var reader = await command.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + return (Convert.ToInt32(reader.GetInt64(0)), Convert.ToInt32(reader.GetInt64(1))); + } + + private static (string FamilyDirectory, string FileName) Fixture(SourceFamily sourceFamily) => + sourceFamily switch + { + SourceFamily.GhgProtocol => ("ghg_protocol", "ghg_protocol_sample_factors.csv"), + SourceFamily.DefraDesnz => ("defra_desnz", "defra_desnz_normalized_factors.csv"), + SourceFamily.IpccEfdb => ("ipcc_efdb", "ipcc_efdb_sample_factors.csv"), + _ => throw new ArgumentOutOfRangeException(nameof(sourceFamily), sourceFamily, "Unknown source family."), + }; + + private static string SourceFamilyMasterIdColumn(SourceFamily sourceFamily) => + sourceFamily switch + { + SourceFamily.GhgProtocol => "ghg_emission_factor_master_id", + SourceFamily.DefraDesnz => "defra_emission_factor_master_id", + SourceFamily.IpccEfdb => "ipcc_emission_factor_master_id", + _ => throw new ArgumentOutOfRangeException(nameof(sourceFamily), sourceFamily, "Unknown source family."), + }; + + private static async Task CountYearStateRowsAsync( + NpgsqlDataSource dataSource, + SourceFamily sourceFamily, + int year) + { + await using var command = dataSource.CreateCommand(""" + SELECT count(*) + FROM source_family_year_states + WHERE source_family = $1 + AND ingested_year = $2 + """); + command.Parameters.AddWithValue(sourceFamily.ToPostgreSQLRuntimeValue()); + command.Parameters.AddWithValue(year); + var count = await command.ExecuteScalarAsync(); + return Convert.ToInt32(count, System.Globalization.CultureInfo.InvariantCulture); + } + + private static string FixturePath(string familyDirectory, string fileName) + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixtureDirectory = Path.Combine( + directory.FullName, + "tests", + "fixtures", + "source_documents", + familyDirectory); + if (Directory.Exists(fixtureDirectory)) + { + return Path.Combine(fixtureDirectory, fileName); + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate source document fixture directory."); + } + + private static string Sha256Hex(string content) => + Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(content))).ToLowerInvariant(); + + private static string? FirstSearchPathSchema(string? searchPath) => + searchPath? + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .FirstOrDefault(); + + private static string? Get(IReadOnlyDictionary values, string key) => + values.TryGetValue(key, out var value) ? value : null; + + private sealed record IntegrationGate( + bool Enabled, + bool ConnectionAttempted, + bool HasSettings, + PostgreSQLRuntimeConnectionSettings? Settings, + string Reason); +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgProtocolNormalizedContentParserTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgProtocolNormalizedContentParserTests.cs new file mode 100644 index 00000000..7f1426df --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgProtocolNormalizedContentParserTests.cs @@ -0,0 +1,255 @@ +using System.Text.Json; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class GhgProtocolNormalizedContentParserTests +{ + [Fact] + public void GhgProtocolHeaderIsDeterministic() + { + using var expectations = LoadParityExpectations(); + + Assert.Equal( + JsonStringArray(expectations.RootElement.GetProperty("header")), + GhgProtocolNormalizedContentParser.Header); + } + + [Fact] + public void ValidGhgProtocolContentReturnsNormalizedRows() + { + var request = CreateRequest(); + var result = GhgProtocolNormalizedContentParser.Parse( + request, + CreateContentMap("ghg_protocol_sample_factors.csv")); + + Assert.Equal(ParserRunStatus.Completed, result.Status); + Assert.Equal(2, result.RowCount); + Assert.Single(result.ValidationIssues); + Assert.True(result.Validate().IsValid); + Assert.All(result.Rows, row => Assert.True(row.Validate().IsValid)); + Assert.Equal("GHG_PROTOCOL_CONTENT_UNSUPPORTED_ROW_SKIPPED", result.ValidationIssues[0].Code); + Assert.Equal(ParserValidationIssueSeverity.Warning, result.ValidationIssues[0].Severity); + + var first = result.Rows[0]; + Assert.Equal(SourceFamily.GhgProtocol, first.SourceFamily); + Assert.Equal("ghg_protocol", first.SourceKey); + Assert.Equal("ghg_protocol_2024_v1_GHG-ELEC-001_row_2", first.RowIdentifier); + Assert.Equal(2, first.SourceRowNumber); + Assert.Equal(2024, first.ReportingYear); + Assert.Equal( + [ + new ParserNormalizedField("source_family", "ghg_protocol"), + new ParserNormalizedField("source_year", "2024"), + new ParserNormalizedField("source_version", "v1"), + new ParserNormalizedField("factor_id", "GHG-ELEC-001"), + new ParserNormalizedField("factor_name", "Grid electricity"), + new ParserNormalizedField("factor_value", "0.233"), + new ParserNormalizedField("unit", "kg CO2e/kWh"), + new ParserNormalizedField("category", "Stationary combustion"), + new ParserNormalizedField("subcategory", "Electricity"), + new ParserNormalizedField("scope", "Scope 2"), + new ParserNormalizedField("gas", "CO2e"), + new ParserNormalizedField("provenance_artifact_reference", ArtifactReference), + new ParserNormalizedField("provenance_checksum_algorithm", "sha256"), + new ParserNormalizedField("provenance_checksum_value", ChecksumValue), + new ParserNormalizedField("provenance_row_number", "2"), + new ParserNormalizedField("provenance_note", "fixture row 1"), + new ParserNormalizedField("source_family_master_id", "ghg_master_2024_v1_GHG-ELEC-001"), + new ParserNormalizedField("source_family_detail_id", "ghg_detail_2024_v1_GHG-ELEC-001"), + new ParserNormalizedField("master_external_key", "2024:v1:GHG-ELEC-001"), + new ParserNormalizedField("detail_external_key", "GHG-ELEC-001:kg CO2e/kWh"), + ], + first.Fields); + } + + [Fact] + public void ValidGhgProtocolContentMatchesSharedParityExpectations() + { + using var expectations = LoadParityExpectations(); + var root = expectations.RootElement; + var request = CreateRequest(); + var result = GhgProtocolNormalizedContentParser.Parse( + request, + CreateContentMap("ghg_protocol_sample_factors.csv")); + var expectedRows = root.GetProperty("sample_rows").EnumerateArray().ToArray(); + + Assert.Equal(root.GetProperty("sample_status").GetProperty("dotnet").GetString(), result.Status.ToString()); + Assert.Equal( + JsonStringArray(root.GetProperty("sample_issue_codes")), + result.ValidationIssues.Select(issue => issue.Code)); + Assert.Equal(expectedRows.Length, result.RowCount); + + for (var index = 0; index < expectedRows.Length; index++) + { + var expected = expectedRows[index]; + var actual = result.Rows[index]; + + Assert.Equal(expected.GetProperty("row_identifier").GetString(), actual.RowIdentifier); + Assert.Equal(expected.GetProperty("source_row_number").GetInt32(), actual.SourceRowNumber); + Assert.Equal(expected.GetProperty("reporting_year").GetInt32(), actual.ReportingYear); + Assert.Equal(JsonFieldArray(expected.GetProperty("fields")), actual.Fields); + } + } + + [Fact] + public void GhgProtocolParserIsDeterministicForFixtureInput() + { + var request = CreateRequest(); + var content = CreateContentMap("ghg_protocol_sample_factors.csv"); + + var first = GhgProtocolNormalizedContentParser.Parse(request, content); + var second = GhgProtocolNormalizedContentParser.Parse(request, content); + + Assert.Equal(first, second); + Assert.Equal(2, first.RowCount); + } + + [Fact] + public void MalformedGhgProtocolRowsReturnStructuredErrors() + { + using var expectations = LoadParityExpectations(); + var root = expectations.RootElement; + var result = GhgProtocolNormalizedContentParser.Parse( + CreateRequest(), + CreateContentMap("ghg_protocol_malformed_factors.csv")); + + Assert.Equal(root.GetProperty("malformed_status").GetProperty("dotnet").GetString(), result.Status.ToString()); + Assert.Equal(0, result.RowCount); + Assert.Equal( + root.GetProperty("malformed_issues").EnumerateArray().Select(issue => issue.GetProperty("code").GetString()), + result.ValidationIssues.Select(issue => issue.Code)); + Assert.Equal( + root.GetProperty("malformed_issues").EnumerateArray().Select(issue => issue.GetProperty("field_key").GetString()), + result.ValidationIssues.Select(issue => issue.FieldKey)); + Assert.Equal( + root.GetProperty("malformed_issues").EnumerateArray().Select(issue => (int?)issue.GetProperty("source_row_number").GetInt32()), + result.ValidationIssues.Select(issue => issue.SourceRowNumber)); + } + + [Fact] + public void UnsupportedGhgProtocolRowsAreSkippedWithWarnings() + { + using var expectations = LoadParityExpectations(); + var root = expectations.RootElement; + var content = string.Join( + "\n", + [ + string.Join(",", GhgProtocolNormalizedContentParser.Header), + "metadata,2024,v1,NOTE-001,Workbook note,0,none,Notes,,,,skip", + ]); + + var result = GhgProtocolNormalizedContentParser.Parse( + CreateRequest(), + new Dictionary { [ArtifactReference] = content }); + + Assert.Equal(root.GetProperty("unsupported_only_status").GetProperty("dotnet").GetString(), result.Status.ToString()); + Assert.Equal(0, result.RowCount); + Assert.Equal( + JsonStringArray(root.GetProperty("unsupported_only_issue_codes")), + result.ValidationIssues.Select(issue => issue.Code)); + Assert.Equal(ParserValidationIssueSeverity.Warning, result.ValidationIssues[0].Severity); + Assert.Equal("record_type", result.ValidationIssues[0].FieldKey); + } + + [Fact] + public void InvalidGhgProtocolHeaderReturnsFailedIssue() + { + var result = GhgProtocolNormalizedContentParser.Parse( + CreateRequest(), + new Dictionary { [ArtifactReference] = "record_type,source_year\nemission_factor,2024\n" }); + + Assert.Equal(ParserRunStatus.Failed, result.Status); + Assert.Equal("GHG_PROTOCOL_CONTENT_INVALID_HEADER", result.ValidationIssues[0].Code); + Assert.Equal("header", result.ValidationIssues[0].FieldKey); + } + + private const string ArtifactReference = "tests/fixtures/source_documents/ghg_protocol/ghg_protocol_sample_factors.csv"; + private const string ChecksumValue = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + + private static ParserAdapterRunRequest CreateRequest() + { + var parserKey = ParserSelectionRegistry.GetParserKey(SourceFamily.GhgProtocol); + var artifact = new ParserInputArtifact( + SourceFamily.GhgProtocol, + SourceFamily.GhgProtocol.ToWireName(), + parserKey, + ParserSourceFormat.DiscoveryReference, + ArtifactReference, + "ghg_protocol_sample_factors.csv", + "sha256", + ChecksumValue, + isDryRunChecksum: false, + "text/csv", + ".csv", + 2024); + + return new ParserAdapterRunRequest( + SourceFamily.GhgProtocol, + SourceFamily.GhgProtocol.ToWireName(), + parserKey, + [artifact], + requestedReportingYear: 2024); + } + + private static IReadOnlyDictionary CreateContentMap(string fixtureName) => + new Dictionary + { + [ArtifactReference] = File.ReadAllText(Path.Combine(FixtureDirectory(), fixtureName)), + }; + + private static JsonDocument LoadParityExpectations() => + JsonDocument.Parse(File.ReadAllText(Path.Combine(ParityFixtureDirectory(), "ghg_protocol_normalized_output_expectations.json"))); + + private static IReadOnlyList JsonStringArray(JsonElement array) => + array.EnumerateArray().Select(item => item.GetString() ?? string.Empty).ToArray(); + + private static IReadOnlyList JsonFieldArray(JsonElement array) => + array + .EnumerateArray() + .Select(field => + { + var values = field.EnumerateArray().ToArray(); + return new ParserNormalizedField(values[0].GetString() ?? string.Empty, values[1].GetString()); + }) + .ToArray(); + + private static string FixtureDirectory() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixtureDirectory = Path.Combine( + directory.FullName, + "tests", + "fixtures", + "source_documents", + "ghg_protocol"); + if (Directory.Exists(fixtureDirectory)) + { + return fixtureDirectory; + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate GHG Protocol fixture directory."); + } + + private static string ParityFixtureDirectory() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixtureDirectory = Path.Combine(directory.FullName, "tests", "fixtures", "parity"); + if (Directory.Exists(fixtureDirectory)) + { + return fixtureDirectory; + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate parity fixture directory."); + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgSourceDiscoveryBoundaryTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgSourceDiscoveryBoundaryTests.cs new file mode 100644 index 00000000..9a2b8bc5 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgSourceDiscoveryBoundaryTests.cs @@ -0,0 +1,305 @@ +using System.Reflection; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class GhgSourceDiscoveryBoundaryTests +{ + [Fact] + public void RequestIsDeterministicAndRuntimePassive() + { + var first = GhgSourceDiscoveryBoundary.CreateRequest(); + var second = GhgSourceDiscoveryBoundary.CreateRequest(); + + Assert.Equal(first, second); + Assert.Equal(SourceFamily.GhgProtocol, first.SourceFamily); + Assert.Equal("ghg_protocol", first.SourceKey); + Assert.Equal("discovery://ghg_protocol/acquisition", first.DiscoveryReferenceUri); + Assert.Equal(GhgSourceDiscoveryMode.RuntimePassive, first.Mode); + Assert.False(first.AllowNetwork); + Assert.False(first.AllowDownload); + Assert.False(first.AllowParse); + Assert.False(first.AllowDatabaseWrites); + Assert.False(first.AllowScheduler); + Assert.True(GhgSourceDiscoveryBoundary.Validate(first).IsValid); + } + + [Fact] + public void ResultDeclaresGhgCandidateWithoutRuntimeWork() + { + var result = GhgSourceDiscoveryBoundary.CreateResult(); + + Assert.Equal(GhgSourceDiscoveryStatus.Declared, result.Status); + Assert.Equal(1, result.CandidateCount); + Assert.Equal(["ghg_source_discovery_candidate_001_ghg_protocol"], result.CandidateIds); + Assert.True(result.NoNetwork); + Assert.True(result.NoDownload); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.True(result.NoSql); + Assert.True(result.NoScheduler); + Assert.True(GhgSourceDiscoveryBoundary.Validate(result).IsValid); + + var candidate = result.Candidates[0]; + Assert.Equal(SourceFamily.GhgProtocol, candidate.SourceFamily); + Assert.Equal("ghg_protocol", candidate.SourceKey); + Assert.Equal("GHG Protocol", candidate.Title); + Assert.Equal("discovery://ghg_protocol/acquisition", candidate.ReferenceUri); + Assert.Equal("discovery", candidate.ArtifactKind); + Assert.Equal(GhgSourceDiscoveryStatus.Declared, candidate.Status); + Assert.Equal("dn045_ghg_discovery_boundary", candidate.VersionLabel); + Assert.Equal("runtime_passive_discovery_unavailable", candidate.DiscoveredAtLabel); + Assert.False(candidate.DownloadAllowed); + } + + [Fact] + public void BoundaryIsGhgOnly() + { + var result = GhgSourceDiscoveryBoundary.CreateResult(); + + Assert.Equal([SourceFamily.GhgProtocol], result.Candidates.Select(candidate => candidate.SourceFamily)); + Assert.Equal(["ghg_protocol"], result.Candidates.Select(candidate => candidate.SourceKey)); + Assert.DoesNotContain("defra_desnz", result.CandidateIds); + Assert.DoesNotContain("ipcc_efdb", result.CandidateIds); + } + + [Fact] + public void InvalidRequestFailsClosedWithNoCandidates() + { + var request = new GhgSourceDiscoveryRequest( + SourceFamily.GhgProtocol, + "defra_desnz", + "discovery://ghg_protocol/acquisition", + allowNetwork: true, + allowDownload: true, + allowParse: true, + allowDatabaseWrites: true, + allowScheduler: true); + + var result = GhgSourceDiscoveryBoundary.CreateResult(request); + + Assert.Equal(GhgSourceDiscoveryStatus.Invalid, result.Status); + Assert.Empty(result.Candidates); + Assert.True(result.NoNetwork); + Assert.True(result.NoDownload); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.Equal( + [ + "GHG_SOURCE_DISCOVERY_SOURCE_KEY_MISMATCH", + "GHG_SOURCE_DISCOVERY_NETWORK_NOT_ALLOWED", + "GHG_SOURCE_DISCOVERY_DOWNLOAD_NOT_ALLOWED", + "GHG_SOURCE_DISCOVERY_PARSE_NOT_ALLOWED", + "GHG_SOURCE_DISCOVERY_DATABASE_WRITES_NOT_ALLOWED", + "GHG_SOURCE_DISCOVERY_SCHEDULER_NOT_ALLOWED", + ], + result.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void CandidateInvalidInputsFailClosed() + { + var candidate = new GhgSourceDocumentCandidate( + SourceFamily.DefraDesnz, + "defra_desnz", + "candidate-1", + "", + "discovery://ghg_protocol/acquisition", + "xlsx", + GhgSourceDiscoveryStatus.Invalid, + documentYear: 0, + reportingYear: -1, + downloadAllowed: true); + + var result = GhgSourceDiscoveryBoundary.Validate(candidate); + + Assert.False(result.IsValid); + Assert.Equal( + [ + "GHG_SOURCE_DISCOVERY_CANDIDATE_MISSING_TITLE", + "GHG_SOURCE_DISCOVERY_CANDIDATE_INVALID_DOCUMENT_YEAR", + "GHG_SOURCE_DISCOVERY_CANDIDATE_INVALID_REPORTING_YEAR", + "GHG_SOURCE_DISCOVERY_CANDIDATE_SOURCE_FAMILY_MISMATCH", + "GHG_SOURCE_DISCOVERY_CANDIDATE_SOURCE_KEY_MISMATCH", + "GHG_SOURCE_DISCOVERY_CANDIDATE_ARTIFACT_KIND_MISMATCH", + "GHG_SOURCE_DISCOVERY_CANDIDATE_UNSUPPORTED_STATUS", + "GHG_SOURCE_DISCOVERY_CANDIDATE_DOWNLOAD_NOT_ALLOWED", + ], + result.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void CandidateReferenceIsMetadataOnly() + { + var candidate = new GhgSourceDocumentCandidate( + SourceFamily.GhgProtocol, + "ghg_protocol", + "ghg-source-remote-candidate", + "GHG Protocol remote metadata", + "https://example.invalid/not-fetched.csv", + "discovery"); + + var result = GhgSourceDiscoveryBoundary.Validate(candidate); + + Assert.True(result.IsValid); + Assert.Empty(result.Issues); + } + + [Fact] + public void ValidationDoesNotRequireNetworkFileDatabaseParserDownloaderOrSchedulerRuntime() + { + var candidate = new GhgSourceDocumentCandidate( + SourceFamily.GhgProtocol, + "ghg_protocol", + "ghg-source-local-reference-candidate", + "GHG Protocol local metadata", + "/definitely/not-present/ghg-protocol-factors.csv", + "discovery"); + var result = new GhgSourceDiscoveryResult( + GhgSourceDiscoveryStatus.Declared, + GhgSourceDiscoveryBoundary.CreateRequest(), + [candidate]); + + Assert.True(GhgSourceDiscoveryBoundary.Validate(candidate).IsValid); + Assert.True(GhgSourceDiscoveryBoundary.Validate(result).IsValid); + Assert.True(result.NoNetwork); + Assert.True(result.NoDownload); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.True(result.NoSql); + Assert.True(result.NoScheduler); + } + + [Fact] + public void ResultValidationRejectsSideEffectFlags() + { + var valid = GhgSourceDiscoveryBoundary.CreateResult(); + var result = new GhgSourceDiscoveryResult( + valid.Status, + valid.Request, + valid.Candidates, + valid.Issues, + noNetwork: false, + noSql: false); + + var validation = GhgSourceDiscoveryBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "GHG_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED", + "GHG_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED", + "GHG_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + ], + validation.Issues.Select(issue => issue.Code)); + Assert.Equal(["no_network", "no_sql"], validation.Issues.Take(2).Select(issue => issue.FieldName)); + } + + [Fact] + public void ResultValidationRejectsDeclaredResultsWithIssueMetadata() + { + var valid = GhgSourceDiscoveryBoundary.CreateResult(); + var result = new GhgSourceDiscoveryResult( + GhgSourceDiscoveryStatus.Declared, + valid.Request, + valid.Candidates, + [ + new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_TEST_ISSUE", + "test issue", + "test"), + ]); + + var validation = GhgSourceDiscoveryBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "GHG_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + "GHG_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + ], + validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void ResultValidationRejectsUndefinedStatus() + { + var valid = GhgSourceDiscoveryBoundary.CreateResult(); + var result = new GhgSourceDiscoveryResult( + (GhgSourceDiscoveryStatus)999, + valid.Request, + valid.Candidates, + [ + new GhgSourceDiscoveryIssue( + "GHG_SOURCE_DISCOVERY_TEST_ISSUE", + "test issue", + "test"), + ]); + + var validation = GhgSourceDiscoveryBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + ["GHG_SOURCE_DISCOVERY_RESULT_INVALID_STATUS"], + validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void BoundaryPublicSurfaceDoesNotExposeRuntimeExecutionMethods() + { + var publicMethodNames = typeof(GhgSourceDiscoveryBoundary) + .GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Select(method => method.Name) + .ToArray(); + + Assert.Contains("CreateRequest", publicMethodNames); + Assert.Contains("CreateResult", publicMethodNames); + Assert.Equal(3, publicMethodNames.Count(methodName => methodName == "Validate")); + Assert.DoesNotContain("Discover", publicMethodNames); + Assert.DoesNotContain("Fetch", publicMethodNames); + Assert.DoesNotContain("Parse", publicMethodNames); + Assert.DoesNotContain("Execute", publicMethodNames); + } + + [Fact] + public void BoundaryTypesDoNotExposeRuntimeExecutionMethods() + { + var publicMethodNames = new[] + { + typeof(GhgSourceDiscoveryRequest), + typeof(GhgSourceDocumentCandidate), + typeof(GhgSourceDiscoveryResult), + typeof(GhgSourceDiscoveryIssue), + typeof(GhgSourceDiscoveryValidationResult), + } + .SelectMany(type => type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + .Where(method => !method.Name.StartsWith("get_", StringComparison.Ordinal)) + .Select(method => method.Name) + .ToArray(); + + Assert.DoesNotContain("Discover", publicMethodNames); + Assert.DoesNotContain("Fetch", publicMethodNames); + Assert.DoesNotContain("Parse", publicMethodNames); + Assert.DoesNotContain("Execute", publicMethodNames); + Assert.DoesNotContain("Schedule", publicMethodNames); + Assert.DoesNotContain("Persist", publicMethodNames); + Assert.DoesNotContain("Open", publicMethodNames); + Assert.DoesNotContain("Read", publicMethodNames); + Assert.DoesNotContain("Write", publicMethodNames); + } + + [Fact] + public void GhgDiscoveryWireNamesArePythonAligned() + { + Assert.Equal("runtime_passive", GhgSourceDiscoveryMode.RuntimePassive.ToWireName()); + Assert.Equal("declared", GhgSourceDiscoveryStatus.Declared.ToWireName()); + Assert.Equal("invalid", GhgSourceDiscoveryStatus.Invalid.ToWireName()); + Assert.True(ContractWireNames.TryParseGhgSourceDiscoveryModeWireName("runtime_passive", out var parsedMode)); + Assert.Equal(GhgSourceDiscoveryMode.RuntimePassive, parsedMode); + Assert.True(ContractWireNames.TryParseGhgSourceDiscoveryStatusWireName("declared", out var parsedStatus)); + Assert.Equal(GhgSourceDiscoveryStatus.Declared, parsedStatus); + Assert.False(ContractWireNames.TryParseGhgSourceDiscoveryStatusWireName("unknown", out _)); + Assert.Throws(() => ((GhgSourceDiscoveryMode)999).ToWireName()); + Assert.Throws(() => ((GhgSourceDiscoveryStatus)999).ToWireName()); + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgSourceDownloadExecutionBoundaryTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgSourceDownloadExecutionBoundaryTests.cs new file mode 100644 index 00000000..128081fc --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/GhgSourceDownloadExecutionBoundaryTests.cs @@ -0,0 +1,438 @@ +using System.Reflection; +using System.Security.Cryptography; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class GhgSourceDownloadExecutionBoundaryTests +{ + [Fact] + public void RequestFromCandidateIsExplicitOptIn() + { + var candidate = DownloadableCandidate(); + using var temp = new TemporaryDirectory(); + + var request = GhgSourceDownloadExecutionBoundary.CreateRequest( + candidate, + temp.Path, + "ghg/corporate-standard.pdf"); + + Assert.Equal(SourceFamily.GhgProtocol, request.SourceFamily); + Assert.Equal("ghg_protocol", request.SourceKey); + Assert.Equal("ghg_source_discovery_candidate_001_ghg_protocol", request.CandidateId); + Assert.Equal("GHG Protocol", request.CandidateTitle); + Assert.Equal("mock://ghg_protocol/corporate-standard.pdf", request.SourceReferenceUri); + Assert.Equal("pdf", request.ArtifactKind); + Assert.True(request.CandidateDownloadAllowed); + Assert.False(request.AllowDownloadExecution); + Assert.False(request.AllowFileWrite); + Assert.Equal("application/pdf", request.ContentType); + Assert.Equal(".pdf", request.Extension); + Assert.Equal("dn046_mock_download", request.VersionLabel); + + var validation = GhgSourceDownloadExecutionBoundary.Validate(request); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "GHG_SOURCE_DOWNLOAD_EXECUTION_NOT_ALLOWED", + "GHG_SOURCE_DOWNLOAD_FILE_WRITE_NOT_ALLOWED", + ], + validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void DefaultDiscoveryCandidateIsNotDownloadable() + { + using var temp = new TemporaryDirectory(); + var candidate = GhgSourceDiscoveryBoundary.CreateResult().Candidates[0]; + var request = GhgSourceDownloadExecutionBoundary.CreateRequest( + candidate, + temp.Path, + "ghg/source.discovery", + allowDownloadExecution: true, + allowFileWrite: true); + + var result = GhgSourceDownloadExecutionBoundary.Execute(request, UnexpectedTransport); + + Assert.Equal(GhgSourceDownloadExecutionStatus.Blocked, result.Status); + Assert.False(result.Downloaded); + Assert.Null(result.Artifact); + Assert.Equal( + [ + "GHG_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE", + "GHG_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE", + ], + result.Issues.Select(issue => issue.Code)); + Assert.False(File.Exists(Path.Combine(temp.Path, "ghg/source.discovery"))); + } + + [Fact] + public void InvalidDownloadRequestFailsClosedBeforeTransport() + { + using var temp = new TemporaryDirectory(); + var request = ValidRequest(temp.Path) with + { + SourceFamily = SourceFamily.DefraDesnz, + SourceKey = "defra_desnz", + AllowParse = true, + AllowDatabaseWrites = true, + AllowScheduler = true, + }; + + var result = GhgSourceDownloadExecutionBoundary.Execute(request, UnexpectedTransport); + + Assert.Equal(GhgSourceDownloadExecutionStatus.Blocked, result.Status); + Assert.False(result.Downloaded); + Assert.Null(result.Artifact); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.True(result.NoSql); + Assert.True(result.NoScheduler); + Assert.Equal( + [ + "GHG_SOURCE_DOWNLOAD_SOURCE_FAMILY_MISMATCH", + "GHG_SOURCE_DOWNLOAD_SOURCE_KEY_MISMATCH", + "GHG_SOURCE_DOWNLOAD_PARSE_NOT_ALLOWED", + "GHG_SOURCE_DOWNLOAD_DATABASE_WRITES_NOT_ALLOWED", + "GHG_SOURCE_DOWNLOAD_SCHEDULER_NOT_ALLOWED", + ], + result.Issues.Select(issue => issue.Code)); + } + + [Theory] + [InlineData("source_reference_uri", "https://example.invalid/ghg.pdf", "GHG_SOURCE_DOWNLOAD_NETWORK_NOT_ALLOWED")] + [InlineData("source_reference_uri", "http://example.invalid/ghg.pdf", "GHG_SOURCE_DOWNLOAD_INSECURE_HTTP_NOT_ALLOWED")] + [InlineData("source_reference_uri", "file:///tmp/ghg.pdf", "GHG_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI")] + [InlineData("source_reference_uri", "s3://bucket/ghg.pdf", "GHG_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI")] + [InlineData("source_reference_uri", "ghg/corporate-standard.pdf", "GHG_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME")] + [InlineData("source_reference_uri", "://ghg/corporate-standard.pdf", "GHG_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI")] + [InlineData("source_reference_uri", "https:///ghg.pdf", "GHG_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI")] + [InlineData("target_root", "relative/root", "GHG_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_ABSOLUTE")] + [InlineData("target_relative_path", "../outside.pdf", "GHG_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE")] + [InlineData("target_relative_path", "/absolute.pdf", "GHG_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_ABSOLUTE")] + [InlineData("target_relative_path", "download://ghg/source.pdf", "GHG_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_URI")] + public void UnsafeRequestInputsFailClosed(string fieldName, string value, string expectedCode) + { + using var temp = new TemporaryDirectory(); + var request = WithField(ValidRequest(temp.Path), fieldName, value); + + var validation = GhgSourceDownloadExecutionBoundary.Validate(request); + + Assert.False(validation.IsValid); + Assert.Contains(expectedCode, validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void SuccessfulDownloadIsExplicitAndUsesInjectedTransport() + { + using var temp = new TemporaryDirectory(); + var payload = "deterministic ghg source bytes"u8.ToArray(); + var calls = new List(); + var request = ValidRequest(temp.Path); + + var result = GhgSourceDownloadExecutionBoundary.Execute( + request, + sourceReferenceUri => + { + calls.Add(sourceReferenceUri); + return new GhgSourceDownloadTransportResponse( + payload, + "application/pdf", + "mock://ghg_protocol/final.pdf"); + }); + + var targetPath = Path.Combine(temp.Path, "ghg/corporate-standard.pdf"); + var checksum = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); + Assert.Equal(["mock://ghg_protocol/corporate-standard.pdf"], calls); + Assert.Equal(payload, File.ReadAllBytes(targetPath)); + Assert.Equal(GhgSourceDownloadExecutionStatus.Downloaded, result.Status); + Assert.True(result.Downloaded); + Assert.Equal( + new GhgSourceDownloadedArtifact( + SourceFamily.GhgProtocol, + "ghg_protocol", + "ghg_source_discovery_candidate_001_ghg_protocol", + "ghg_source_download_artifact_ghg_source_discovery_candidate_001_ghg_protocol", + "pdf", + "mock://ghg_protocol/corporate-standard.pdf", + targetPath, + "corporate-standard.pdf", + checksum, + payload.LongLength, + "application/pdf", + ".pdf", + "mock://ghg_protocol/final.pdf", + VersionLabel: "dn046_mock_download"), + result.Artifact); + Assert.True(GhgSourceDownloadExecutionBoundary.Validate(result).IsValid); + } + + [Fact] + public void TargetExistsBlocksBeforeTransportByDefault() + { + using var temp = new TemporaryDirectory(); + var request = ValidRequest(temp.Path); + var targetPath = Path.Combine(temp.Path, request.TargetRelativePath); + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + File.WriteAllBytes(targetPath, "existing"u8.ToArray()); + + var result = GhgSourceDownloadExecutionBoundary.Execute(request, UnexpectedTransport); + + Assert.Equal(GhgSourceDownloadExecutionStatus.Blocked, result.Status); + Assert.Equal(["GHG_SOURCE_DOWNLOAD_TARGET_EXISTS"], result.Issues.Select(issue => issue.Code)); + Assert.Equal("existing"u8.ToArray(), File.ReadAllBytes(targetPath)); + } + + [Fact] + public void ExistingFinalTargetSymlinkIsRejected() + { + if (OperatingSystem.IsWindows()) + { + return; + } + + using var temp = new TemporaryDirectory(); + var outside = Path.Combine(temp.Path, "outside"); + var targetParent = Path.Combine(temp.Path, "target-root", "ghg"); + Directory.CreateDirectory(outside); + Directory.CreateDirectory(targetParent); + var targetPath = Path.Combine(targetParent, "escape.pdf"); + File.CreateSymbolicLink(targetPath, Path.Combine(outside, "escape.pdf")); + var request = ValidRequest(Path.Combine(temp.Path, "target-root")) with + { + TargetRelativePath = "ghg/escape.pdf", + AllowOverwrite = true, + }; + + var result = GhgSourceDownloadExecutionBoundary.Execute( + request, + _ => new GhgSourceDownloadTransportResponse("escape"u8.ToArray())); + + Assert.Equal(GhgSourceDownloadExecutionStatus.Blocked, result.Status); + Assert.Null(result.Artifact); + Assert.Equal(["GHG_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE"], result.Issues.Select(issue => issue.Code)); + Assert.False(File.Exists(Path.Combine(outside, "escape.pdf"))); + } + + [Fact] + public void ParentSymlinkSwapDuringTransportCannotEscapeTargetRoot() + { + if (OperatingSystem.IsWindows()) + { + return; + } + + using var temp = new TemporaryDirectory(); + var targetRoot = Path.Combine(temp.Path, "target-root"); + var outside = Path.Combine(temp.Path, "outside"); + var targetParent = Path.Combine(targetRoot, "ghg"); + Directory.CreateDirectory(targetParent); + Directory.CreateDirectory(outside); + var request = ValidRequest(targetRoot) with { TargetRelativePath = "ghg/escape.pdf" }; + + var result = GhgSourceDownloadExecutionBoundary.Execute( + request, + _ => + { + Directory.Delete(targetParent, recursive: true); + Directory.CreateSymbolicLink(targetParent, outside); + return new GhgSourceDownloadTransportResponse("escape"u8.ToArray()); + }); + + Assert.NotEqual(GhgSourceDownloadExecutionStatus.Downloaded, result.Status); + Assert.False(result.Downloaded); + Assert.Null(result.Artifact); + Assert.False(File.Exists(Path.Combine(outside, "escape.pdf"))); + Assert.True(Directory.Exists(targetParent)); + } + + [Fact] + public void ChecksumMismatchFailsWithoutWritingFile() + { + using var temp = new TemporaryDirectory(); + var request = ValidRequest(temp.Path) with { ExpectedChecksumSha256 = new string('a', 64) }; + + var result = GhgSourceDownloadExecutionBoundary.Execute( + request, + _ => new GhgSourceDownloadTransportResponse("unexpected"u8.ToArray())); + + Assert.Equal(GhgSourceDownloadExecutionStatus.Failed, result.Status); + Assert.Null(result.Artifact); + Assert.Equal(["GHG_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH"], result.Issues.Select(issue => issue.Code)); + Assert.False(File.Exists(Path.Combine(temp.Path, request.TargetRelativePath))); + } + + [Fact] + public void TransportErrorsAndEmptyContentAreFailedResults() + { + using var temp = new TemporaryDirectory(); + using var other = new TemporaryDirectory(); + + var failed = GhgSourceDownloadExecutionBoundary.Execute( + ValidRequest(temp.Path), + _ => throw new InvalidOperationException("offline")); + var empty = GhgSourceDownloadExecutionBoundary.Execute( + ValidRequest(other.Path), + _ => new GhgSourceDownloadTransportResponse(Array.Empty())); + + Assert.Equal(GhgSourceDownloadExecutionStatus.Failed, failed.Status); + Assert.Equal(["GHG_SOURCE_DOWNLOAD_TRANSPORT_FAILED"], failed.Issues.Select(issue => issue.Code)); + Assert.Equal(GhgSourceDownloadExecutionStatus.Failed, empty.Status); + Assert.Equal(["GHG_SOURCE_DOWNLOAD_RESPONSE_EMPTY_CONTENT"], empty.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void TransportResponseValidationFailsClosed() + { + using var missing = new TemporaryDirectory(); + using var missingContent = new TemporaryDirectory(); + using var blankMetadata = new TemporaryDirectory(); + + var missingResponse = GhgSourceDownloadExecutionBoundary.Execute( + ValidRequest(missing.Path), + _ => null!); + var missingContentResponse = GhgSourceDownloadExecutionBoundary.Execute( + ValidRequest(missingContent.Path), + _ => new GhgSourceDownloadTransportResponse(null!)); + var blankMetadataResponse = GhgSourceDownloadExecutionBoundary.Execute( + ValidRequest(blankMetadata.Path), + _ => new GhgSourceDownloadTransportResponse("content"u8.ToArray(), " ", " ")); + + Assert.Equal(GhgSourceDownloadExecutionStatus.Failed, missingResponse.Status); + Assert.Equal(["GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING"], missingResponse.Issues.Select(issue => issue.Code)); + Assert.Equal(GhgSourceDownloadExecutionStatus.Failed, missingContentResponse.Status); + Assert.Equal( + ["GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT"], + missingContentResponse.Issues.Select(issue => issue.Code)); + Assert.Equal(GhgSourceDownloadExecutionStatus.Failed, blankMetadataResponse.Status); + Assert.Equal( + [ + "GHG_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "GHG_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + ], + blankMetadataResponse.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void ResultValidationRejectsSideEffectFlags() + { + using var temp = new TemporaryDirectory(); + var result = GhgSourceDownloadExecutionBoundary.Execute( + ValidRequest(temp.Path), + _ => new GhgSourceDownloadTransportResponse("content"u8.ToArray())) with + { + NoDatabaseWrites = false, + NoSql = false, + }; + + var validation = GhgSourceDownloadExecutionBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "GHG_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED", + "GHG_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED", + ], + validation.Issues.Select(issue => issue.Code)); + Assert.Equal(["no_database_writes", "no_sql"], validation.Issues.Select(issue => issue.FieldName)); + } + + [Theory] + [InlineData(GhgSourceDownloadExecutionStatus.Blocked)] + [InlineData(GhgSourceDownloadExecutionStatus.Failed)] + public void ResultValidationRejectsBlockedOrFailedResultsWithoutIssues(GhgSourceDownloadExecutionStatus status) + { + using var temp = new TemporaryDirectory(); + var result = new GhgSourceDownloadExecutionResult(status, ValidRequest(temp.Path)); + + var validation = GhgSourceDownloadExecutionBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal(["GHG_SOURCE_DOWNLOAD_RESULT_MISSING_ISSUES"], validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void BoundaryPublicSurfaceOnlyExposesExplicitExecutionMethods() + { + var publicMethodNames = typeof(GhgSourceDownloadExecutionBoundary) + .GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Select(method => method.Name) + .ToArray(); + + Assert.Contains("CreateRequest", publicMethodNames); + Assert.Contains("Execute", publicMethodNames); + Assert.Equal(2, publicMethodNames.Count(methodName => methodName == "Validate")); + Assert.DoesNotContain("Parse", publicMethodNames); + Assert.DoesNotContain("Persist", publicMethodNames); + Assert.DoesNotContain("Schedule", publicMethodNames); + } + + [Fact] + public void GhgDownloadExecutionWireNamesArePythonAligned() + { + Assert.Equal("blocked", GhgSourceDownloadExecutionStatus.Blocked.ToWireName()); + Assert.Equal("downloaded", GhgSourceDownloadExecutionStatus.Downloaded.ToWireName()); + Assert.Equal("failed", GhgSourceDownloadExecutionStatus.Failed.ToWireName()); + Assert.True(ContractWireNames.TryParseGhgSourceDownloadExecutionStatusWireName("downloaded", out var parsed)); + Assert.Equal(GhgSourceDownloadExecutionStatus.Downloaded, parsed); + Assert.False(ContractWireNames.TryParseGhgSourceDownloadExecutionStatusWireName("unknown", out _)); + Assert.Throws(() => ((GhgSourceDownloadExecutionStatus)999).ToWireName()); + } + + private static GhgSourceDownloadExecutionRequest ValidRequest(string targetRoot) => + GhgSourceDownloadExecutionBoundary.CreateRequest( + DownloadableCandidate(), + targetRoot, + "ghg/corporate-standard.pdf", + allowDownloadExecution: true, + allowFileWrite: true); + + private static GhgSourceDocumentCandidate DownloadableCandidate() => + new( + SourceFamily.GhgProtocol, + "ghg_protocol", + "ghg_source_discovery_candidate_001_ghg_protocol", + "GHG Protocol", + "mock://ghg_protocol/corporate-standard.pdf", + "pdf", + contentType: "application/pdf", + extension: ".pdf", + versionLabel: "dn046_mock_download", + downloadAllowed: true); + + private static GhgSourceDownloadExecutionRequest WithField( + GhgSourceDownloadExecutionRequest request, + string fieldName, + string value) => + fieldName switch + { + "source_reference_uri" => request with { SourceReferenceUri = value }, + "target_root" => request with { TargetRoot = value }, + "target_relative_path" => request with { TargetRelativePath = value }, + _ => throw new ArgumentOutOfRangeException(nameof(fieldName), fieldName, "Unknown test field."), + }; + + private static GhgSourceDownloadTransportResponse UnexpectedTransport(string sourceReferenceUri) => + throw new InvalidOperationException($"transport should not be called for {sourceReferenceUri}"); + + private sealed class TemporaryDirectory : IDisposable + { + public string Path { get; } = System.IO.Path.Combine( + System.IO.Path.GetTempPath(), + $"carbonops-ghg-download-{Guid.NewGuid():N}"); + + public TemporaryDirectory() + { + Directory.CreateDirectory(Path); + } + + public void Dispose() + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccEfdbNormalizedContentParserTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccEfdbNormalizedContentParserTests.cs new file mode 100644 index 00000000..9295be0e --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccEfdbNormalizedContentParserTests.cs @@ -0,0 +1,312 @@ +using System.Text.Json; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class IpccEfdbNormalizedContentParserTests +{ + [Fact] + public void IpccEfdbHeaderIsDeterministic() + { + using var expectations = LoadParityExpectations(); + + Assert.Equal( + JsonStringArray(expectations.RootElement.GetProperty("header")), + IpccEfdbNormalizedContentParser.Header); + } + + [Fact] + public void ValidIpccEfdbContentReturnsNormalizedRows() + { + var request = CreateRequest(); + var result = IpccEfdbNormalizedContentParser.Parse( + request, + CreateContentMap("ipcc_efdb_sample_factors.csv")); + + Assert.Equal(ParserRunStatus.Completed, result.Status); + Assert.Equal(2, result.RowCount); + Assert.Single(result.ValidationIssues); + Assert.True(result.Validate().IsValid); + Assert.All(result.Rows, row => Assert.True(row.Validate().IsValid)); + Assert.Equal("IPCC_EFDB_CONTENT_UNSUPPORTED_ROW_SKIPPED", result.ValidationIssues[0].Code); + Assert.Equal(ParserValidationIssueSeverity.Warning, result.ValidationIssues[0].Severity); + + var first = result.Rows[0]; + Assert.Equal(SourceFamily.IpccEfdb, first.SourceFamily); + Assert.Equal("ipcc_efdb", first.SourceKey); + Assert.Equal("ipcc_efdb_2006_efdb-v2024_IPCC-ENERGY-CO2_row_2", first.RowIdentifier); + Assert.Equal(2, first.SourceRowNumber); + Assert.Equal(2006, first.ReportingYear); + Assert.Equal( + [ + new ParserNormalizedField("source_family", "ipcc_efdb"), + new ParserNormalizedField("source_year", "2006"), + new ParserNormalizedField("source_version", "efdb-v2024"), + new ParserNormalizedField("factor_id", "IPCC-ENERGY-CO2"), + new ParserNormalizedField("factor_name", "Stationary combustion CO2"), + new ParserNormalizedField("factor_value", "56.1"), + new ParserNormalizedField("unit", "t CO2/TJ"), + new ParserNormalizedField("category", "Energy"), + new ParserNormalizedField("subcategory", "Stationary combustion"), + new ParserNormalizedField("ipcc_sector", "1A"), + new ParserNormalizedField("gas", "CO2"), + new ParserNormalizedField("region", "Global"), + new ParserNormalizedField("technology", "Default"), + new ParserNormalizedField("provenance_artifact_reference", ArtifactReference), + new ParserNormalizedField("provenance_checksum_algorithm", "sha256"), + new ParserNormalizedField("provenance_checksum_value", ChecksumValue), + new ParserNormalizedField("provenance_row_number", "2"), + new ParserNormalizedField("provenance", "worksheet:EFDB row 12"), + new ParserNormalizedField("source_family_master_id", "ipcc_master_2006_efdb-v2024_IPCC-ENERGY-CO2"), + new ParserNormalizedField("source_family_detail_id", "ipcc_detail_2006_efdb-v2024_IPCC-ENERGY-CO2"), + new ParserNormalizedField("master_external_key", "2006:efdb-v2024:IPCC-ENERGY-CO2"), + new ParserNormalizedField("detail_external_key", "IPCC-ENERGY-CO2:t CO2/TJ:CO2:1A"), + ], + first.Fields); + + var second = result.Rows[1]; + Assert.Equal("ipcc_efdb_2019_efdb-v2024_IPCC-WASTE-CH4_row_4", second.RowIdentifier); + Assert.Equal(2019, second.ReportingYear); + Assert.Contains(new ParserNormalizedField("category", "Waste"), second.Fields); + Assert.Contains(new ParserNormalizedField("ipcc_sector", "4D"), second.Fields); + } + + [Fact] + public void IpccEfdbParserIsDeterministicForFixtureInput() + { + var request = CreateRequest(); + var content = CreateContentMap("ipcc_efdb_sample_factors.csv"); + + var first = IpccEfdbNormalizedContentParser.Parse(request, content); + var second = IpccEfdbNormalizedContentParser.Parse(request, content); + + Assert.Equal(first, second); + Assert.Equal(2, first.RowCount); + } + + [Fact] + public void ValidIpccEfdbContentMatchesSharedParityExpectations() + { + using var expectations = LoadParityExpectations(); + var root = expectations.RootElement; + var request = CreateRequest(); + var result = IpccEfdbNormalizedContentParser.Parse( + request, + CreateContentMap("ipcc_efdb_sample_factors.csv")); + var expectedRows = root.GetProperty("sample_rows").EnumerateArray().ToArray(); + + Assert.Equal( + root.GetProperty("sample_status").GetProperty("dotnet").GetString(), + result.Status.ToString()); + Assert.Equal( + JsonStringArray(root.GetProperty("sample_issue_codes")), + result.ValidationIssues.Select(issue => issue.Code)); + Assert.Equal(expectedRows.Length, result.RowCount); + + for (var index = 0; index < expectedRows.Length; index++) + { + var expected = expectedRows[index]; + var actual = result.Rows[index]; + + Assert.Equal(expected.GetProperty("row_identifier").GetString(), actual.RowIdentifier); + Assert.Equal(expected.GetProperty("source_row_number").GetInt32(), actual.SourceRowNumber); + Assert.Equal(expected.GetProperty("reporting_year").GetInt32(), actual.ReportingYear); + Assert.Equal(JsonFieldArray(expected.GetProperty("fields")), actual.Fields); + } + } + + [Fact] + public void MalformedIpccEfdbRowsReturnStructuredErrors() + { + using var expectations = LoadParityExpectations(); + var root = expectations.RootElement; + var result = IpccEfdbNormalizedContentParser.Parse( + CreateRequest(), + CreateContentMap("ipcc_efdb_malformed_factors.csv")); + + Assert.Equal( + root.GetProperty("malformed_status").GetProperty("dotnet").GetString(), + result.Status.ToString()); + Assert.Equal(0, result.RowCount); + Assert.Equal( + root.GetProperty("malformed_issues") + .EnumerateArray() + .Select(issue => issue.GetProperty("code").GetString()), + result.ValidationIssues.Select(issue => issue.Code)); + Assert.Equal( + root.GetProperty("malformed_issues") + .EnumerateArray() + .Select(issue => issue.GetProperty("field_key").GetString()), + result.ValidationIssues.Select(issue => issue.FieldKey)); + Assert.Equal( + root.GetProperty("malformed_issues") + .EnumerateArray() + .Select(issue => (int?)issue.GetProperty("source_row_number").GetInt32()), + result.ValidationIssues.Select(issue => issue.SourceRowNumber)); + Assert.Equal("year", result.ValidationIssues[0].Context.Single(context => context.Key == "raw_value").Value); + Assert.Equal("not-a-number", result.ValidationIssues[1].Context.Single(context => context.Key == "raw_value").Value); + } + + [Fact] + public void UnsupportedIpccEfdbRowsAreSkippedWithWarnings() + { + using var expectations = LoadParityExpectations(); + var root = expectations.RootElement; + var content = string.Join( + "\n", + [ + string.Join(",", IpccEfdbNormalizedContentParser.Header), + "metadata,2006,efdb-v2024,IPCC-NOTE-001,Workbook note,0,none,Notes,,metadata,CO2,,,skip", + ]); + + var result = IpccEfdbNormalizedContentParser.Parse( + CreateRequest(), + new Dictionary { [ArtifactReference] = content }); + + Assert.Equal( + root.GetProperty("unsupported_only_status").GetProperty("dotnet").GetString(), + result.Status.ToString()); + Assert.Equal(0, result.RowCount); + Assert.Equal( + JsonStringArray(root.GetProperty("unsupported_only_issue_codes")), + result.ValidationIssues.Select(issue => issue.Code)); + Assert.Equal(ParserValidationIssueSeverity.Warning, result.ValidationIssues[0].Severity); + Assert.Equal("record_type", result.ValidationIssues[0].FieldKey); + } + + [Fact] + public void InvalidIpccEfdbHeaderReturnsFailedIssue() + { + var result = IpccEfdbNormalizedContentParser.Parse( + CreateRequest(), + new Dictionary { [ArtifactReference] = "record_type,source_year\nemission_factor,2006\n" }); + + Assert.Equal(ParserRunStatus.Failed, result.Status); + Assert.Equal("IPCC_EFDB_CONTENT_INVALID_HEADER", result.ValidationIssues[0].Code); + Assert.Equal("header", result.ValidationIssues[0].FieldKey); + } + + [Fact] + public void NonIpccSourceFamilyReturnsFailedIssue() + { + var parserKey = ParserSelectionRegistry.GetParserKey(SourceFamily.GhgProtocol); + var artifact = new ParserInputArtifact( + SourceFamily.GhgProtocol, + SourceFamily.GhgProtocol.ToWireName(), + parserKey, + ParserSourceFormat.DiscoveryReference, + ArtifactReference, + "ipcc_efdb_sample_factors.csv", + "sha256", + ChecksumValue, + isDryRunChecksum: false, + "text/csv", + ".csv", + 2024); + var request = new ParserAdapterRunRequest( + SourceFamily.GhgProtocol, + SourceFamily.GhgProtocol.ToWireName(), + parserKey, + [artifact], + requestedReportingYear: 2024); + + var result = IpccEfdbNormalizedContentParser.Parse( + request, + CreateContentMap("ipcc_efdb_sample_factors.csv")); + + Assert.Equal(ParserRunStatus.Failed, result.Status); + Assert.Equal("IPCC_EFDB_CONTENT_SOURCE_FAMILY_MISMATCH", result.ValidationIssues[0].Code); + Assert.Equal("source_family", result.ValidationIssues[0].FieldKey); + } + + private const string ArtifactReference = "tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_sample_factors.csv"; + private const string ChecksumValue = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"; + + private static ParserAdapterRunRequest CreateRequest() + { + var parserKey = ParserSelectionRegistry.GetParserKey(SourceFamily.IpccEfdb); + var artifact = new ParserInputArtifact( + SourceFamily.IpccEfdb, + SourceFamily.IpccEfdb.ToWireName(), + parserKey, + ParserSourceFormat.DiscoveryReference, + ArtifactReference, + "ipcc_efdb_sample_factors.csv", + "sha256", + ChecksumValue, + isDryRunChecksum: false, + "text/csv", + ".csv", + 2024); + + return new ParserAdapterRunRequest( + SourceFamily.IpccEfdb, + SourceFamily.IpccEfdb.ToWireName(), + parserKey, + [artifact], + requestedReportingYear: 2024); + } + + private static IReadOnlyDictionary CreateContentMap(string fixtureName) => + new Dictionary + { + [ArtifactReference] = File.ReadAllText(Path.Combine(FixtureDirectory(), fixtureName)), + }; + + private static JsonDocument LoadParityExpectations() => + JsonDocument.Parse(File.ReadAllText(Path.Combine( + ParityFixtureDirectory(), + "ipcc_efdb_normalized_output_expectations.json"))); + + private static IReadOnlyList JsonStringArray(JsonElement array) => + array.EnumerateArray().Select(item => item.GetString() ?? string.Empty).ToArray(); + + private static IReadOnlyList JsonFieldArray(JsonElement array) => + array + .EnumerateArray() + .Select(field => + { + var values = field.EnumerateArray().ToArray(); + return new ParserNormalizedField(values[0].GetString() ?? string.Empty, values[1].GetString()); + }) + .ToArray(); + + private static string FixtureDirectory() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixtureDirectory = Path.Combine( + directory.FullName, + "tests", + "fixtures", + "source_documents", + "ipcc_efdb"); + if (Directory.Exists(fixtureDirectory)) + { + return fixtureDirectory; + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate IPCC EFDB fixture directory."); + } + + private static string ParityFixtureDirectory() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixtureDirectory = Path.Combine(directory.FullName, "tests", "fixtures", "parity"); + if (Directory.Exists(fixtureDirectory)) + { + return fixtureDirectory; + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate parity fixture directory."); + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccSourceDiscoveryBoundaryTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccSourceDiscoveryBoundaryTests.cs new file mode 100644 index 00000000..88c3572d --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccSourceDiscoveryBoundaryTests.cs @@ -0,0 +1,305 @@ +using System.Reflection; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class IpccSourceDiscoveryBoundaryTests +{ + [Fact] + public void RequestIsDeterministicAndRuntimePassive() + { + var first = IpccSourceDiscoveryBoundary.CreateRequest(); + var second = IpccSourceDiscoveryBoundary.CreateRequest(); + + Assert.Equal(first, second); + Assert.Equal(SourceFamily.IpccEfdb, first.SourceFamily); + Assert.Equal("ipcc_efdb", first.SourceKey); + Assert.Equal("discovery://ipcc_efdb/homepage", first.DiscoveryReferenceUri); + Assert.Equal(IpccSourceDiscoveryMode.RuntimePassive, first.Mode); + Assert.False(first.AllowNetwork); + Assert.False(first.AllowDownload); + Assert.False(first.AllowParse); + Assert.False(first.AllowDatabaseWrites); + Assert.False(first.AllowScheduler); + Assert.True(IpccSourceDiscoveryBoundary.Validate(first).IsValid); + } + + [Fact] + public void ResultDeclaresIpccCandidateWithoutRuntimeWork() + { + var result = IpccSourceDiscoveryBoundary.CreateResult(); + + Assert.Equal(IpccSourceDiscoveryStatus.Declared, result.Status); + Assert.Equal(1, result.CandidateCount); + Assert.Equal(["ipcc_source_discovery_candidate_001_ipcc_efdb"], result.CandidateIds); + Assert.True(result.NoNetwork); + Assert.True(result.NoDownload); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.True(result.NoSql); + Assert.True(result.NoScheduler); + Assert.True(IpccSourceDiscoveryBoundary.Validate(result).IsValid); + + var candidate = result.Candidates[0]; + Assert.Equal(SourceFamily.IpccEfdb, candidate.SourceFamily); + Assert.Equal("ipcc_efdb", candidate.SourceKey); + Assert.Equal("IPCC EFDB", candidate.Title); + Assert.Equal("discovery://ipcc_efdb/homepage", candidate.ReferenceUri); + Assert.Equal("discovery", candidate.ArtifactKind); + Assert.Equal(IpccSourceDiscoveryStatus.Declared, candidate.Status); + Assert.Equal("dn049_ipcc_discovery_boundary", candidate.VersionLabel); + Assert.Equal("runtime_passive_discovery_unavailable", candidate.DiscoveredAtLabel); + Assert.False(candidate.DownloadAllowed); + } + + [Fact] + public void BoundaryIsIpccOnly() + { + var result = IpccSourceDiscoveryBoundary.CreateResult(); + + Assert.Equal([SourceFamily.IpccEfdb], result.Candidates.Select(candidate => candidate.SourceFamily)); + Assert.Equal(["ipcc_efdb"], result.Candidates.Select(candidate => candidate.SourceKey)); + Assert.DoesNotContain("ghg_protocol", result.CandidateIds); + Assert.DoesNotContain("defra_desnz", result.CandidateIds); + } + + [Fact] + public void InvalidRequestFailsClosedWithNoCandidates() + { + var request = new IpccSourceDiscoveryRequest( + SourceFamily.IpccEfdb, + "ghg_protocol", + "discovery://ipcc_efdb/homepage", + allowNetwork: true, + allowDownload: true, + allowParse: true, + allowDatabaseWrites: true, + allowScheduler: true); + + var result = IpccSourceDiscoveryBoundary.CreateResult(request); + + Assert.Equal(IpccSourceDiscoveryStatus.Invalid, result.Status); + Assert.Empty(result.Candidates); + Assert.True(result.NoNetwork); + Assert.True(result.NoDownload); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.Equal( + [ + "IPCC_SOURCE_DISCOVERY_SOURCE_KEY_MISMATCH", + "IPCC_SOURCE_DISCOVERY_NETWORK_NOT_ALLOWED", + "IPCC_SOURCE_DISCOVERY_DOWNLOAD_NOT_ALLOWED", + "IPCC_SOURCE_DISCOVERY_PARSE_NOT_ALLOWED", + "IPCC_SOURCE_DISCOVERY_DATABASE_WRITES_NOT_ALLOWED", + "IPCC_SOURCE_DISCOVERY_SCHEDULER_NOT_ALLOWED", + ], + result.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void CandidateInvalidInputsFailClosed() + { + var candidate = new IpccSourceDocumentCandidate( + SourceFamily.GhgProtocol, + "ghg_protocol", + "candidate-1", + "", + "discovery://ipcc_efdb/homepage", + "xlsx", + IpccSourceDiscoveryStatus.Invalid, + documentYear: 0, + reportingYear: -1, + downloadAllowed: true); + + var result = IpccSourceDiscoveryBoundary.Validate(candidate); + + Assert.False(result.IsValid); + Assert.Equal( + [ + "IPCC_SOURCE_DISCOVERY_CANDIDATE_MISSING_TITLE", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_INVALID_DOCUMENT_YEAR", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_INVALID_REPORTING_YEAR", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_SOURCE_FAMILY_MISMATCH", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_SOURCE_KEY_MISMATCH", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_ARTIFACT_KIND_MISMATCH", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_UNSUPPORTED_STATUS", + "IPCC_SOURCE_DISCOVERY_CANDIDATE_DOWNLOAD_NOT_ALLOWED", + ], + result.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void CandidateReferenceIsMetadataOnly() + { + var candidate = new IpccSourceDocumentCandidate( + SourceFamily.IpccEfdb, + "ipcc_efdb", + "ipcc-source-remote-candidate", + "IPCC EFDB remote metadata", + "https://example.invalid/not-fetched.xlsx", + "discovery"); + + var result = IpccSourceDiscoveryBoundary.Validate(candidate); + + Assert.True(result.IsValid); + Assert.Empty(result.Issues); + } + + [Fact] + public void ValidationDoesNotRequireNetworkFileDatabaseParserDownloaderOrSchedulerRuntime() + { + var candidate = new IpccSourceDocumentCandidate( + SourceFamily.IpccEfdb, + "ipcc_efdb", + "ipcc-source-local-reference-candidate", + "IPCC EFDB local metadata", + "/definitely/not-present/ipcc-efdb-factors.xlsx", + "discovery"); + var result = new IpccSourceDiscoveryResult( + IpccSourceDiscoveryStatus.Declared, + IpccSourceDiscoveryBoundary.CreateRequest(), + [candidate]); + + Assert.True(IpccSourceDiscoveryBoundary.Validate(candidate).IsValid); + Assert.True(IpccSourceDiscoveryBoundary.Validate(result).IsValid); + Assert.True(result.NoNetwork); + Assert.True(result.NoDownload); + Assert.True(result.NoParse); + Assert.True(result.NoDatabaseWrites); + Assert.True(result.NoSql); + Assert.True(result.NoScheduler); + } + + [Fact] + public void ResultValidationRejectsSideEffectFlags() + { + var valid = IpccSourceDiscoveryBoundary.CreateResult(); + var result = new IpccSourceDiscoveryResult( + valid.Status, + valid.Request, + valid.Candidates, + valid.Issues, + noNetwork: false, + noSql: false); + + var validation = IpccSourceDiscoveryBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "IPCC_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED", + "IPCC_SOURCE_DISCOVERY_RESULT_SIDE_EFFECT_FLAG_ENABLED", + "IPCC_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + ], + validation.Issues.Select(issue => issue.Code)); + Assert.Equal(["no_network", "no_sql"], validation.Issues.Take(2).Select(issue => issue.FieldName)); + } + + [Fact] + public void ResultValidationRejectsDeclaredResultsWithIssueMetadata() + { + var valid = IpccSourceDiscoveryBoundary.CreateResult(); + var result = new IpccSourceDiscoveryResult( + IpccSourceDiscoveryStatus.Declared, + valid.Request, + valid.Candidates, + [ + new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_TEST_ISSUE", + "test issue", + "test"), + ]); + + var validation = IpccSourceDiscoveryBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "IPCC_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + "IPCC_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + ], + validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void ResultValidationRejectsUndefinedStatus() + { + var valid = IpccSourceDiscoveryBoundary.CreateResult(); + var result = new IpccSourceDiscoveryResult( + (IpccSourceDiscoveryStatus)999, + valid.Request, + valid.Candidates, + [ + new IpccSourceDiscoveryIssue( + "IPCC_SOURCE_DISCOVERY_TEST_ISSUE", + "test issue", + "test"), + ]); + + var validation = IpccSourceDiscoveryBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + ["IPCC_SOURCE_DISCOVERY_RESULT_INVALID_STATUS"], + validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void BoundaryPublicSurfaceDoesNotExposeRuntimeExecutionMethods() + { + var publicMethodNames = typeof(IpccSourceDiscoveryBoundary) + .GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Select(method => method.Name) + .ToArray(); + + Assert.Contains("CreateRequest", publicMethodNames); + Assert.Contains("CreateResult", publicMethodNames); + Assert.Equal(3, publicMethodNames.Count(methodName => methodName == "Validate")); + Assert.DoesNotContain("Discover", publicMethodNames); + Assert.DoesNotContain("Fetch", publicMethodNames); + Assert.DoesNotContain("Parse", publicMethodNames); + Assert.DoesNotContain("Execute", publicMethodNames); + } + + [Fact] + public void BoundaryTypesDoNotExposeRuntimeExecutionMethods() + { + var publicMethodNames = new[] + { + typeof(IpccSourceDiscoveryRequest), + typeof(IpccSourceDocumentCandidate), + typeof(IpccSourceDiscoveryResult), + typeof(IpccSourceDiscoveryIssue), + typeof(IpccSourceDiscoveryValidationResult), + } + .SelectMany(type => type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + .Where(method => !method.Name.StartsWith("get_", StringComparison.Ordinal)) + .Select(method => method.Name) + .ToArray(); + + Assert.DoesNotContain("Discover", publicMethodNames); + Assert.DoesNotContain("Fetch", publicMethodNames); + Assert.DoesNotContain("Parse", publicMethodNames); + Assert.DoesNotContain("Execute", publicMethodNames); + Assert.DoesNotContain("Schedule", publicMethodNames); + Assert.DoesNotContain("Persist", publicMethodNames); + Assert.DoesNotContain("Open", publicMethodNames); + Assert.DoesNotContain("Read", publicMethodNames); + Assert.DoesNotContain("Write", publicMethodNames); + } + + [Fact] + public void IpccDiscoveryWireNamesArePythonAligned() + { + Assert.Equal("runtime_passive", IpccSourceDiscoveryMode.RuntimePassive.ToWireName()); + Assert.Equal("declared", IpccSourceDiscoveryStatus.Declared.ToWireName()); + Assert.Equal("invalid", IpccSourceDiscoveryStatus.Invalid.ToWireName()); + Assert.True(ContractWireNames.TryParseIpccSourceDiscoveryModeWireName("runtime_passive", out var parsedMode)); + Assert.Equal(IpccSourceDiscoveryMode.RuntimePassive, parsedMode); + Assert.True(ContractWireNames.TryParseIpccSourceDiscoveryStatusWireName("declared", out var parsedStatus)); + Assert.Equal(IpccSourceDiscoveryStatus.Declared, parsedStatus); + Assert.False(ContractWireNames.TryParseIpccSourceDiscoveryStatusWireName("unknown", out _)); + Assert.Throws(() => ((IpccSourceDiscoveryMode)999).ToWireName()); + Assert.Throws(() => ((IpccSourceDiscoveryStatus)999).ToWireName()); + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccSourceDownloadExecutionBoundaryTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccSourceDownloadExecutionBoundaryTests.cs new file mode 100644 index 00000000..3067b2df --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/IpccSourceDownloadExecutionBoundaryTests.cs @@ -0,0 +1,502 @@ +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class IpccSourceDownloadExecutionBoundaryTests +{ + private static readonly DateTimeOffset RetrievedAt = + new(2026, 5, 12, 10, 30, 0, TimeSpan.FromHours(3)); + + [Fact] + public void RequestFromDiscoveryCandidateIsExplicitOptIn() + { + var candidate = DownloadableCandidate(); + using var temp = new TemporaryDirectory(); + + var request = IpccSourceDownloadExecutionBoundary.CreateRequest( + candidate, + temp.Path, + "ipcc/efdb.xlsx"); + + Assert.Equal(SourceFamily.IpccEfdb, request.SourceFamily); + Assert.Equal("ipcc_efdb", request.SourceKey); + Assert.Equal("ipcc_source_discovery_candidate_001_ipcc_efdb", request.CandidateId); + Assert.Equal("IPCC EFDB", request.CandidateTitle); + Assert.Equal("mock://ipcc_efdb/efdb.xlsx", request.SourceReferenceUri); + Assert.Equal("xlsx", request.ArtifactKind); + Assert.True(request.CandidateDownloadAllowed); + Assert.False(request.AllowDownloadExecution); + Assert.False(request.AllowFileWrite); + Assert.Equal("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", request.ContentType); + Assert.Equal(".xlsx", request.Extension); + Assert.Equal(2006, request.DocumentYear); + Assert.Equal(2024, request.ReportingYear); + Assert.Equal("efdb-v2024", request.VersionLabel); + + var validation = IpccSourceDownloadExecutionBoundary.Validate(request); + + Assert.False(validation.IsValid); + Assert.Equal( + [ + "IPCC_SOURCE_DOWNLOAD_EXECUTION_NOT_ALLOWED", + "IPCC_SOURCE_DOWNLOAD_FILE_WRITE_NOT_ALLOWED", + ], + validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void DefaultDiscoveryCandidateIsNotDownloadable() + { + using var temp = new TemporaryDirectory(); + var candidate = IpccSourceDiscoveryBoundary.CreateResult().Candidates[0]; + var request = IpccSourceDownloadExecutionBoundary.CreateRequest( + candidate, + temp.Path, + "ipcc/source.discovery", + allowDownloadExecution: true, + allowFileWrite: true); + + var result = IpccSourceDownloadExecutionBoundary.Execute(request, UnexpectedTransport); + + Assert.Equal(IpccSourceDownloadExecutionStatus.Blocked, result.Status); + Assert.False(result.Downloaded); + Assert.Null(result.Artifact); + Assert.Equal( + [ + "IPCC_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE", + "IPCC_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE", + ], + result.Issues.Select(issue => issue.Code)); + Assert.False(File.Exists(Path.Combine(temp.Path, "ipcc/source.discovery"))); + } + + [Theory] + [InlineData( + "source_reference_uri", + "https://example.invalid/ipcc.xlsx", + "IPCC_SOURCE_DOWNLOAD_NETWORK_NOT_ALLOWED")] + [InlineData( + "source_reference_uri", + "http://example.invalid/ipcc.xlsx", + "IPCC_SOURCE_DOWNLOAD_INSECURE_HTTP_NOT_ALLOWED")] + [InlineData( + "source_reference_uri", + "file:///tmp/ipcc.xlsx", + "IPCC_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI")] + [InlineData( + "source_reference_uri", + "ipcc/efdb.xlsx", + "IPCC_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME")] + [InlineData("target_root", "relative/root", "IPCC_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_ABSOLUTE")] + [InlineData("target_relative_path", "../outside.xlsx", "IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE")] + [InlineData("target_relative_path", "/absolute.xlsx", "IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_ABSOLUTE")] + public void UnsafeRequestInputsFailClosed(string fieldName, string value, string expectedCode) + { + using var temp = new TemporaryDirectory(); + var request = WithField(ValidRequest(temp.Path), fieldName, value); + + var validation = IpccSourceDownloadExecutionBoundary.Validate(request); + + Assert.False(validation.IsValid); + Assert.Contains(expectedCode, validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void SuccessfulDownloadUsesDiscoveryMetadataAndInjectedTransport() + { + using var temp = new TemporaryDirectory(); + var payload = "deterministic ipcc source bytes"u8.ToArray(); + var checksum = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); + var calls = new List(); + var request = ValidRequest(temp.Path) with { ExpectedChecksumSha256 = checksum }; + + var result = IpccSourceDownloadExecutionBoundary.Execute( + request, + sourceReferenceUri => + { + calls.Add(sourceReferenceUri); + return new IpccSourceDownloadTransportResponse( + payload, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "mock://ipcc_efdb/final.xlsx"); + }, + () => RetrievedAt); + + var targetPath = Path.Combine(temp.Path, "ipcc/efdb.xlsx"); + Assert.Equal(["mock://ipcc_efdb/efdb.xlsx"], calls); + Assert.Equal(payload, File.ReadAllBytes(targetPath)); + Assert.Equal(IpccSourceDownloadExecutionStatus.Downloaded, result.Status); + Assert.True(result.Downloaded); + Assert.False(result.AlreadyKnown); + Assert.Equal( + new IpccSourceDownloadedArtifact( + SourceFamily.IpccEfdb, + "ipcc_efdb", + "ipcc_source_discovery_candidate_001_ipcc_efdb", + "ipcc_source_download_artifact_ipcc_source_discovery_candidate_001_ipcc_efdb", + "xlsx", + "mock://ipcc_efdb/efdb.xlsx", + targetPath, + "efdb.xlsx", + checksum, + payload.LongLength, + RetrievedAt.ToUniversalTime(), + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".xlsx", + "mock://ipcc_efdb/final.xlsx", + DocumentYear: 2006, + ReportingYear: 2024, + VersionLabel: "efdb-v2024"), + result.Artifact); + Assert.True(IpccSourceDownloadExecutionBoundary.Validate(result).IsValid); + } + + [Fact] + public void DownloadExecutionMatchesSharedParityFixture() + { + using var document = JsonDocument.Parse(File.ReadAllText(ParityFixturePath())); + var fixture = document.RootElement; + using var temp = new TemporaryDirectory(); + + var successfulFixture = fixture.GetProperty("successful_download"); + var request = FixtureRequest(temp.Path, fixture) with + { + ExpectedChecksumSha256 = RequiredString(successfulFixture, "checksum_sha256"), + }; + var payload = Encoding.UTF8.GetBytes(RequiredString(successfulFixture, "payload_text")); + + var successful = IpccSourceDownloadExecutionBoundary.Execute( + request, + _ => new IpccSourceDownloadTransportResponse( + payload, + RequiredString(fixture, "content_type"), + RequiredString(successfulFixture, "final_uri")), + () => RetrievedAt); + + Assert.Equal(RequiredString(successfulFixture, "status"), successful.Status.ToWireName()); + Assert.Equal(successfulFixture.GetProperty("downloaded").GetBoolean(), successful.Downloaded); + Assert.Equal(successfulFixture.GetProperty("already_known").GetBoolean(), successful.AlreadyKnown); + Assert.NotNull(successful.Artifact); + Assert.Equal(SourceFamily.IpccEfdb, successful.Artifact.SourceFamily); + Assert.Equal(RequiredString(fixture, "source_key"), successful.Artifact.SourceKey); + Assert.Equal(RequiredString(fixture, "candidate_id"), successful.Artifact.CandidateId); + Assert.Equal(RequiredString(fixture, "artifact_kind"), successful.Artifact.ArtifactKind); + Assert.Equal(RequiredString(successfulFixture, "checksum_sha256"), successful.Artifact.ChecksumSha256); + Assert.Equal(successfulFixture.GetProperty("size_bytes").GetInt64(), successful.Artifact.SizeBytes); + Assert.Equal(RequiredString(fixture, "content_type"), successful.Artifact.ContentType); + Assert.Equal(RequiredString(fixture, "extension"), successful.Artifact.Extension); + Assert.Equal(RequiredString(successfulFixture, "final_uri"), successful.Artifact.FinalUri); + Assert.Equal(fixture.GetProperty("document_year").GetInt32(), successful.Artifact.DocumentYear); + Assert.Equal(fixture.GetProperty("reporting_year").GetInt32(), successful.Artifact.ReportingYear); + Assert.Equal(RequiredString(fixture, "version_label"), successful.Artifact.VersionLabel); + AssertIssueCodes(successful.Issues, successfulFixture); + + var existingFixture = fixture.GetProperty("existing_known_document"); + using var existingTemp = new TemporaryDirectory(); + var existingRequest = FixtureRequest(existingTemp.Path, fixture) with + { + ExpectedChecksumSha256 = RequiredString(existingFixture, "checksum_sha256"), + }; + var existingTarget = Path.Combine(existingTemp.Path, RequiredString(fixture, "target_relative_path")); + Directory.CreateDirectory(Path.GetDirectoryName(existingTarget)!); + File.WriteAllBytes( + existingTarget, + Encoding.UTF8.GetBytes(RequiredString(existingFixture, "payload_text"))); + + var existing = IpccSourceDownloadExecutionBoundary.Execute( + existingRequest, + UnexpectedTransport, + () => RetrievedAt); + var dotnetExisting = existingFixture.GetProperty("dotnet"); + + Assert.Equal(RequiredString(dotnetExisting, "status"), existing.Status.ToWireName()); + Assert.Equal(dotnetExisting.GetProperty("downloaded").GetBoolean(), existing.Downloaded); + Assert.Equal(dotnetExisting.GetProperty("already_known").GetBoolean(), existing.AlreadyKnown); + Assert.NotNull(existing.Artifact); + Assert.Equal(RequiredString(existingFixture, "checksum_sha256"), existing.Artifact.ChecksumSha256); + Assert.Equal(existingFixture.GetProperty("size_bytes").GetInt64(), existing.Artifact.SizeBytes); + AssertIssueCodes(existing.Issues, existingFixture); + + var mismatchFixture = fixture.GetProperty("checksum_mismatch"); + var mismatch = IpccSourceDownloadExecutionBoundary.Execute( + FixtureRequest(Path.Combine(temp.Path, "mismatch"), fixture) with + { + ExpectedChecksumSha256 = RequiredString(mismatchFixture, "expected_checksum_sha256"), + }, + _ => new IpccSourceDownloadTransportResponse( + Encoding.UTF8.GetBytes(RequiredString(mismatchFixture, "payload_text")))); + + Assert.Equal(RequiredString(mismatchFixture, "status"), mismatch.Status.ToWireName()); + Assert.Null(mismatch.Artifact); + AssertIssueCodes(mismatch.Issues, mismatchFixture); + + var blankMetadataFixture = fixture.GetProperty("blank_response_metadata"); + var blankMetadata = IpccSourceDownloadExecutionBoundary.Execute( + FixtureRequest(Path.Combine(temp.Path, "blank-metadata"), fixture), + _ => new IpccSourceDownloadTransportResponse("content"u8.ToArray(), " ", " ")); + + Assert.Equal(RequiredString(blankMetadataFixture, "status"), blankMetadata.Status.ToWireName()); + Assert.Null(blankMetadata.Artifact); + AssertIssueCodes(blankMetadata.Issues, blankMetadataFixture); + + var defaultCandidateFixture = fixture.GetProperty("default_candidate_blocked"); + var defaultCandidateRequest = IpccSourceDownloadExecutionBoundary.CreateRequest( + IpccSourceDiscoveryBoundary.CreateResult().Candidates[0], + Path.Combine(temp.Path, "default-candidate"), + RequiredString(fixture, "target_relative_path"), + allowDownloadExecution: true, + allowFileWrite: true); + + var defaultCandidate = IpccSourceDownloadExecutionBoundary.Execute( + defaultCandidateRequest, + UnexpectedTransport); + + Assert.Equal(RequiredString(defaultCandidateFixture, "status"), defaultCandidate.Status.ToWireName()); + Assert.Null(defaultCandidate.Artifact); + AssertIssueCodes(defaultCandidate.Issues, defaultCandidateFixture); + } + + [Fact] + public void ExistingKnownDocumentIsIdempotentAndDoesNotCallTransport() + { + using var temp = new TemporaryDirectory(); + var payload = "deterministic ipcc source bytes"u8.ToArray(); + var checksum = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); + var request = ValidRequest(temp.Path) with { ExpectedChecksumSha256 = checksum }; + var targetPath = Path.Combine(temp.Path, request.TargetRelativePath); + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + File.WriteAllBytes(targetPath, payload); + + var result = IpccSourceDownloadExecutionBoundary.Execute( + request, + UnexpectedTransport, + () => RetrievedAt); + + Assert.Equal(IpccSourceDownloadExecutionStatus.AlreadyKnown, result.Status); + Assert.False(result.Downloaded); + Assert.True(result.AlreadyKnown); + Assert.Empty(result.Issues); + Assert.NotNull(result.Artifact); + Assert.Equal(checksum, result.Artifact.ChecksumSha256); + Assert.Equal(targetPath, result.Artifact.LocalPath); + Assert.Equal(RetrievedAt.ToUniversalTime(), result.Artifact.RetrievedAtUtc); + Assert.True(IpccSourceDownloadExecutionBoundary.Validate(result).IsValid); + } + + [Fact] + public void ExistingUnknownDocumentBlocksBeforeTransport() + { + using var temp = new TemporaryDirectory(); + var request = ValidRequest(temp.Path); + var targetPath = Path.Combine(temp.Path, request.TargetRelativePath); + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + File.WriteAllBytes(targetPath, "existing"u8.ToArray()); + + var result = IpccSourceDownloadExecutionBoundary.Execute(request, UnexpectedTransport); + + Assert.Equal(IpccSourceDownloadExecutionStatus.Blocked, result.Status); + Assert.Equal(["IPCC_SOURCE_DOWNLOAD_TARGET_EXISTS"], result.Issues.Select(issue => issue.Code)); + Assert.Equal("existing"u8.ToArray(), File.ReadAllBytes(targetPath)); + } + + [Fact] + public void ChecksumMismatchFailsWithoutWritingFile() + { + using var temp = new TemporaryDirectory(); + var request = ValidRequest(temp.Path) with { ExpectedChecksumSha256 = new string('a', 64) }; + + var result = IpccSourceDownloadExecutionBoundary.Execute( + request, + _ => new IpccSourceDownloadTransportResponse("unexpected"u8.ToArray())); + + Assert.Equal(IpccSourceDownloadExecutionStatus.Failed, result.Status); + Assert.Null(result.Artifact); + Assert.Equal(["IPCC_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH"], result.Issues.Select(issue => issue.Code)); + Assert.False(File.Exists(Path.Combine(temp.Path, request.TargetRelativePath))); + } + + [Fact] + public void ResultValidationRejectsNonUtcRetrievalTimestamp() + { + using var temp = new TemporaryDirectory(); + var payload = "content"u8.ToArray(); + var result = IpccSourceDownloadExecutionBoundary.Execute( + ValidRequest(temp.Path), + _ => new IpccSourceDownloadTransportResponse(payload), + () => RetrievedAt) with + { + Artifact = new IpccSourceDownloadedArtifact( + SourceFamily.IpccEfdb, + "ipcc_efdb", + "ipcc_source_discovery_candidate_001_ipcc_efdb", + "ipcc_source_download_artifact_ipcc_source_discovery_candidate_001_ipcc_efdb", + "xlsx", + "mock://ipcc_efdb/efdb.xlsx", + Path.Combine(temp.Path, "ipcc/efdb.xlsx"), + "efdb.xlsx", + Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(), + payload.LongLength, + RetrievedAt), + }; + + var validation = IpccSourceDownloadExecutionBoundary.Validate(result); + + Assert.False(validation.IsValid); + Assert.Equal( + ["IPCC_SOURCE_DOWNLOAD_ARTIFACT_RETRIEVED_AT_NOT_UTC"], + validation.Issues.Select(issue => issue.Code)); + } + + [Fact] + public void BoundaryPublicSurfaceOnlyExposesExplicitExecutionMethods() + { + var publicMethodNames = typeof(IpccSourceDownloadExecutionBoundary) + .GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Select(method => method.Name) + .ToArray(); + + Assert.Contains("CreateRequest", publicMethodNames); + Assert.Contains("Execute", publicMethodNames); + Assert.Equal(2, publicMethodNames.Count(methodName => methodName == "Validate")); + Assert.DoesNotContain("Parse", publicMethodNames); + Assert.DoesNotContain("Persist", publicMethodNames); + Assert.DoesNotContain("Schedule", publicMethodNames); + } + + [Fact] + public void IpccDownloadExecutionWireNamesArePythonAligned() + { + Assert.Equal("blocked", IpccSourceDownloadExecutionStatus.Blocked.ToWireName()); + Assert.Equal("downloaded", IpccSourceDownloadExecutionStatus.Downloaded.ToWireName()); + Assert.Equal("failed", IpccSourceDownloadExecutionStatus.Failed.ToWireName()); + Assert.Equal("already_known", IpccSourceDownloadExecutionStatus.AlreadyKnown.ToWireName()); + Assert.True(ContractWireNames.TryParseIpccSourceDownloadExecutionStatusWireName( + "already_known", + out var parsed)); + Assert.Equal(IpccSourceDownloadExecutionStatus.AlreadyKnown, parsed); + Assert.False(ContractWireNames.TryParseIpccSourceDownloadExecutionStatusWireName("unknown", out _)); + Assert.Throws(() => ((IpccSourceDownloadExecutionStatus)999).ToWireName()); + } + + private static IpccSourceDownloadExecutionRequest ValidRequest(string targetRoot) => + IpccSourceDownloadExecutionBoundary.CreateRequest( + DownloadableCandidate(), + targetRoot, + "ipcc/efdb.xlsx", + allowDownloadExecution: true, + allowFileWrite: true); + + private static IpccSourceDownloadExecutionRequest FixtureRequest( + string targetRoot, + JsonElement fixture) => + IpccSourceDownloadExecutionBoundary.CreateRequest( + new IpccSourceDocumentCandidate( + SourceFamily.IpccEfdb, + RequiredString(fixture, "source_key"), + RequiredString(fixture, "candidate_id"), + RequiredString(fixture, "candidate_title"), + RequiredString(fixture, "source_reference_uri"), + RequiredString(fixture, "artifact_kind"), + documentYear: fixture.GetProperty("document_year").GetInt32(), + reportingYear: fixture.GetProperty("reporting_year").GetInt32(), + contentType: RequiredString(fixture, "content_type"), + extension: RequiredString(fixture, "extension"), + versionLabel: RequiredString(fixture, "version_label"), + downloadAllowed: true), + targetRoot, + RequiredString(fixture, "target_relative_path"), + allowDownloadExecution: true, + allowFileWrite: true); + + private static IpccSourceDocumentCandidate DownloadableCandidate() => + new( + SourceFamily.IpccEfdb, + "ipcc_efdb", + "ipcc_source_discovery_candidate_001_ipcc_efdb", + "IPCC EFDB", + "mock://ipcc_efdb/efdb.xlsx", + "xlsx", + documentYear: 2006, + reportingYear: 2024, + contentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + extension: ".xlsx", + versionLabel: "efdb-v2024", + downloadAllowed: true); + + private static IpccSourceDownloadExecutionRequest WithField( + IpccSourceDownloadExecutionRequest request, + string fieldName, + string value) => + fieldName switch + { + "source_reference_uri" => request with { SourceReferenceUri = value }, + "target_root" => request with { TargetRoot = value }, + "target_relative_path" => request with { TargetRelativePath = value }, + _ => throw new ArgumentOutOfRangeException(nameof(fieldName), fieldName, "Unknown test field."), + }; + + private static IpccSourceDownloadTransportResponse UnexpectedTransport(string sourceReferenceUri) => + throw new InvalidOperationException($"transport should not be called for {sourceReferenceUri}"); + + private static void AssertIssueCodes( + IReadOnlyList issues, + JsonElement fixture) + { + Assert.Equal( + fixture + .GetProperty("issue_codes") + .EnumerateArray() + .Select(code => code.GetString() ?? string.Empty), + issues.Select(issue => issue.Code)); + } + + private static string RequiredString(JsonElement element, string propertyName) => + element.GetProperty(propertyName).GetString() + ?? throw new InvalidOperationException($"Missing string property {propertyName}."); + + private static string ParityFixturePath() + { + var current = new DirectoryInfo(AppContext.BaseDirectory); + while (current is not null) + { + var candidate = Path.Combine( + current.FullName, + "tests", + "fixtures", + "parity", + "ipcc_source_download_execution_expectations.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + current = current.Parent; + } + + throw new FileNotFoundException("IPCC source download parity fixture was not found."); + } + + private sealed class TemporaryDirectory : IDisposable + { + public string Path { get; } = System.IO.Path.Combine( + System.IO.Path.GetTempPath(), + $"carbonops-ipcc-download-{Guid.NewGuid():N}"); + + public TemporaryDirectory() + { + Directory.CreateDirectory(Path); + } + + public void Dispose() + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ParsedFactorPersistenceWriterTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ParsedFactorPersistenceWriterTests.cs new file mode 100644 index 00000000..42e4c627 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ParsedFactorPersistenceWriterTests.cs @@ -0,0 +1,296 @@ +using System.Text.Json; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class ParsedFactorPersistenceWriterTests +{ + [Theory] + [InlineData(SourceFamily.GhgProtocol, "ghg", "GHG-001", "kgco2e", "CO2e")] + [InlineData(SourceFamily.DefraDesnz, "defra", "DEFRA-001", "kWh", "CO2e")] + [InlineData(SourceFamily.IpccEfdb, "ipcc", "IPCC-001", "kg", "CH4")] + public void WriterMapsNormalizedOutputIntoSourceFamilyRecords( + SourceFamily sourceFamily, + string expectedPrefix, + string factorId, + string unit, + string gas) + { + var row = CreateRow(sourceFamily, factorId, unit, gas); + + var command = ParsedFactorPersistenceWriter.BuildCommand( + new ParserNormalizedOutputBatch([row]), + sourceDocumentId: "source-document-001"); + + Assert.Empty(command.Issues); + Assert.Single(command.MasterRecords); + Assert.Single(command.DetailRecords); + Assert.Equal(sourceFamily, command.MasterRecords[0].SourceFamily); + Assert.Equal(sourceFamily, command.DetailRecords[0].SourceFamily); + Assert.Equal("source-document-001", command.MasterRecords[0].SourceDocumentId); + Assert.Equal($"{expectedPrefix}_master_2024_fixture-v1_{factorId}", command.MasterRecords[0].SourceFamilyMasterId); + Assert.Equal($"2024:fixture-v1:{factorId}", command.MasterRecords[0].MasterExternalKey); + Assert.Equal($"{expectedPrefix}_detail_2024_fixture-v1_{factorId}", command.DetailRecords[0].SourceFamilyDetailId); + Assert.Equal(command.MasterRecords[0].SourceFamilyMasterId, command.DetailRecords[0].SourceFamilyMasterId); + Assert.Equal($"{factorId}:{unit}:{gas}", command.DetailRecords[0].DetailExternalKey); + Assert.Equal("1.25", command.DetailRecords[0].FactorValue); + Assert.Equal(unit, command.DetailRecords[0].FactorUnit); + } + + [Fact] + public void WriterMatchesSharedParityFixtureForFallbackPersistenceIntent() + { + using var document = JsonDocument.Parse(File.ReadAllText(ParityFixturePath())); + var root = document.RootElement; + var rowExpectation = root.GetProperty("normalized_row"); + var expectedCommand = root.GetProperty("expected_command"); + var sourceFamily = ParseSourceFamily(rowExpectation.GetProperty("source_family").GetString()!); + var row = new ParserNormalizedOutputRow( + sourceFamily, + rowExpectation.GetProperty("source_key").GetString()!, + new ParserKey(rowExpectation.GetProperty("parser_key").GetString()!), + rowExpectation.GetProperty("artifact_reference").GetString()!, + rowExpectation.GetProperty("row_id").GetString()!, + rowExpectation.GetProperty("source_row_number").GetInt32(), + rowExpectation.GetProperty("fields").EnumerateArray().Select(field => + new ParserNormalizedField(field[0].GetString()!, field[1].GetString())), + reportingYear: rowExpectation.GetProperty("reporting_year").GetInt32()); + + var command = ParsedFactorPersistenceWriter.BuildCommand(new ParserNormalizedOutputBatch([row])); + + Assert.Empty(command.Issues); + Assert.Equal(expectedCommand.GetProperty("master_count").GetInt32(), command.MasterRecords.Count); + Assert.Equal(expectedCommand.GetProperty("detail_count").GetInt32(), command.DetailRecords.Count); + Assert.Equal(expectedCommand.GetProperty("skipped_duplicate_count").GetInt32(), command.SkippedDuplicateCount); + var expectedMaster = expectedCommand.GetProperty("master_record"); + var expectedDetail = expectedCommand.GetProperty("detail_record"); + var master = command.MasterRecords[0]; + var detail = command.DetailRecords[0]; + Assert.Equal(expectedMaster.GetProperty("source_family").GetString(), PersistenceFamilyName(master.SourceFamily)); + Assert.Equal(expectedMaster.GetProperty("source_family_master_id").GetString(), master.SourceFamilyMasterId); + Assert.Equal(expectedMaster.GetProperty("source_document_id").GetString(), master.SourceDocumentId); + Assert.Equal(expectedMaster.GetProperty("master_external_key").GetString(), master.MasterExternalKey); + Assert.Equal(expectedMaster.GetProperty("lifecycle_status").GetString(), master.LifecycleStatus); + Assert.Equal(expectedMaster.GetProperty("record_checksum_sha256").GetString(), master.RecordChecksumSha256); + Assert.Equal(expectedMaster.GetProperty("created_at").GetString(), master.CreatedAt); + Assert.Equal(expectedMaster.GetProperty("updated_at").GetString(), master.UpdatedAt); + Assert.Equal(expectedDetail.GetProperty("source_family").GetString(), PersistenceFamilyName(detail.SourceFamily)); + Assert.Equal(expectedDetail.GetProperty("source_family_detail_id").GetString(), detail.SourceFamilyDetailId); + Assert.Equal(expectedDetail.GetProperty("source_family_master_id").GetString(), detail.SourceFamilyMasterId); + Assert.Equal(expectedDetail.GetProperty("detail_external_key").GetString(), detail.DetailExternalKey); + Assert.Equal(expectedDetail.GetProperty("factor_value").GetString(), detail.FactorValue); + Assert.Equal(expectedDetail.GetProperty("factor_unit").GetString(), detail.FactorUnit); + Assert.Equal(expectedDetail.GetProperty("lifecycle_status").GetString(), detail.LifecycleStatus); + Assert.Equal(expectedDetail.GetProperty("record_checksum_sha256").GetString(), detail.RecordChecksumSha256); + Assert.Equal(expectedDetail.GetProperty("created_at").GetString(), detail.CreatedAt); + Assert.Equal(expectedDetail.GetProperty("updated_at").GetString(), detail.UpdatedAt); + } + + [Fact] + public void WriterPersistsMappedCommandThroughRepository() + { + var repository = new FakeSourceFamilyRepository(); + var batch = new ParserNormalizedOutputBatch( + [ + CreateRow(SourceFamily.GhgProtocol, "GHG-001", "kgco2e", "CO2e"), + CreateRow(SourceFamily.DefraDesnz, "DEFRA-001", "kWh", "CO2e"), + CreateRow(SourceFamily.IpccEfdb, "IPCC-001", "kg", "CH4"), + ]); + + var result = ParsedFactorPersistenceWriter.Persist( + batch, + repository, + sourceDocumentId: "source-document-001"); + + Assert.Equal(ParsedFactorPersistenceStatus.Declared, result.Status); + Assert.Equal("fake_source_family", result.ProviderName); + Assert.Equal(3, result.AttemptedMasterCount); + Assert.Equal(3, result.AttemptedDetailCount); + Assert.Equal(3, result.PersistedMasterCount); + Assert.Equal(3, result.PersistedDetailCount); + Assert.Empty(result.Issues); + Assert.NotNull(result.Command); + Assert.Single(repository.Calls); + } + + [Fact] + public void WriterDeduplicatesIdenticalFactorIdentityDeterministically() + { + var row = CreateRow(SourceFamily.DefraDesnz, "DEFRA-001", "kWh", "CO2e"); + + var command = ParsedFactorPersistenceWriter.BuildCommand( + new ParserNormalizedOutputBatch([row, row]), + sourceDocumentId: "source-document-001"); + + Assert.Empty(command.Issues); + Assert.Equal(2, command.SkippedDuplicateCount); + Assert.Single(command.MasterRecords); + Assert.Single(command.DetailRecords); + } + + [Fact] + public void WriterResolvesDuplicateSourceDocumentIdentityDeterministically() + { + var row = CreateRow(SourceFamily.GhgProtocol, "GHG-001", "kgco2e", "CO2e"); + + var first = ParsedFactorPersistenceWriter.BuildCommand(new ParserNormalizedOutputBatch([row])); + var second = ParsedFactorPersistenceWriter.BuildCommand(new ParserNormalizedOutputBatch([row])); + + Assert.Empty(first.Issues); + Assert.Empty(second.Issues); + Assert.StartsWith("source_document_", first.MasterRecords[0].SourceDocumentId); + Assert.Equal(first.MasterRecords[0].SourceDocumentId, second.MasterRecords[0].SourceDocumentId); + } + + [Fact] + public void WriterRejectsDuplicateFactorIdentityWithDifferentContent() + { + var first = CreateRow(SourceFamily.DefraDesnz, "DEFRA-001", "kWh", "CO2e"); + var conflicting = CreateRow(SourceFamily.DefraDesnz, "DEFRA-001", "kWh", "CO2e", factorValue: "9.99"); + + var command = ParsedFactorPersistenceWriter.BuildCommand( + new ParserNormalizedOutputBatch([first, conflicting]), + sourceDocumentId: "source-document-001"); + + Assert.Contains(command.Issues, issue => issue.Code == "PARSED_FACTOR_PERSISTENCE_DUPLICATE_DETAIL_CONFLICT"); + } + + [Fact] + public void WriterRejectsMalformedPersistenceInputBeforeRepositoryCall() + { + var repository = new FakeSourceFamilyRepository(); + var malformed = CreateRow( + SourceFamily.DefraDesnz, + "DEFRA-001", + unit: "", + gas: "CO2e"); + + var result = ParsedFactorPersistenceWriter.Persist( + new ParserNormalizedOutputBatch([malformed]), + repository, + sourceDocumentId: "source-document-001"); + + Assert.Equal(ParsedFactorPersistenceStatus.FailedValidation, result.Status); + Assert.Empty(repository.Calls); + Assert.Contains(result.Issues, issue => + issue.Code == "PARSED_FACTOR_PERSISTENCE_MISSING_REQUIRED_FIELD" && + issue.FieldName == "Rows[0].factor_unit"); + } + + [Fact] + public void WriterReportsNoRecordsWithoutCallingRepository() + { + var repository = new FakeSourceFamilyRepository(); + + var result = ParsedFactorPersistenceWriter.Persist( + new ParserNormalizedOutputBatch([]), + repository); + + Assert.Equal(ParsedFactorPersistenceStatus.NoRecords, result.Status); + Assert.Empty(repository.Calls); + Assert.Equal("PARSED_FACTOR_PERSISTENCE_NO_RECORDS", result.Issues[0].Code); + } + + private static ParserNormalizedOutputRow CreateRow( + SourceFamily sourceFamily, + string factorId, + string unit, + string gas, + string factorValue = "1.25") + { + var parserKey = ParserSelectionRegistry.GetParserKey(sourceFamily); + var sourceKey = sourceFamily.ToWireName(); + var prefix = sourceFamily switch + { + SourceFamily.GhgProtocol => "ghg", + SourceFamily.DefraDesnz => "defra", + SourceFamily.IpccEfdb => "ipcc", + _ => throw new ArgumentOutOfRangeException(nameof(sourceFamily), sourceFamily, "Unknown source family."), + }; + + return new ParserNormalizedOutputRow( + sourceFamily, + sourceKey, + parserKey, + $"artifact://{sourceKey}/factors.csv", + $"{sourceKey}_{factorId}_row_1", + sourceRowNumber: 1, + [ + new ParserNormalizedField("source_family", sourceKey), + new ParserNormalizedField("source_year", "2024"), + new ParserNormalizedField("source_version", "fixture-v1"), + new ParserNormalizedField("factor_id", factorId), + new ParserNormalizedField("factor_value", factorValue), + new ParserNormalizedField("unit", unit), + new ParserNormalizedField("gas", gas), + new ParserNormalizedField("provenance_artifact_reference", $"artifact://{sourceKey}/factors.csv"), + new ParserNormalizedField("provenance_checksum_value", "c".PadLeft(64, 'c')), + new ParserNormalizedField("source_family_master_id", $"{prefix}_master_2024_fixture-v1_{factorId}"), + new ParserNormalizedField("source_family_detail_id", $"{prefix}_detail_2024_fixture-v1_{factorId}"), + new ParserNormalizedField("master_external_key", $"2024:fixture-v1:{factorId}"), + new ParserNormalizedField("detail_external_key", $"{factorId}:{unit}:{gas}"), + ], + reportingYear: 2024); + } + + private static string ParityFixturePath() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixturePath = Path.Combine( + directory.FullName, + "tests", + "fixtures", + "parity", + "parsed_factor_persistence_writer_expectations.json"); + if (File.Exists(fixturePath)) + { + return fixturePath; + } + + directory = directory.Parent; + } + + throw new FileNotFoundException("Parsed factor persistence parity fixture was not found."); + } + + private static SourceFamily ParseSourceFamily(string value) => + value switch + { + "ghg_protocol" => SourceFamily.GhgProtocol, + "defra_desnz" => SourceFamily.DefraDesnz, + "ipcc_efdb" => SourceFamily.IpccEfdb, + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown source family."), + }; + + private static string PersistenceFamilyName(SourceFamily value) => + value switch + { + SourceFamily.GhgProtocol => "ghg", + SourceFamily.DefraDesnz => "defra", + SourceFamily.IpccEfdb => "ipcc", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown source family."), + }; + + private sealed class FakeSourceFamilyRepository : ISourceFamilyRepository + { + public List<(IReadOnlyList Masters, IReadOnlyList Details)> Calls { get; } = []; + + public string ProviderName => "fake_source_family"; + + public SourceFamilyRepositoryPersistResult PersistSourceFamilyRecords( + IEnumerable masterRecords, + IEnumerable detailRecords) + { + var masterSnapshot = masterRecords.ToArray(); + var detailSnapshot = detailRecords.ToArray(); + Calls.Add((masterSnapshot, detailSnapshot)); + + return SourceFamilyRepositoryRegistry.CreatePersistResult( + ProviderName, + masterSnapshot, + detailSnapshot); + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ParserRunRepositoryContractTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ParserRunRepositoryContractTests.cs new file mode 100644 index 00000000..cb6e73f5 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ParserRunRepositoryContractTests.cs @@ -0,0 +1,139 @@ +using System.Reflection; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class ParserRunRepositoryContractTests +{ + [Fact] + public void ParserRunRepositoryInterfaceSupportsMetadataOnlyPersistenceContract() + { + IParserRunRepository repository = new InMemoryParserRunRepository(); + + var result = repository.PersistRuns(ParserRunResultRegistry.CreateDefaultDryRunResultSet().Results); + + Assert.Equal("in_memory", repository.ProviderName); + Assert.Equal(ParserRunRepositoryPersistStatus.Declared, result.Status); + Assert.Equal(3, result.PersistedCount); + Assert.Empty(result.Issues); + } + + [Fact] + public void ParserRunRepositoryValidationRequiresProviderName() + { + var validation = ParserRunRepositoryRegistry.ValidateInputs( + "", + ParserRunResultRegistry.CreateDefaultDryRunResultSet().Results); + + Assert.False(validation.IsValid); + Assert.Equal( + "PARSER_RUN_REPOSITORY_MISSING_PROVIDER_NAME", + validation.Issues[0].Code); + Assert.Equal("ProviderName", validation.Issues[0].FieldName); + } + + [Fact] + public void ParserRunRepositoryValidationRejectsNullRuns() + { + var validation = ParserRunRepositoryRegistry.ValidateInputs( + "in_memory", + [null]); + + Assert.False(validation.IsValid); + Assert.Equal("PARSER_RUN_REPOSITORY_INVALID_RUN", validation.Issues[0].Code); + Assert.Equal("Runs[0]", validation.Issues[0].FieldName); + } + + [Fact] + public void ParserRunRepositoryPersistResultReportsValidationFailure() + { + var result = ParserRunRepositoryRegistry.CreatePersistResult( + "", + [null]); + + Assert.Equal(ParserRunRepositoryPersistStatus.FailedValidation, result.Status); + Assert.Equal(0, result.PersistedCount); + Assert.Equal(2, result.Issues.Count); + } + + [Fact] + public void ParserRunRepositoryPersistResultSnapshotsIssueCollections() + { + var issues = new List + { + new("CUSTOM_REPOSITORY_WARNING", "custom issue", "Runs", "warning"), + }; + + var result = ParserRunRepositoryRegistry.CreatePersistResult( + "in_memory", + ParserRunResultRegistry.CreateDefaultDryRunResultSet().Results, + issues); + issues.Clear(); + + Assert.Equal(ParserRunRepositoryPersistStatus.FailedValidation, result.Status); + Assert.Equal(0, result.PersistedCount); + Assert.Single(result.Issues); + Assert.Equal("CUSTOM_REPOSITORY_WARNING", result.Issues[0].Code); + } + + [Fact] + public void ParserRunRepositoryPersistStatusValuesAreDeterministic() + { + Assert.Equal( + [ + ParserRunRepositoryPersistStatus.Declared, + ParserRunRepositoryPersistStatus.FailedValidation, + ], + Enum.GetValues()); + } + + [Fact] + public void ParserRunRepositoryContractRemainsRuntimePassive() + { + var publicMembers = new[] + { + typeof(IParserRunRepository), + typeof(ParserRunRepositoryIssue), + typeof(ParserRunRepositoryPersistResult), + typeof(ParserRunRepositoryRegistry), + typeof(ParserRunRepositoryValidationResult), + } + .SelectMany(type => type.GetMembers(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + .Select(member => member.Name) + .ToArray(); + var blockedTerms = new[] + { + "Db", + "Sql", + "Postgres", + "Http", + "Open", + "ReadFile", + "Write", + "StatFile", + "Exists", + "Fetch", + "Calculate", + "Factor", + }; + + foreach (var term in blockedTerms) + { + Assert.DoesNotContain(publicMembers, member => member.Contains(term, StringComparison.OrdinalIgnoreCase)); + } + + Assert.DoesNotContain("Parse", publicMembers); + Assert.DoesNotContain("Execute", publicMembers); + } + + private sealed class InMemoryParserRunRepository : IParserRunRepository + { + public string ProviderName => "in_memory"; + + public ParserRunRepositoryPersistResult PersistRuns( + IEnumerable runs) + { + return ParserRunRepositoryRegistry.CreatePersistResult(ProviderName, runs); + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/Phase1IngestionOrchestratorTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/Phase1IngestionOrchestratorTests.cs new file mode 100644 index 00000000..c9b24ad7 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/Phase1IngestionOrchestratorTests.cs @@ -0,0 +1,371 @@ +using System.Text.Json; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class Phase1IngestionOrchestratorTests +{ + [Fact] + public void OrchestratorRunsSelectedPhaseOneSourcesEndToEndWithFakes() + { + var log = new List(); + var orchestrator = CreateOrchestrator(log); + var request = new Phase1IngestionOrchestratorRequest( + [SourceFamily.GhgProtocol, SourceFamily.DefraDesnz, SourceFamily.IpccEfdb], + correlationId: "test-correlation"); + + var result = orchestrator.Run(request); + + Assert.Equal(Phase1IngestionRunStatus.Completed, result.Status); + Assert.Equal( + [SourceFamily.GhgProtocol, SourceFamily.DefraDesnz, SourceFamily.IpccEfdb], + result.SelectedSourceFamilies); + Assert.Equal(3, result.SourceFamilyCount); + Assert.Equal(3, result.CompletedSourceFamilyCount); + Assert.Equal(0, result.FailedSourceFamilyCount); + Assert.Equal(3, result.TotalSourceDocumentMetadataCount); + Assert.Equal(3, result.TotalParserAcceptedRowCount); + Assert.Equal(3, result.TotalPersistedMasterCount); + Assert.Equal(3, result.TotalPersistedDetailCount); + Assert.Empty(result.Failures); + Assert.Equal( + [ + "ghg_protocol:discover_download", + "ghg_protocol:normalize", + "defra_desnz:discover_download", + "defra_desnz:normalize", + "ipcc_efdb:discover_download", + "ipcc_efdb:normalize", + ], + log); + } + + [Fact] + public void OrchestratorEmitsCorrelationFriendlyOperationalEventsWhenSinkIsProvided() + { + var runtimeLog = new List(); + var events = new List(); + var orchestrator = CreateOrchestrator(runtimeLog); + var request = new Phase1IngestionOrchestratorRequest( + [SourceFamily.GhgProtocol], + runId: "run-001", + correlationId: "corr-001", + operationalEventSink: events.Add); + + var result = orchestrator.Run(request); + + Assert.Equal(Phase1IngestionRunStatus.Completed, result.Status); + Assert.Equal(3, events.Count); + Assert.Equal("phase1_ingestion_orchestrator_started", EventName(events[0])); + Assert.Equal("phase1_source_family_completed", EventName(events[1])); + Assert.Equal("phase1_ingestion_orchestrator_completed", EventName(events[2])); + Assert.Contains("\"run_id\":\"run-001\"", events[1], StringComparison.Ordinal); + Assert.Contains("\"correlation_id\":\"corr-001\"", events[1], StringComparison.Ordinal); + Assert.Contains("\"source_family\":\"ghg_protocol\"", events[1], StringComparison.Ordinal); + Assert.Contains("\"source_key\":\"ghg_protocol\"", events[1], StringComparison.Ordinal); + Assert.Contains("\"status\":\"completed\"", events[1], StringComparison.Ordinal); + Assert.Contains("\"document_id\":\"ghg_protocol_artifact\"", events[1], StringComparison.Ordinal); + Assert.Contains("\"accepted_row_count\":1", events[1], StringComparison.Ordinal); + Assert.Contains("\"validation_issue_count\":0", events[1], StringComparison.Ordinal); + Assert.Contains("\"parsed_factor_master_count\":1", events[1], StringComparison.Ordinal); + Assert.Contains("\"execution_mode\":\"sequential\"", events[0], StringComparison.Ordinal); + Assert.Contains("\"max_degree_of_parallelism\":1", events[0], StringComparison.Ordinal); + Assert.DoesNotContain("password=", string.Join("\n", events), StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("connection_string=", string.Join("\n", events), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void OrchestratorRequiresExplicitSourceFamilySelection() + { + var log = new List(); + var orchestrator = CreateOrchestrator(log); + var request = new Phase1IngestionOrchestratorRequest([]); + + var result = orchestrator.Run(request); + + Assert.Equal(Phase1IngestionRunStatus.NotExecutable, result.Status); + Assert.Equal(0, result.SourceFamilyCount); + Assert.Equal(["PHASE1_SOURCE_FAMILY_SELECTION_REQUIRED"], result.Failures.Select(failure => failure.Code)); + Assert.Empty(log); + } + + [Fact] + public void OrchestratorCanRunSingleSelectedSourceFamily() + { + var log = new List(); + var orchestrator = CreateOrchestrator(log); + var request = new Phase1IngestionOrchestratorRequest([SourceFamily.DefraDesnz]); + + var result = orchestrator.Run(request); + + Assert.Equal(Phase1IngestionRunStatus.Completed, result.Status); + Assert.Single(result.FamilyResults); + Assert.Equal(SourceFamily.DefraDesnz, result.FamilyResults[0].SourceFamily); + Assert.Equal(Phase1IngestionFamilyRunStatus.Completed, result.FamilyResults[0].Status); + Assert.Equal("phase1_ingestion_orchestrator_run", result.FamilyResults[0].AcquisitionRun?.RunId); + Assert.Equal(["defra_desnz:discover_download", "defra_desnz:normalize"], log); + } + + [Fact] + public void DuplicateSourceFamilySelectionIsIdempotent() + { + var log = new List(); + var orchestrator = CreateOrchestrator(log); + var request = new Phase1IngestionOrchestratorRequest( + [SourceFamily.IpccEfdb, SourceFamily.IpccEfdb]); + + var result = orchestrator.Run(request); + + Assert.Equal(Phase1IngestionRunStatus.Completed, result.Status); + Assert.Equal([SourceFamily.IpccEfdb], result.SelectedSourceFamilies); + Assert.Single(result.FamilyResults); + Assert.Equal(["ipcc_efdb:discover_download", "ipcc_efdb:normalize"], log); + } + + [Fact] + public void PartialFailureSemanticsAreDeterministicPerSourceFamily() + { + var log = new List(); + var orchestrator = CreateOrchestrator( + log, + failingParserFamilies: new HashSet { SourceFamily.DefraDesnz }); + var request = new Phase1IngestionOrchestratorRequest( + [SourceFamily.GhgProtocol, SourceFamily.DefraDesnz, SourceFamily.IpccEfdb]); + + var result = orchestrator.Run(request); + + Assert.Equal(Phase1IngestionRunStatus.CompletedWithFailures, result.Status); + Assert.Equal( + [ + Phase1IngestionFamilyRunStatus.Completed, + Phase1IngestionFamilyRunStatus.Failed, + Phase1IngestionFamilyRunStatus.Completed, + ], + result.FamilyResults.Select(family => family.Status)); + Assert.Equal(2, result.CompletedSourceFamilyCount); + Assert.Equal(1, result.FailedSourceFamilyCount); + Assert.Contains(result.FamilyResults[1].Failures, failure => + failure.Stage == "parser" && + failure.Code.EndsWith("_CONTENT_INVALID_HEADER", StringComparison.Ordinal)); + Assert.Equal(2, result.TotalPersistedMasterCount); + Assert.Equal(2, result.TotalPersistedDetailCount); + Assert.Equal( + [ + "ghg_protocol:discover_download", + "ghg_protocol:normalize", + "defra_desnz:discover_download", + "defra_desnz:normalize", + "ipcc_efdb:discover_download", + "ipcc_efdb:normalize", + ], + log); + } + + [Fact] + public void BoundedParallelExecutionIsAnExplicitExtensionPoint() + { + var log = new List(); + var orchestrator = CreateOrchestrator(log); + var request = new Phase1IngestionOrchestratorRequest( + [SourceFamily.GhgProtocol], + Phase1IngestionExecutionMode.BoundedParallel, + maxDegreeOfParallelism: 2); + + var result = orchestrator.Run(request); + + Assert.Equal(Phase1IngestionExecutionMode.BoundedParallel, result.Request.ExecutionMode); + Assert.Equal(2, result.Request.MaxDegreeOfParallelism); + Assert.Equal(Phase1IngestionRunStatus.NotExecutable, result.Status); + Assert.Equal([SourceFamily.GhgProtocol], result.SelectedSourceFamilies); + Assert.Equal(0, result.CompletedSourceFamilyCount); + Assert.Equal(["PHASE1_INGESTION_BOUNDED_PARALLEL_NOT_ENABLED"], result.Failures.Select(failure => failure.Code)); + Assert.Empty(log); + } + + [Fact] + public void RuntimeConfigReadinessBlocksBeforeSourceExecutionWithoutLoadingSecrets() + { + var log = new List(); + var orchestrator = CreateOrchestrator(log); + var request = new Phase1IngestionOrchestratorRequest( + [SourceFamily.IpccEfdb], + runtimeConfigGate: new PostgreSQLRuntimeConfigGate(Requested: true)); + + var result = orchestrator.Run(request); + + Assert.Equal(PostgreSQLRuntimeConfigGateStatus.Blocked, result.RuntimeConfigDecision.Status); + Assert.False(result.RuntimeConfigDecision.LoadsEnvironment); + Assert.False(result.RuntimeConfigDecision.LoadsConfigFiles); + Assert.False(result.RuntimeConfigDecision.LoadsCredentials); + Assert.Equal(Phase1IngestionRunStatus.NotExecutable, result.Status); + Assert.Equal(0, result.CompletedSourceFamilyCount); + Assert.Equal(["POSTGRESQL_RUNTIME_CONFIG_BLOCKED"], result.Failures.Select(failure => failure.Code)); + Assert.Empty(log); + } + + [Fact] + public void SchemaBootstrapReadinessBlocksBeforeSourceExecution() + { + var log = new List(); + var orchestrator = CreateOrchestrator(log); + var request = new Phase1IngestionOrchestratorRequest( + [SourceFamily.IpccEfdb], + schemaBootstrapReport: PostgreSQLSchemaBootstrapBoundary.BuildReport()); + + var result = orchestrator.Run(request); + + Assert.Equal(Phase1IngestionRunStatus.NotExecutable, result.Status); + Assert.Equal(0, result.CompletedSourceFamilyCount); + Assert.Equal("postgresql_schema_bootstrap", result.Failures[0].Stage); + Assert.Equal("PHASE1_INGESTION_POSTGRESQL_SCHEMA_NOT_READY", result.Failures[0].Code); + Assert.Empty(log); + } + + private static Phase1IngestionOrchestrator CreateOrchestrator( + List log, + IReadOnlySet? failingParserFamilies = null) + { + var runtimes = new[] + { + new FakeSourceFamilyRuntime(SourceFamily.GhgProtocol, log, failingParserFamilies ?? new HashSet()), + new FakeSourceFamilyRuntime(SourceFamily.DefraDesnz, log, failingParserFamilies ?? new HashSet()), + new FakeSourceFamilyRuntime(SourceFamily.IpccEfdb, log, failingParserFamilies ?? new HashSet()), + }; + return new Phase1IngestionOrchestrator(new Phase1IngestionOrchestratorDependencies( + runtimes, + new FakeSourceAcquisitionRunRepository(), + new FakeSourceDocumentRepository(), + new FakeParserRunRepository(), + new FakeSourceFamilyRepository())); + } + + private static string EventName(string json) + { + using var document = JsonDocument.Parse(json); + return document.RootElement.GetProperty("event").GetString() ?? ""; + } + + private sealed class FakeSourceFamilyRuntime( + SourceFamily sourceFamily, + ICollection log, + IReadOnlySet failingParserFamilies) : IPhase1SourceFamilyIngestionRuntime + { + public SourceFamily SourceFamily { get; } = sourceFamily; + + public SourceAcquisitionRunResult DiscoverAndDownload(string runId, string? correlationId) + { + var sourceKey = SourceFamily.ToWireName(); + log.Add($"{sourceKey}:discover_download"); + var candidate = new SourceDiscoveryCandidate( + SourceFamily, + sourceKey, + $"{sourceKey}_candidate", + $"{sourceKey} fixture", + 2024, + $"fixture://{sourceKey}/factors.csv", + ParserSourceFormat.DiscoveryReference, + "application/x-carbonops-discovery-reference", + ".csv", + new SourceDocumentChecksum("sha256", new string('a', 64), IsDryRunPlaceholder: false), + "fixture-v1"); + var artifact = new SourceDownloadArtifact( + SourceFamily, + sourceKey, + candidate.CandidateId, + $"{sourceKey}_artifact", + ParserSourceFormat.DiscoveryReference, + candidate.SourceReference, + $"fixture://{sourceKey}/local-factors.csv", + candidate.Title, + candidate.ContentType, + candidate.Extension, + candidate.Checksum, + sizeBytes: 128, + reportingYear: candidate.ReportingYear, + versionLabel: candidate.VersionLabel); + + return new SourceAcquisitionRunResult( + SourceFamily, + sourceKey, + SourceAcquisitionRunStatus.Completed, + [candidate], + [artifact], + runId, + correlationId, + candidate.ReportingYear, + candidate.VersionLabel); + } + + public ParserAdapterRunResult Normalize(ParserAdapterRunRequest request) + { + log.Add($"{SourceFamily.ToWireName()}:normalize"); + var content = failingParserFamilies.Contains(SourceFamily) + ? "not,the,expected,header" + : ContentFor(SourceFamily); + var contentByReference = request.Artifacts.ToDictionary( + artifact => artifact.ArtifactReference, + _ => content, + StringComparer.Ordinal); + + return SourceFamily switch + { + SourceFamily.GhgProtocol => GhgProtocolNormalizedContentParser.Parse(request, contentByReference), + SourceFamily.DefraDesnz => DefraDesnzNormalizedContentParser.Parse(request, contentByReference), + SourceFamily.IpccEfdb => IpccEfdbNormalizedContentParser.Parse(request, contentByReference), + _ => throw new ArgumentOutOfRangeException(nameof(SourceFamily), SourceFamily, "Unknown source family."), + }; + } + + private static string ContentFor(SourceFamily sourceFamily) => + sourceFamily switch + { + SourceFamily.GhgProtocol => string.Join( + "\n", + string.Join(",", GhgProtocolNormalizedContentParser.Header), + "emission_factor,2024,fixture-v1,GHG-001,Grid electricity,1.25,kg CO2e/kWh,Energy,Electricity,Scope 2,CO2e,fixture"), + SourceFamily.DefraDesnz => string.Join( + "\n", + string.Join(",", DefraDesnzNormalizedContentParser.Header), + "2024,fixture-v1,Energy,Electricity,Grid electricity,DEFRA-001,Grid electricity,1.25,kWh,CO2e,fixture"), + SourceFamily.IpccEfdb => string.Join( + "\n", + string.Join(",", IpccEfdbNormalizedContentParser.Header), + "emission_factor,2024,fixture-v1,IPCC-001,Combustion,1.25,kg,Energy,Fuel,1A,CO2,Global,Default,fixture"), + _ => throw new ArgumentOutOfRangeException(nameof(sourceFamily), sourceFamily, "Unknown source family."), + }; + } + + private sealed class FakeSourceAcquisitionRunRepository : ISourceAcquisitionRunRepository + { + public string ProviderName => "fake_source_acquisition_runs"; + + public SourceAcquisitionRunRepositoryPersistResult PersistRuns(IEnumerable runs) => + SourceAcquisitionRunRepositoryRegistry.CreatePersistResult(ProviderName, runs); + } + + private sealed class FakeSourceDocumentRepository : ISourceDocumentRepository + { + public string ProviderName => "fake_source_documents"; + + public SourceDocumentRepositoryPersistResult PersistSourceDocuments(IEnumerable records) => + SourceDocumentRepositoryRegistry.CreatePersistResult(ProviderName, records); + } + + private sealed class FakeParserRunRepository : IParserRunRepository + { + public string ProviderName => "fake_parser_runs"; + + public ParserRunRepositoryPersistResult PersistRuns(IEnumerable runs) => + ParserRunRepositoryRegistry.CreatePersistResult(ProviderName, runs); + } + + private sealed class FakeSourceFamilyRepository : ISourceFamilyRepository + { + public string ProviderName => "fake_source_family"; + + public SourceFamilyRepositoryPersistResult PersistSourceFamilyRecords( + IEnumerable masterRecords, + IEnumerable detailRecords) => + SourceFamilyRepositoryRegistry.CreatePersistResult(ProviderName, masterRecords, detailRecords); + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/Phase1OperationalDiagnosticsTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/Phase1OperationalDiagnosticsTests.cs new file mode 100644 index 00000000..4dd3f7d9 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/Phase1OperationalDiagnosticsTests.cs @@ -0,0 +1,276 @@ +using System.Text.Json; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class Phase1OperationalDiagnosticsTests +{ + [Fact] + public void PostgreSQLOptionsDiagnosticsRedactSensitiveRuntimeValues() + { + var options = new PostgreSQLPersistenceOptions( + "db.internal.example", + 5432, + "carbonops_prod", + "service_user", + PasswordSet: true, + SslMode: "require", + ApplicationName: "carbonops-phase1", + ConnectTimeoutSeconds: 10); + + var summary = Phase1OperationalDiagnostics.SummarizePostgreSQLOptionsForDiagnostics(options); + + Assert.Equal(Phase1OperationalDiagnostics.Redacted, summary["application_name"]); + Assert.Equal(10, summary["connect_timeout_seconds"]); + Assert.Equal(Phase1OperationalDiagnostics.Redacted, summary["database"]); + Assert.Equal(Phase1OperationalDiagnostics.Redacted, summary["host"]); + Assert.Equal(true, summary["password_set"]); + Assert.Equal(5432, summary["port"]); + Assert.Equal("require", summary["ssl_mode"]); + Assert.Equal(Phase1OperationalDiagnostics.Redacted, summary["username"]); + } + + [Fact] + public void RedactionRemovesSecretFieldsAndConnectionUserInfo() + { + var value = new Dictionary + { + ["password"] = "super-secret", + ["connectionString"] = "Host=db;Username=svc;" + "Password" + "=raw-secret", + ["apiKey"] = "api-secret", + ["nested"] = new Dictionary + { + ["message"] = "failed dsn=postgresql://svc:secret@db.internal/carbonops " + "connectionString" + "=postgresql://svc:raw-secret@db.internal/carbonops " + "token" + "=abc123", + }, + ["safe_count"] = 3, + }; + + var redacted = Phase1OperationalDiagnostics.RedactDiagnosticValue("payload", value); + var redactedMapping = Assert.IsAssignableFrom>(redacted); + var nestedMapping = Assert.IsAssignableFrom>( + redactedMapping["nested"]); + + Assert.Equal( + "failed dsn= " + "connectionString" + "= " + "token" + "=", + nestedMapping["message"]); + Assert.Equal(Phase1OperationalDiagnostics.Redacted, redactedMapping["apiKey"]); + Assert.Equal(Phase1OperationalDiagnostics.Redacted, redactedMapping["connectionString"]); + Assert.Equal(Phase1OperationalDiagnostics.Redacted, redactedMapping["password"]); + Assert.Equal(3, redactedMapping["safe_count"]); + + var json = JsonSerializer.Serialize(redacted); + Assert.DoesNotContain("super-secret", json, StringComparison.Ordinal); + Assert.DoesNotContain("svc:secret", json, StringComparison.Ordinal); + Assert.DoesNotContain("abc123", json, StringComparison.Ordinal); + Assert.DoesNotContain("api-secret", json, StringComparison.Ordinal); + Assert.DoesNotContain("raw-secret", json, StringComparison.Ordinal); + } + + [Fact] + public void RedactionRemovesPrefixedAndSuffixedSensitiveKeyNames() + { + var json = Phase1OperationalDiagnostics.SerializeOperationalEvent( + "phase1_test_event", + new Dictionary + { + ["primaryConnectionString"] = "Host=db;Username=svc;Password=connection-secret", + ["providerApiKey"] = "provider-api-secret", + ["externalDatabaseUrl"] = "postgresql://svc:database-secret@db.internal/carbonops", + ["privateAccessKey"] = "private-access-secret", + ["safe_count"] = 4, + }); + + Assert.Contains("\"primaryConnectionString\":\"\"", json, StringComparison.Ordinal); + Assert.Contains("\"providerApiKey\":\"\"", json, StringComparison.Ordinal); + Assert.Contains("\"externalDatabaseUrl\":\"\"", json, StringComparison.Ordinal); + Assert.Contains("\"privateAccessKey\":\"\"", json, StringComparison.Ordinal); + Assert.Contains("\"safe_count\":4", json, StringComparison.Ordinal); + Assert.DoesNotContain("connection-secret", json, StringComparison.Ordinal); + Assert.DoesNotContain("provider-api-secret", json, StringComparison.Ordinal); + Assert.DoesNotContain("database-secret", json, StringComparison.Ordinal); + Assert.DoesNotContain("private-access-secret", json, StringComparison.Ordinal); + } + + [Fact] + public void OperationalEventJsonShapeIsStable() + { + var json = Phase1OperationalDiagnostics.SerializeOperationalEvent( + "phase1_test_event", + new Dictionary + { + ["z_count"] = 2, + ["a_context"] = new Dictionary + { + ["source_family"] = "ghg_protocol", + }, + }); + + Assert.Equal( + "{\"a_context\":{\"source_family\":\"ghg_protocol\"},\"event\":\"phase1_test_event\",\"z_count\":2}", + json); + } + + [Fact] + public void DiagnosticsShapeMatchesSharedParityFixture() + { + using var document = JsonDocument.Parse(File.ReadAllText(ParityFixturePath())); + var fixture = document.RootElement; + + Assert.Equal( + ["correlation_id", "execution_mode", "max_degree_of_parallelism", "run_id", "source_families"], + Strings(fixture.GetProperty("request_keys"))); + Assert.Equal( + ["correlation_id", "documents", "failures", "parser", "persistence", "run_id", "source_family", "source_key", "status"], + Strings(fixture.GetProperty("family_keys"))); + Assert.Equal( + ["checksum_sha256", "document_id", "source_family", "source_key"], + Strings(fixture.GetProperty("document_keys"))); + Assert.Equal( + ["accepted_row_count", "failure_count", "result_status", "run_id", "validation_issue_count"], + Strings(fixture.GetProperty("parser_keys"))); + Assert.Equal( + ["code", "field_name", "message", "severity", "source_family", "source_key", "stage"], + Strings(fixture.GetProperty("failure_keys"))); + Assert.Equal( + [ + "completed_family_count", + "failed_family_count", + "failure_count", + "parsed_factor_row_count", + "parser_run_count", + "persisted_detail_count", + "persisted_master_count", + "persisted_parser_run_count", + "persisted_source_document_count", + "persisted_source_run_count", + "requested_family_count", + "source_artifact_count", + "source_candidate_count", + ], + Strings(fixture.GetProperty("summary_keys"))); + Assert.Equal( + [ + "phase1_ingestion_orchestrator_started", + "phase1_source_family_completed", + "phase1_ingestion_orchestrator_completed", + ], + Strings(fixture.GetProperty("orchestrator_event_names"))); + Assert.Equal( + [ + "phase1_service_host_starting", + "phase1_service_host_started", + "phase1_service_host_scheduled_run_started", + "phase1_service_host_scheduled_run_completed", + "phase1_service_host_scheduled_run_skipped", + ], + Strings(fixture.GetProperty("service_host_event_names"))); + Assert.Equal(Phase1OperationalDiagnostics.Redacted, fixture.GetProperty("redacted").GetString()); + Assert.Contains( + "intentionally coarser", + fixture.GetProperty("intentional_status_difference").GetString(), + StringComparison.Ordinal); + } + + [Fact] + public void FamilyDiagnosticsIncludeSafeDocumentParserPersistenceAndFailureShape() + { + var checksum = new string('A', 64); + var familyResult = new Phase1IngestionFamilyResult( + SourceFamily.GhgProtocol, + "ghg_protocol", + Phase1IngestionFamilyRunStatus.Failed, + acquisitionRun: new SourceAcquisitionRunResult( + SourceFamily.GhgProtocol, + "ghg_protocol", + SourceAcquisitionRunStatus.Completed, + [], + [ + new SourceDownloadArtifact( + SourceFamily.GhgProtocol, + "ghg_protocol", + "candidate-001", + "document-001", + ParserSourceFormat.DiscoveryReference, + "https://example.invalid/source.csv", + "local/document.csv", + "Document", + "text/csv", + checksum: new SourceDocumentChecksum("sha256", checksum, IsDryRunPlaceholder: false)), + ], + "run-001", + "corr-001"), + sourceDocumentPersistResult: new SourceDocumentRepositoryPersistResult( + "fake_documents", + SourceDocumentRepositoryPersistStatus.Declared, + 1), + parserRun: new ParserAdapterRunResult( + SourceFamily.GhgProtocol, + "ghg_protocol", + ParserSelectionRegistry.GetParserKey(SourceFamily.GhgProtocol), + ParserRunStatus.Failed, + ["document-001"], + [], + [ + new ParserValidationIssue( + SourceFamily.GhgProtocol, + "ghg_protocol", + ParserSelectionRegistry.GetParserKey(SourceFamily.GhgProtocol), + ParserValidationIssueSeverity.Error, + "GHG_PROTOCOL_CONTENT_INVALID_HEADER", + "failed password" + "=raw-secret"), + ], + "run-001-ghg_protocol-parser", + "corr-001"), + failures: + [ + new Phase1IngestionFailure( + SourceFamily.GhgProtocol, + "ghg_protocol", + "parser", + "GHG_PROTOCOL_CONTENT_INVALID_HEADER", + "failed password" + "=raw-secret"), + ]); + + var json = JsonSerializer.Serialize( + Phase1OperationalDiagnostics.SummarizeFamilyResultForDiagnostics( + familyResult, + "run-001", + "corr-001")); + + Assert.Contains("\"checksum_sha256\":\"" + checksum.ToLowerInvariant() + "\"", json, StringComparison.Ordinal); + Assert.Contains("\"document_id\":\"document-001\"", json, StringComparison.Ordinal); + Assert.Contains("\"source_key\":\"ghg_protocol\"", json, StringComparison.Ordinal); + Assert.Contains("\"accepted_row_count\":0", json, StringComparison.Ordinal); + Assert.Contains("\"validation_issue_count\":1", json, StringComparison.Ordinal); + Assert.Contains("\"failure_count\":1", json, StringComparison.Ordinal); + Assert.Contains("\"status\":\"failed\"", json, StringComparison.Ordinal); + Assert.Contains("\"source_document_count\":1", json, StringComparison.Ordinal); + Assert.Contains("\"code\":\"GHG_PROTOCOL_CONTENT_INVALID_HEADER\"", json, StringComparison.Ordinal); + Assert.DoesNotContain("raw-secret", json, StringComparison.Ordinal); + } + + private static string[] Strings(JsonElement element) => + element.EnumerateArray().Select(item => item.GetString() ?? string.Empty).ToArray(); + + private static string ParityFixturePath() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixturePath = Path.Combine( + directory.FullName, + "tests", + "fixtures", + "parity", + "phase1_operational_diagnostics_expectations.json"); + if (File.Exists(fixturePath)) + { + return fixturePath; + } + + directory = directory.Parent; + } + + throw new FileNotFoundException("Phase 1 operational diagnostics parity fixture was not found."); + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/Phase1ServiceHostTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/Phase1ServiceHostTests.cs new file mode 100644 index 00000000..30af23c6 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/Phase1ServiceHostTests.cs @@ -0,0 +1,282 @@ +using CarbonOps.Parser.Contracts; +using System.Text.Json; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class Phase1ServiceHostTests +{ + [Fact] + public void ServiceHostValidatesRequiredPostgreSQLRuntimeConfig() + { + var config = new Phase1ServiceHostConfig( + [SourceFamily.GhgProtocol], + new PostgreSQLPersistenceOptions( + "localhost", + 5432, + "carbonops", + "carbonops", + PasswordSet: false)); + + var issues = Phase1ScheduledIngestionServiceHost.ValidateConfig(config); + + Assert.Equal( + ["PHASE1_SERVICE_HOST_POSTGRESQL_PASSWORD_NOT_CONFIRMED"], + issues.Select(issue => issue.Code)); + Assert.Equal("postgresql_options.password_set", issues[0].FieldName); + } + + [Fact] + public void ServiceHostStartupChecksPhase1SchemaBeforeReady() + { + var checker = new FakeSchemaBootstrapChecker(present: false); + var host = new Phase1ScheduledIngestionServiceHost( + Config(), + _ => throw new InvalidOperationException("orchestrator should not run"), + checker.Check); + + var result = host.Start(); + + Assert.Equal(Phase1ServiceHostStatus.Blocked, result.Status); + Assert.Equal(Phase1ServiceHostStatus.Blocked, host.Status); + var call = Assert.Single(checker.Calls); + Assert.Equal(PostgreSQLSchemaBootstrapMode.CheckOnly, call.Mode); + Assert.True(call.FailOnMissing); + Assert.NotNull(result.SchemaBootstrapReport); + Assert.NotEmpty(result.SchemaBootstrapReport.MissingTableNames); + Assert.Equal("PHASE1_SERVICE_HOST_POSTGRESQL_SCHEMA_NOT_READY", result.Issues[0].Code); + Assert.True(result.SchemaBootstrapReport.NoExecution); + Assert.False(result.SchemaBootstrapReport.OpensConnection); + Assert.False(result.SchemaBootstrapReport.RunsSql); + } + + [Fact] + public void ScheduledTriggerRunsOrchestratorForSelectedSourceFamilies() + { + var runner = new FakeOrchestratorRunner(); + var events = new List(); + var host = new Phase1ScheduledIngestionServiceHost( + Config( + sourceFamilies: [SourceFamily.DefraDesnz, SourceFamily.IpccEfdb], + runIdPrefix: "phase1-test"), + runner.Run, + new FakeSchemaBootstrapChecker(present: true).Check, + events.Add); + + var startup = host.Start(); + var result = host.TriggerScheduledRun(); + + Assert.True(startup.IsReady); + Assert.Equal(Phase1ScheduledRunStatus.Started, result.Status); + Assert.Equal("phase1-test-000001", result.RunId); + Assert.Equal([SourceFamily.DefraDesnz, SourceFamily.IpccEfdb], runner.Requests[0].SourceFamilies); + var schemaBootstrapReport = Assert.IsType( + runner.Requests[0].SchemaBootstrapReport); + Assert.Empty(schemaBootstrapReport.MissingTableNames); + var orchestratorResult = Assert.IsType( + result.OrchestratorResult); + Assert.Equal(Phase1IngestionRunStatus.Completed, orchestratorResult.Status); + Assert.Equal(Phase1ServiceHostStatus.Ready, host.Status); + Assert.NotNull(runner.Requests[0].OperationalEventSink); + Assert.Equal( + [ + "phase1_service_host_starting", + "phase1_service_host_started", + "phase1_service_host_scheduled_run_started", + "phase1_service_host_scheduled_run_completed", + ], + events.Select(EventName)); + Assert.Contains("\"run_id\":\"phase1-test-000001\"", events[2], StringComparison.Ordinal); + Assert.Contains("\"status\":\"started\"", events[3], StringComparison.Ordinal); + Assert.Contains("\"status\":\"completed\"", events[3], StringComparison.Ordinal); + } + + [Fact] + public void ServiceHostStartupDiagnosticsRedactPostgreSQLRuntimeConfig() + { + var events = new List(); + var host = new Phase1ScheduledIngestionServiceHost( + Config(), + _ => throw new InvalidOperationException("orchestrator should not run"), + new FakeSchemaBootstrapChecker(present: true).Check, + events.Add); + + host.Start(); + + Assert.Equal("phase1_service_host_starting", EventName(events[0])); + using var document = JsonDocument.Parse(events[0]); + var postgresqlOptions = document.RootElement.GetProperty("postgresql_options"); + Assert.Equal(Phase1OperationalDiagnostics.Redacted, postgresqlOptions.GetProperty("host").GetString()); + Assert.Equal(Phase1OperationalDiagnostics.Redacted, postgresqlOptions.GetProperty("database").GetString()); + Assert.Equal(Phase1OperationalDiagnostics.Redacted, postgresqlOptions.GetProperty("username").GetString()); + Assert.True(postgresqlOptions.GetProperty("password_set").GetBoolean()); + Assert.DoesNotContain("localhost", events[0], StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("carbonops", events[0], StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ServiceHostSkippedRunDiagnosticsEmitThroughSink() + { + var events = new List(); + var host = new Phase1ScheduledIngestionServiceHost( + Config(), + _ => throw new InvalidOperationException("orchestrator should not run"), + new FakeSchemaBootstrapChecker(present: true).Check, + events.Add); + + var result = host.TriggerScheduledRun(); + + Assert.Equal(Phase1ScheduledRunStatus.SkippedNotStarted, result.Status); + var skipped = Assert.Single(events); + Assert.Equal("phase1_service_host_scheduled_run_skipped", EventName(skipped)); + Assert.Contains("\"status\":\"skipped_not_started\"", skipped, StringComparison.Ordinal); + Assert.Contains("PHASE1_SERVICE_HOST_NOT_READY", skipped, StringComparison.Ordinal); + } + + [Fact] + public void ScheduledTriggerSkipsOverlappingRun() + { + Phase1ScheduledIngestionServiceHost? host = null; + Phase1ScheduledRunResult? nestedResult = null; + + Phase1IngestionOrchestratorResult Runner(Phase1IngestionOrchestratorRequest request) + { + nestedResult = host!.TriggerScheduledRun(); + return OrchestratorResult(request); + } + + host = new Phase1ScheduledIngestionServiceHost( + Config(), + Runner, + new FakeSchemaBootstrapChecker(present: true).Check); + + host.Start(); + var result = host.TriggerScheduledRun(); + + Assert.Equal(Phase1ScheduledRunStatus.Started, result.Status); + Assert.NotNull(nestedResult); + Assert.Equal(Phase1ScheduledRunStatus.SkippedAlreadyRunning, nestedResult.Status); + Assert.Equal("PHASE1_SERVICE_HOST_RUN_ALREADY_IN_PROGRESS", nestedResult.Issues[0].Code); + Assert.Equal(Phase1ServiceHostStatus.Ready, host.Status); + } + + [Fact] + public void GracefulShutdownBlocksNewRunsAndStopsAfterActiveRun() + { + Phase1ScheduledIngestionServiceHost? host = null; + Phase1ServiceHostStatus? shutdownStatus = null; + Phase1ScheduledRunResult? nestedResult = null; + + Phase1IngestionOrchestratorResult Runner(Phase1IngestionOrchestratorRequest request) + { + shutdownStatus = host!.RequestShutdown(); + nestedResult = host.TriggerScheduledRun(); + return OrchestratorResult(request); + } + + host = new Phase1ScheduledIngestionServiceHost( + Config(), + Runner, + new FakeSchemaBootstrapChecker(present: true).Check); + + host.Start(); + var result = host.TriggerScheduledRun(); + var afterShutdownResult = host.TriggerScheduledRun(); + + Assert.Equal(Phase1ScheduledRunStatus.Started, result.Status); + Assert.Equal(Phase1ServiceHostStatus.ShutdownRequested, shutdownStatus); + Assert.NotNull(nestedResult); + Assert.Equal(Phase1ScheduledRunStatus.SkippedShuttingDown, nestedResult.Status); + Assert.Equal(Phase1ServiceHostStatus.Stopped, host.Status); + Assert.Equal(Phase1ScheduledRunStatus.SkippedShuttingDown, afterShutdownResult.Status); + } + + [Fact] + public void ScheduledRunnerErrorReleasesOverlapGuardAndReturnsReady() + { + var failedOnce = false; + var host = new Phase1ScheduledIngestionServiceHost( + Config(), + request => + { + if (!failedOnce) + { + failedOnce = true; + throw new InvalidOperationException($"boom: {request.RunId}"); + } + + return OrchestratorResult(request); + }, + new FakeSchemaBootstrapChecker(present: true).Check); + + host.Start(); + var exception = Assert.Throws(() => host.TriggerScheduledRun()); + + Assert.Equal("boom: phase1-scheduled-000001", exception.Message); + Assert.Equal(Phase1ServiceHostStatus.Ready, host.Status); + var followUp = host.TriggerScheduledRun(); + Assert.Equal(Phase1ScheduledRunStatus.Started, followUp.Status); + Assert.Equal("phase1-scheduled-000002", followUp.RunId); + } + + private static Phase1ServiceHostConfig Config( + SourceFamily[]? sourceFamilies = null, + string runIdPrefix = "phase1-scheduled") => + new( + sourceFamilies ?? [SourceFamily.GhgProtocol], + new PostgreSQLPersistenceOptions( + "localhost", + 5432, + "carbonops", + "carbonops", + PasswordSet: true), + runIdPrefix); + + private static Phase1IngestionOrchestratorResult OrchestratorResult( + Phase1IngestionOrchestratorRequest request) => + new( + request, + PostgreSQLRuntimeConfigGateEvaluator.Evaluate(request.RuntimeConfigGate), + [], + status: Phase1IngestionRunStatus.Completed, + selectedSourceFamilies: request.SourceFamilies); + + private static string EventName(string json) + { + using var document = JsonDocument.Parse(json); + return document.RootElement.GetProperty("event").GetString() ?? string.Empty; + } + + private sealed class FakeSchemaBootstrapChecker(bool present) + { + private readonly List<(PostgreSQLSchemaBootstrapMode Mode, bool FailOnMissing)> calls = []; + + public IReadOnlyList<(PostgreSQLSchemaBootstrapMode Mode, bool FailOnMissing)> Calls => calls; + + public PostgreSQLSchemaBootstrapReport Check( + PostgreSQLSchemaBootstrapMode mode, + bool failOnMissing) + { + calls.Add((mode, failOnMissing)); + return PostgreSQLSchemaBootstrapBoundary.BuildReport( + mode, + presentTableNames: present + ? PostgreSQLSchemaBootstrapBoundary.RequiredPhase1TableNames + : [], + failOnMissing: failOnMissing); + } + } + + private sealed class FakeOrchestratorRunner + { + private readonly List requests = []; + + public IReadOnlyList Requests => requests; + + public Phase1IngestionOrchestratorResult Run( + Phase1IngestionOrchestratorRequest request) + { + requests.Add(request); + return OrchestratorResult(request); + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/PostgreSQLRuntimeConfigGateContractTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/PostgreSQLRuntimeConfigGateContractTests.cs new file mode 100644 index 00000000..d4892c12 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/PostgreSQLRuntimeConfigGateContractTests.cs @@ -0,0 +1,170 @@ +using System.Reflection; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class PostgreSQLRuntimeConfigGateContractTests +{ + [Fact] + public void DescriptionDeclaresRuntimePassiveBoundary() + { + var description = PostgreSQLRuntimeConfigGateEvaluator.Describe(); + + Assert.Equal(PostgreSQLRuntimeConfigGateStatus.Disabled, description.DefaultStatus); + Assert.True(description.DisabledByDefault); + Assert.True(description.AcceptsCallerIntent); + Assert.False(description.LoadsEnvironment); + Assert.False(description.LoadsConfigFiles); + Assert.False(description.LoadsCredentials); + Assert.False(description.OpensConnection); + Assert.False(description.RunsSql); + Assert.Contains("Runtime configuration gate metadata only.", description.Notes); + } + + [Fact] + public void DefaultEvaluationIsDisabledAndDoesNotLoadRuntimeConfiguration() + { + var decision = PostgreSQLRuntimeConfigGateEvaluator.Evaluate(); + + Assert.Equal(PostgreSQLRuntimeConfigGateStatus.Disabled, decision.Status); + Assert.False(decision.Requested); + Assert.False(decision.ConfigLoadingEnabled); + Assert.False(decision.RuntimeEnabled); + Assert.False(decision.LoadsEnvironment); + Assert.False(decision.LoadsConfigFiles); + Assert.False(decision.LoadsCredentials); + Assert.Equal(4, decision.RequiredFutureComponentCount); + Assert.Equal(1, decision.IssueCount); + Assert.Equal("POSTGRESQL_RUNTIME_CONFIG_DISABLED_BY_DEFAULT", decision.Issues[0].Code); + } + + [Fact] + public void RequestedEvaluationIsBlockedUntilFutureGateMetadataIsComplete() + { + var decision = PostgreSQLRuntimeConfigGateEvaluator.Evaluate( + new PostgreSQLRuntimeConfigGate( + Requested: true, + SafetyGateApproved: true)); + + Assert.Equal(PostgreSQLRuntimeConfigGateStatus.Blocked, decision.Status); + Assert.True(decision.Requested); + Assert.False(decision.ConfigLoadingEnabled); + Assert.False(decision.RuntimeEnabled); + Assert.Equal( + [ + "postgresql_persistence_options_contract", + "explicit_runtime_configuration_opt_in", + "approved_secret_source", + ], + decision.RequiredFutureComponents); + Assert.Equal("POSTGRESQL_RUNTIME_CONFIG_BLOCKED", decision.Issues[0].Code); + Assert.Equal("requested", decision.Issues[0].FieldName); + } + + [Fact] + public void CompleteCallerMetadataStillDoesNotEnableRuntimeConfigurationLoading() + { + var decision = PostgreSQLRuntimeConfigGateEvaluator.Evaluate( + new PostgreSQLRuntimeConfigGate( + Requested: true, + SafetyGateApproved: true, + OptionsContractAvailable: true, + ExplicitRuntimeOptIn: true, + SecretSourceApproved: true)); + + Assert.Equal(PostgreSQLRuntimeConfigGateStatus.NotEnabled, decision.Status); + Assert.True(decision.Requested); + Assert.False(decision.ConfigLoadingEnabled); + Assert.False(decision.RuntimeEnabled); + Assert.Empty(decision.RequiredFutureComponents); + Assert.Equal("POSTGRESQL_RUNTIME_CONFIG_NOT_ENABLED", decision.Issues[0].Code); + Assert.Contains("does not enable runtime config loading", decision.Reason); + } + + [Fact] + public void DecisionsSnapshotCollectionInputs() + { + var requiredComponents = new List { "component" }; + var notes = new List { "note" }; + var issues = new List + { + new("CODE", "message"), + }; + + var decision = new PostgreSQLRuntimeConfigGateDecision( + PostgreSQLRuntimeConfigGateStatus.Blocked, + requested: true, + reason: "reason", + configLoadingEnabled: false, + runtimeEnabled: false, + loadsEnvironment: false, + loadsConfigFiles: false, + loadsCredentials: false, + requiredFutureComponents: requiredComponents, + safeOperationalNotes: notes, + issues: issues); + requiredComponents.Clear(); + notes.Clear(); + issues.Clear(); + + Assert.Equal(1, decision.RequiredFutureComponentCount); + Assert.Single(decision.SafeOperationalNotes); + Assert.Equal(1, decision.IssueCount); + } + + [Fact] + public void StatusValuesMapToStableWireNames() + { + Assert.Equal( + [ + PostgreSQLRuntimeConfigGateStatus.Disabled, + PostgreSQLRuntimeConfigGateStatus.Blocked, + PostgreSQLRuntimeConfigGateStatus.NotEnabled, + ], + Enum.GetValues()); + Assert.Equal("disabled", PostgreSQLRuntimeConfigGateStatus.Disabled.ToWireName()); + Assert.Equal("blocked", PostgreSQLRuntimeConfigGateStatus.Blocked.ToWireName()); + Assert.Equal("not_enabled", PostgreSQLRuntimeConfigGateStatus.NotEnabled.ToWireName()); + Assert.True(ContractWireNames.TryParsePostgreSQLRuntimeConfigGateStatusWireName("disabled", out var disabled)); + Assert.True(ContractWireNames.TryParsePostgreSQLRuntimeConfigGateStatusWireName("blocked", out var blocked)); + Assert.True( + ContractWireNames.TryParsePostgreSQLRuntimeConfigGateStatusWireName( + "not_enabled", + out var notEnabled)); + Assert.False(ContractWireNames.TryParsePostgreSQLRuntimeConfigGateStatusWireName("enabled", out _)); + + Assert.Equal(PostgreSQLRuntimeConfigGateStatus.Disabled, disabled); + Assert.Equal(PostgreSQLRuntimeConfigGateStatus.Blocked, blocked); + Assert.Equal(PostgreSQLRuntimeConfigGateStatus.NotEnabled, notEnabled); + Assert.Throws(() => ((PostgreSQLRuntimeConfigGateStatus)999).ToWireName()); + } + + [Fact] + public void RuntimeConfigGatePublicApiIsRuntimePassive() + { + var evaluatorMethods = typeof(PostgreSQLRuntimeConfigGateEvaluator) + .GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Select(method => method.Name) + .ToArray(); + var blockedMethodNames = new[] + { + "Load", + "Read", + "Connect", + "Open", + "Execute", + "Run", + "Migrate", + "Create", + "Drop", + "Delete", + }; + + Assert.Equal(["Evaluate", "Describe"], evaluatorMethods); + + foreach (var methodName in blockedMethodNames) + { + Assert.DoesNotContain(evaluatorMethods, method => method.Contains(methodName, StringComparison.OrdinalIgnoreCase)); + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/PostgreSQLRuntimeSchemaAndYearStateTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/PostgreSQLRuntimeSchemaAndYearStateTests.cs new file mode 100644 index 00000000..e28aadcb --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/PostgreSQLRuntimeSchemaAndYearStateTests.cs @@ -0,0 +1,163 @@ +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class PostgreSQLRuntimeSchemaAndYearStateTests +{ + [Fact] + public void BootstrapDdlIsAdditiveOnlyAndIncludesRequiredTables() + { + var statements = PostgreSQLRuntimeSchemaDdl.RenderIdempotentSchemaStatements(); + var rendered = string.Join("\n", statements); + + Assert.All( + PostgreSQLRuntimeSchemaCatalog.RequiredTableNames, + tableName => Assert.Contains($"CREATE TABLE IF NOT EXISTS {tableName}", rendered, StringComparison.Ordinal)); + Assert.Contains("CREATE INDEX IF NOT EXISTS idx_source_family_year_states_family_year", rendered, StringComparison.Ordinal); + Assert.Contains("source_family_year_states", PostgreSQLSchemaBootstrapBoundary.RequiredPhase1TableNames); + Assert.Equal( + PostgreSQLRuntimeSchemaCatalog.RequiredTableNames, + PostgreSQLSchemaBootstrapBoundary.RequiredPhase1TableNames); + } + + [Fact] + public void BootstrapDdlContainsNoDestructiveSqlTokens() + { + var statements = PostgreSQLRuntimeSchemaDdl.RenderIdempotentSchemaStatements(); + + Assert.All(statements, statement => + { + Assert.False(PostgreSQLRuntimeSchemaDdl.ContainsDestructiveSql(statement), statement); + Assert.DoesNotContain("DROP", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("TRUNCATE", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("DELETE", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("ALTER TABLE", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("RENAME", statement, StringComparison.OrdinalIgnoreCase); + }); + } + + [Fact] + public void RuntimeSettingsFailClosedForMissingOrInvalidDatabaseConfig() + { + var values = ValidConfig(); + values["CARBONOPS_PARSER_POSTGRES_PASSWORD"] = string.Empty; + values["CARBONOPS_PARSER_POSTGRES_SCHEMA"] = "Invalid-Schema"; + + var created = PostgreSQLRuntimeConnectionBoundary.TryCreateFromProductionConfig( + values, + out var settings, + out var issues); + + Assert.False(created); + Assert.NotNull(settings); + Assert.Contains(issues, issue => issue.Code == "POSTGRESQL_RUNTIME_MISSING_PASSWORD"); + Assert.Contains(issues, issue => issue.Code == "POSTGRESQL_RUNTIME_INVALID_IDENTIFIER"); + } + + [Fact] + public void RuntimeDiagnosticsRedactSecretValues() + { + Assert.True(PostgreSQLRuntimeConnectionBoundary.TryCreateFromProductionConfig( + ValidConfig(), + out var settings, + out var issues)); + Assert.Empty(issues); + + var diagnostics = PostgreSQLRuntimeConnectionBoundary.BuildSafeDiagnostics(settings!); + var rendered = string.Join("\n", diagnostics.Select(item => $"{item.Key}={item.Value}")); + + Assert.Contains("postgresql_password=[redacted]", rendered, StringComparison.Ordinal); + Assert.Contains("postgresql_connection_string=[redacted]", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("runtime-secret-not-returned", rendered, StringComparison.Ordinal); + } + + [Fact] + public async Task YearStateReturnsInitialYearWhenNoDataExists() + { + var session = new InMemoryYearStateSession(); + var repository = new PostgreSQLSourceFamilyYearStateRepository(session); + + var state = await repository.GetYearStateAsync(SourceFamily.GhgProtocol); + + Assert.Equal(SourceFamily.GhgProtocol, state.SourceFamily); + Assert.Null(state.LatestYear); + Assert.Equal(2024, state.InitialYear); + Assert.Equal(2024, state.NextYear); + } + + [Fact] + public async Task YearStateReturnsNextYearWhenSuccessfulYearExists() + { + var session = new InMemoryYearStateSession(); + var repository = new PostgreSQLSourceFamilyYearStateRepository(session); + + await repository.RecordSuccessfulYearAsync(SourceFamily.DefraDesnz, 2025); + var state = await repository.GetYearStateAsync(SourceFamily.DefraDesnz); + + Assert.Equal(2025, state.LatestYear); + Assert.Equal(2026, state.NextYear); + } + + [Fact] + public async Task RecordSuccessfulYearIsIdempotent() + { + var session = new InMemoryYearStateSession(); + var repository = new PostgreSQLSourceFamilyYearStateRepository(session); + + await repository.RecordSuccessfulYearAsync(SourceFamily.IpccEfdb, 2024); + await repository.RecordSuccessfulYearAsync(SourceFamily.IpccEfdb, 2024); + + Assert.Equal(1, session.RecordCount(SourceFamily.IpccEfdb)); + Assert.Equal(2025, await repository.NextTargetYearAsync(SourceFamily.IpccEfdb)); + } + + [Fact] + public void SourceFamilyRuntimeValuesAlignWithPythonPostgreSqlContract() + { + Assert.Equal("ghg_protocol", SourceFamily.GhgProtocol.ToPostgreSQLRuntimeValue()); + Assert.Equal("defra_desnz", SourceFamily.DefraDesnz.ToPostgreSQLRuntimeValue()); + Assert.Equal("ipcc_efdb", SourceFamily.IpccEfdb.ToPostgreSQLRuntimeValue()); + } + + private static Dictionary ValidConfig() => new() + { + ["CARBONOPS_PARSER_ENV"] = "production", + ["CARBONOPS_PARSER_DATABASE_PROVIDER"] = "postgres", + ["CARBONOPS_PARSER_POSTGRES_HOST"] = "db.internal.example", + ["CARBONOPS_PARSER_POSTGRES_PORT"] = "5432", + ["CARBONOPS_PARSER_POSTGRES_DATABASE"] = "carbonops_parser", + ["CARBONOPS_PARSER_POSTGRES_USERNAME"] = "carbonops_runtime", + ["CARBONOPS_PARSER_POSTGRES_PASSWORD"] = "runtime-secret-not-returned", + ["CARBONOPS_PARSER_POSTGRES_SCHEMA"] = "carbonops", + ["CARBONOPS_PARSER_RAW_ARCHIVE_PATH"] = "/var/lib/carbonops/raw", + ["CARBONOPS_PARSER_LOG_LEVEL"] = "info", + }; + + private sealed class InMemoryYearStateSession : IPostgreSQLSourceFamilyYearStateSession + { + private readonly HashSet<(SourceFamily SourceFamily, int Year)> _records = []; + + public Task LatestSuccessfulYearAsync( + SourceFamily sourceFamily, + CancellationToken cancellationToken = default) + { + var years = _records + .Where(record => record.SourceFamily == sourceFamily) + .Select(record => record.Year) + .ToArray(); + return Task.FromResult(years.Length == 0 ? null : (int?)years.Max()); + } + + public Task RecordSuccessfulYearAsync( + SourceFamily sourceFamily, + int ingestedYear, + CancellationToken cancellationToken = default) + { + _records.Add((sourceFamily, ingestedYear)); + return Task.CompletedTask; + } + + public int RecordCount(SourceFamily sourceFamily) => + _records.Count(record => record.SourceFamily == sourceFamily); + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/PostgreSQLSourceSpecificFactorPersistenceTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/PostgreSQLSourceSpecificFactorPersistenceTests.cs new file mode 100644 index 00000000..45ef5a4a --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/PostgreSQLSourceSpecificFactorPersistenceTests.cs @@ -0,0 +1,306 @@ +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class PostgreSQLSourceSpecificFactorPersistenceTests +{ + [Theory] + [InlineData(SourceFamily.GhgProtocol, "ghg_emission_factor_masters", "ghg_emission_factor_details")] + [InlineData(SourceFamily.DefraDesnz, "defra_emission_factor_masters", "defra_emission_factor_details")] + [InlineData(SourceFamily.IpccEfdb, "ipcc_emission_factor_masters", "ipcc_emission_factor_details")] + public void MapperBuildsSourceSpecificMasterAndDetailRecordsForEachFamily( + SourceFamily sourceFamily, + string masterTable, + string detailTable) + { + var mapped = PostgreSQLSourceSpecificFactorPersistenceMapper.Map( + new ParserNormalizedOutputBatch([CreateRow(sourceFamily)])); + + Assert.Empty(mapped.Issues); + var batch = Assert.Single(mapped.Batches); + Assert.Equal(sourceFamily, batch.SourceFamily); + Assert.Equal(2024, batch.SourceYear); + var master = Assert.Single(batch.MasterRecords); + var detail = Assert.Single(batch.DetailRecords); + Assert.Equal(sourceFamily, master.SourceFamily); + Assert.Equal(sourceFamily, detail.SourceFamily); + Assert.Equal(2024, master.SourceYear); + Assert.Equal("fixture-v1", master.SourceVersion); + Assert.Equal("release-a", master.SourceRelease); + Assert.Equal("run-001", master.RunId); + Assert.Equal("artifact://fixture/factors.csv", master.ArtifactReference); + Assert.Equal("a".PadLeft(64, 'a'), master.ArtifactChecksumSha256); + Assert.Equal("2024:fixture-v1:FACTOR-001", master.MasterExternalKey); + Assert.Equal(master.SourceFamilyMasterId, detail.SourceFamilyMasterId); + Assert.Equal("FACTOR-001:kgco2e:CO2e", detail.DetailExternalKey); + Assert.Equal(7, detail.SourceRowNumber); + Assert.Equal("FACTOR-001", detail.FactorId); + Assert.Equal("Fixture factor", detail.FactorName); + Assert.Equal(1.25m, detail.FactorValue); + Assert.Equal("kgco2e", detail.FactorUnit); + Assert.Equal("active", detail.Status); + + Assert.Contains(masterTable, NpgsqlSourceSpecificFactorPersistenceSession.RenderMasterInsertSql(sourceFamily), StringComparison.Ordinal); + Assert.Contains(detailTable, NpgsqlSourceSpecificFactorPersistenceSession.RenderDetailInsertSql(sourceFamily), StringComparison.Ordinal); + } + + [Theory] + [InlineData(SourceFamily.GhgProtocol)] + [InlineData(SourceFamily.DefraDesnz)] + [InlineData(SourceFamily.IpccEfdb)] + public async Task RepositoryInsertsOneMasterAndMatchingDetailForEachFamily(SourceFamily sourceFamily) + { + var session = new InMemorySourceSpecificSession(); + var repository = new PostgreSQLSourceSpecificFactorPersistenceRepository(session); + + var result = await repository.PersistAsync(new ParserNormalizedOutputBatch([CreateRow(sourceFamily)])); + + Assert.Equal(PostgreSQLSourceSpecificFactorPersistenceStatus.Inserted, result.Status); + Assert.Equal(1, result.MasterInserted); + Assert.Equal(0, result.MasterSkippedDuplicate); + Assert.Equal(1, result.DetailInserted); + Assert.Equal(0, result.DetailSkippedDuplicate); + Assert.Equal(0, result.ValidationFailed); + Assert.Contains((sourceFamily, 2024), session.RecordedYears); + } + + [Fact] + public async Task RepositoryRerunSkipsDuplicatesExplicitly() + { + var session = new InMemorySourceSpecificSession(); + var repository = new PostgreSQLSourceSpecificFactorPersistenceRepository(session); + var batch = new ParserNormalizedOutputBatch([CreateRow(SourceFamily.DefraDesnz)]); + + var first = await repository.PersistAsync(batch); + var second = await repository.PersistAsync(batch); + + Assert.Equal(1, first.MasterInserted); + Assert.Equal(1, first.DetailInserted); + Assert.Equal(0, first.MasterSkippedDuplicate); + Assert.Equal(0, first.DetailSkippedDuplicate); + Assert.Equal(0, second.MasterInserted); + Assert.Equal(0, second.DetailInserted); + Assert.Equal(1, second.MasterSkippedDuplicate); + Assert.Equal(1, second.DetailSkippedDuplicate); + Assert.Equal(1, session.RecordedYears.Count(record => record == (SourceFamily.DefraDesnz, 2024))); + } + + [Fact] + public async Task RepositoryDoesNotAdvanceYearStateWhenValidationFails() + { + var session = new InMemorySourceSpecificSession(); + var repository = new PostgreSQLSourceSpecificFactorPersistenceRepository(session); + var malformed = CreateRow(SourceFamily.GhgProtocol, factorValue: "not-a-number"); + + var result = await repository.PersistAsync(new ParserNormalizedOutputBatch([malformed])); + + Assert.Equal(PostgreSQLSourceSpecificFactorPersistenceStatus.FailedValidation, result.Status); + Assert.Equal(0, result.MasterInserted); + Assert.Equal(0, result.DetailInserted); + Assert.True(result.ValidationFailed > 0); + Assert.Empty(session.RecordedYears); + } + + [Fact] + public async Task RepositoryDoesNotAdvanceYearStateWhenInsertFails() + { + var session = new InMemorySourceSpecificSession { FailBeforeYearState = true }; + var repository = new PostgreSQLSourceSpecificFactorPersistenceRepository(session); + + var result = await repository.PersistAsync(new ParserNormalizedOutputBatch([CreateRow(SourceFamily.IpccEfdb)])); + + Assert.Equal(PostgreSQLSourceSpecificFactorPersistenceStatus.FailedDatabase, result.Status); + Assert.Equal(0, result.MasterInserted); + Assert.Equal(0, result.DetailInserted); + Assert.Empty(session.RecordedYears); + } + + [Fact] + public async Task RepositoryDiagnosticsRedactSecrets() + { + var session = new InMemorySourceSpecificSession + { + FailureMessage = "postgresql://user:secret@example/db password=secret-token", + }; + var repository = new PostgreSQLSourceSpecificFactorPersistenceRepository(session); + + var result = await repository.PersistAsync(new ParserNormalizedOutputBatch([CreateRow(SourceFamily.GhgProtocol)])); + + var issue = Assert.Single(result.Issues); + Assert.Equal("POSTGRESQL_SOURCE_SPECIFIC_DATABASE_ERROR", issue.Code); + Assert.Contains("postgresql://***@example/db", issue.Message, StringComparison.Ordinal); + Assert.Contains("password=***", issue.Message, StringComparison.Ordinal); + Assert.DoesNotContain("secret-token", issue.Message, StringComparison.Ordinal); + } + + [Fact] + public void SourceSpecificInsertSqlIsAdditiveAndUsesConflictSkips() + { + var statements = SourceFamilyRegistry.SupportedFamilies + .SelectMany(family => new[] + { + NpgsqlSourceSpecificFactorPersistenceSession.RenderMasterInsertSql(family), + NpgsqlSourceSpecificFactorPersistenceSession.RenderDetailInsertSql(family), + }) + .ToArray(); + + Assert.All(statements, statement => + { + Assert.Contains("INSERT INTO", statement, StringComparison.Ordinal); + Assert.Contains("ON CONFLICT", statement, StringComparison.Ordinal); + Assert.Contains("DO NOTHING", statement, StringComparison.Ordinal); + Assert.DoesNotContain("DROP", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("TRUNCATE", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("DELETE", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("ALTER TABLE", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("RENAME", statement, StringComparison.OrdinalIgnoreCase); + }); + } + + [Fact] + public void YearStateInsertSqlIsExplicitAndIdempotent() + { + var statement = NpgsqlSourceSpecificFactorPersistenceSession.RenderYearStateInsertSql(); + + Assert.Contains("INSERT INTO source_family_year_states", statement, StringComparison.Ordinal); + Assert.Contains("source_family_year_state_id", statement, StringComparison.Ordinal); + Assert.Contains("source_family", statement, StringComparison.Ordinal); + Assert.Contains("ingested_year", statement, StringComparison.Ordinal); + Assert.Contains("ON CONFLICT (source_family, ingested_year)", statement, StringComparison.Ordinal); + Assert.Contains("DO UPDATE SET updated_at = EXCLUDED.updated_at", statement, StringComparison.Ordinal); + Assert.DoesNotContain("DROP", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("TRUNCATE", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("DELETE", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("ALTER TABLE", statement, StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData(SourceFamily.GhgProtocol)] + [InlineData(SourceFamily.DefraDesnz)] + [InlineData(SourceFamily.IpccEfdb)] + public void MasterAndDetailInsertSqlDoNotWriteYearState(SourceFamily sourceFamily) + { + var master = NpgsqlSourceSpecificFactorPersistenceSession.RenderMasterInsertSql(sourceFamily); + var detail = NpgsqlSourceSpecificFactorPersistenceSession.RenderDetailInsertSql(sourceFamily); + + Assert.DoesNotContain("source_family_year_states", master, StringComparison.Ordinal); + Assert.DoesNotContain("source_family_year_states", detail, StringComparison.Ordinal); + Assert.DoesNotContain("DO UPDATE SET", master, StringComparison.Ordinal); + Assert.DoesNotContain("DO UPDATE SET", detail, StringComparison.Ordinal); + } + + [Fact] + public void NpgsqlPersistenceSessionDeclaresTransactionRollbackBoundary() + { + var boundary = NpgsqlSourceSpecificFactorPersistenceSession.DescribeTransactionBoundary(); + + Assert.True(boundary.OpensTransaction); + Assert.True(boundary.CommitsTransaction); + Assert.True(boundary.RollsBackOnFailure); + } + + [Theory] + [InlineData(SourceFamily.GhgProtocol)] + [InlineData(SourceFamily.DefraDesnz)] + [InlineData(SourceFamily.IpccEfdb)] + public void YearStateRecordingRemainsAfterMasterAndDetailInsertSqlInFlow(SourceFamily sourceFamily) + { + var flow = NpgsqlSourceSpecificFactorPersistenceSession.RenderSourceFamilyYearPersistenceSqlFlow(sourceFamily); + + Assert.Equal(["master_insert", "detail_insert", "year_state_insert"], flow.Select(step => step.Name)); + Assert.Contains("source_family_year_states", flow[2].CommandText, StringComparison.Ordinal); + Assert.DoesNotContain("source_family_year_states", flow[0].CommandText, StringComparison.Ordinal); + Assert.DoesNotContain("source_family_year_states", flow[1].CommandText, StringComparison.Ordinal); + } + + private static ParserNormalizedOutputRow CreateRow( + SourceFamily sourceFamily, + string factorValue = "1.25") + { + var sourceKey = sourceFamily.ToWireName(); + return new ParserNormalizedOutputRow( + sourceFamily, + sourceKey, + ParserSelectionRegistry.GetParserKey(sourceFamily), + "artifact://fixture/factors.csv", + $"{sourceKey}_row_7", + sourceRowNumber: 7, + [ + new ParserNormalizedField("source_family", sourceKey), + new ParserNormalizedField("source_year", "2024"), + new ParserNormalizedField("source_version", "fixture-v1"), + new ParserNormalizedField("source_release", "release-a"), + new ParserNormalizedField("run_id", "run-001"), + new ParserNormalizedField("factor_id", "FACTOR-001"), + new ParserNormalizedField("factor_name", "Fixture factor"), + new ParserNormalizedField("factor_value", factorValue), + new ParserNormalizedField("unit", "kgco2e"), + new ParserNormalizedField("gas", "CO2e"), + new ParserNormalizedField("provenance_artifact_reference", "artifact://fixture/factors.csv"), + new ParserNormalizedField("provenance_checksum_value", "a".PadLeft(64, 'a')), + new ParserNormalizedField("provenance_row_number", "7"), + new ParserNormalizedField("master_external_key", "2024:fixture-v1:FACTOR-001"), + new ParserNormalizedField("detail_external_key", "FACTOR-001:kgco2e:CO2e"), + ], + reportingYear: 2024); + } + + private sealed class InMemorySourceSpecificSession : IPostgreSQLSourceSpecificFactorPersistenceSession + { + private readonly HashSet<(SourceFamily SourceFamily, int SourceYear, string SourceVersion, string MasterKey)> _masters = []; + private readonly HashSet<(SourceFamily SourceFamily, Guid MasterId, string DetailKey)> _details = []; + + public string ProviderName => "fake_postgresql"; + + public bool FailBeforeYearState { get; init; } + + public string? FailureMessage { get; init; } + + public List<(SourceFamily SourceFamily, int SourceYear)> RecordedYears { get; } = []; + + public Task PersistSourceFamilyYearAsync( + PostgreSQLSourceSpecificFactorPersistenceBatch batch, + CancellationToken cancellationToken = default) + { + if (FailureMessage is not null) + { + throw new InvalidOperationException(FailureMessage); + } + + var masterInserted = 0; + var detailInserted = 0; + foreach (var master in batch.MasterRecords) + { + if (_masters.Add((master.SourceFamily, master.SourceYear, master.SourceVersion, master.MasterExternalKey))) + { + masterInserted++; + } + } + + foreach (var detail in batch.DetailRecords) + { + if (_details.Add((detail.SourceFamily, detail.SourceFamilyMasterId, detail.DetailExternalKey))) + { + detailInserted++; + } + } + + if (FailBeforeYearState) + { + throw new InvalidOperationException("detail insert failed before year-state update"); + } + + if (!RecordedYears.Contains((batch.SourceFamily, batch.SourceYear))) + { + RecordedYears.Add((batch.SourceFamily, batch.SourceYear)); + } + + return Task.FromResult(new PostgreSQLSourceSpecificFactorPersistenceCounts( + masterInserted, + batch.MasterRecords.Count - masterInserted, + detailInserted, + batch.DetailRecords.Count - detailInserted, + 0)); + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ProductionConfigBoundaryTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ProductionConfigBoundaryTests.cs new file mode 100644 index 00000000..23e76c98 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/ProductionConfigBoundaryTests.cs @@ -0,0 +1,183 @@ +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class ProductionConfigBoundaryTests +{ + [Fact] + public void BoundaryDocumentsAlignedRequiredEnvironmentVariables() + { + var description = ProductionConfigBoundary.Describe(); + + Assert.Equal( + [ + "CARBONOPS_PARSER_ENV", + "CARBONOPS_PARSER_DATABASE_PROVIDER", + "CARBONOPS_PARSER_POSTGRES_HOST", + "CARBONOPS_PARSER_POSTGRES_PORT", + "CARBONOPS_PARSER_POSTGRES_DATABASE", + "CARBONOPS_PARSER_POSTGRES_USERNAME", + "CARBONOPS_PARSER_POSTGRES_PASSWORD", + "CARBONOPS_PARSER_POSTGRES_SCHEMA", + "CARBONOPS_PARSER_RAW_ARCHIVE_PATH", + "CARBONOPS_PARSER_LOG_LEVEL", + ], + description.RequiredEnvironmentVariables); + Assert.Equal(["CARBONOPS_PARSER_POSTGRES_PASSWORD"], description.SecretEnvironmentVariables); + Assert.Equal("postgres", description.Provider); + Assert.True(description.LoadsEnvironment); + Assert.True(description.LoadsConfigFiles); + Assert.True(description.LoadsCredentials); + Assert.False(description.LogsSecretValues); + } + + [Fact] + public void ValidProductionConfigMappingPassesWithoutReturningSecret() + { + var result = ProductionConfigBoundary.Validate(ValidConfig()); + + Assert.True(result.IsValid); + Assert.Empty(result.Issues); + } + + [Fact] + public void MissingRequiredProductionKeysFailClosedWithSafeMessages() + { + var values = ValidConfig(); + values["CARBONOPS_PARSER_POSTGRES_PASSWORD"] = " "; + values["CARBONOPS_PARSER_RAW_ARCHIVE_PATH"] = null; + + var result = ProductionConfigBoundary.Validate(values); + + Assert.False(result.IsValid); + Assert.Equal( + ["CARBONOPS_PARSER_POSTGRES_PASSWORD", "CARBONOPS_PARSER_RAW_ARCHIVE_PATH"], + result.Issues.Select(issue => issue.FieldName)); + Assert.DoesNotContain("runtime-secret-not-returned", result.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("Password" + "=", result.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void InvalidValuesFailWithActionableKeyOnlyMessages() + { + var values = ValidConfig(); + values["CARBONOPS_PARSER_DATABASE_PROVIDER"] = "mysql"; + values["CARBONOPS_PARSER_POSTGRES_PORT"] = "not-a-port"; + values["CARBONOPS_PARSER_LOG_LEVEL"] = "verbose"; + values["CARBONOPS_PARSER_POSTGRES_CONNECTION_STRING"] = "Host=db;Username=svc;" + "Password" + "=raw-secret"; + + var result = ProductionConfigBoundary.Validate(values); + + Assert.Equal( + [ + "PRODUCTION_CONFIG_UNSUPPORTED_DATABASE_PROVIDER", + "PRODUCTION_CONFIG_INVALID_POSTGRES_PORT", + "PRODUCTION_CONFIG_INVALID_LOG_LEVEL", + "PRODUCTION_CONFIG_RAW_CONNECTION_STRING_NOT_ALLOWED", + ], + result.Issues.Select(issue => issue.Code)); + var rendered = string.Join(" ", result.Issues.Select(issue => issue.ToString())); + Assert.DoesNotContain("mysql", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("not-a-port", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("verbose", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("raw-secret", rendered, StringComparison.Ordinal); + } + + [Fact] + public void LoaderAcceptsValidConfigFileOnlyShape() + { + var configPath = WriteConfigFile(ValidConfig()); + + var load = ProductionConfigLoader.Load(configPath, new Dictionary()); + var result = ProductionConfigBoundary.Validate(load.Values); + + Assert.True(load.ConfigFileLoaded); + Assert.True(load.EnvironmentLoaded); + Assert.Empty(load.Issues); + Assert.True(result.IsValid); + Assert.Equal("db.internal.example", load.Values["CARBONOPS_PARSER_POSTGRES_HOST"]); + } + + [Fact] + public void LoaderAcceptsValidEnvironmentOnlyShape() + { + var load = ProductionConfigLoader.Load(null, ValidConfig()); + var result = ProductionConfigBoundary.Validate(load.Values); + + Assert.False(load.ConfigFileLoaded); + Assert.True(load.EnvironmentLoaded); + Assert.Empty(load.Issues); + Assert.True(result.IsValid); + } + + [Fact] + public void LoaderEnvironmentValuesOverrideConfigFileValues() + { + var fileValues = ValidConfig(); + fileValues["CARBONOPS_PARSER_POSTGRES_HOST"] = "file-db.internal.example"; + fileValues["CARBONOPS_PARSER_POSTGRES_PORT"] = "1111"; + var configPath = WriteConfigFile(fileValues); + var environment = new Dictionary(StringComparer.Ordinal) + { + ["CARBONOPS_PARSER_POSTGRES_HOST"] = "env-db.internal.example", + ["CARBONOPS_PARSER_POSTGRES_PORT"] = "5433", + }; + + var load = ProductionConfigLoader.Load(configPath, environment); + + Assert.Equal("env-db.internal.example", load.Values["CARBONOPS_PARSER_POSTGRES_HOST"]); + Assert.Equal("5433", load.Values["CARBONOPS_PARSER_POSTGRES_PORT"]); + } + + [Fact] + public void LoaderMissingRequiredValuesFailClosed() + { + var values = ValidConfig(); + values.Remove("CARBONOPS_PARSER_POSTGRES_HOST"); + var configPath = WriteConfigFile(values); + + var load = ProductionConfigLoader.Load(configPath, new Dictionary()); + var result = ProductionConfigBoundary.Validate(load.Values); + + Assert.False(result.IsValid); + Assert.Contains(result.Issues, issue => + issue.Code == "PRODUCTION_CONFIG_MISSING_REQUIRED_ENV_VAR" && + issue.FieldName == "CARBONOPS_PARSER_POSTGRES_HOST"); + } + + [Fact] + public void LoaderInvalidPortFailsClosed() + { + var values = ValidConfig(); + values["CARBONOPS_PARSER_POSTGRES_PORT"] = "70000"; + var configPath = WriteConfigFile(values); + + var load = ProductionConfigLoader.Load(configPath, new Dictionary()); + var result = ProductionConfigBoundary.Validate(load.Values); + + Assert.False(result.IsValid); + Assert.Contains(result.Issues, issue => issue.Code == "PRODUCTION_CONFIG_INVALID_POSTGRES_PORT"); + } + + private static Dictionary ValidConfig() => new() + { + ["CARBONOPS_PARSER_ENV"] = "production", + ["CARBONOPS_PARSER_DATABASE_PROVIDER"] = "postgres", + ["CARBONOPS_PARSER_POSTGRES_HOST"] = "db.internal.example", + ["CARBONOPS_PARSER_POSTGRES_PORT"] = "5432", + ["CARBONOPS_PARSER_POSTGRES_DATABASE"] = "carbonops_parser", + ["CARBONOPS_PARSER_POSTGRES_USERNAME"] = "carbonops_runtime", + ["CARBONOPS_PARSER_POSTGRES_PASSWORD"] = "runtime-secret-not-returned", + ["CARBONOPS_PARSER_POSTGRES_SCHEMA"] = "carbonops", + ["CARBONOPS_PARSER_RAW_ARCHIVE_PATH"] = "/var/lib/carbonops/raw", + ["CARBONOPS_PARSER_LOG_LEVEL"] = "info", + }; + + private static string WriteConfigFile(IReadOnlyDictionary values) + { + var path = Path.Combine(Path.GetTempPath(), $"carbonops-production-config-{Guid.NewGuid():N}.json"); + var json = System.Text.Json.JsonSerializer.Serialize(values); + File.WriteAllText(path, json); + return path; + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceAcquisitionContractPublicApiTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceAcquisitionContractPublicApiTests.cs index 62419bef..c1f293b4 100644 --- a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceAcquisitionContractPublicApiTests.cs +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceAcquisitionContractPublicApiTests.cs @@ -13,6 +13,30 @@ public void RuntimePassiveSourceAcquisitionContractTypesArePublic() typeof(SourceDiscoveryCandidate), typeof(SourceDiscoveryCandidateBatch), typeof(SourceDiscoveryCandidateRegistry), + typeof(GhgSourceDiscoveryMode), + typeof(GhgSourceDiscoveryStatus), + typeof(GhgSourceDiscoveryRequest), + typeof(GhgSourceDocumentCandidate), + typeof(GhgSourceDiscoveryIssue), + typeof(GhgSourceDiscoveryValidationResult), + typeof(GhgSourceDiscoveryResult), + typeof(GhgSourceDiscoveryBoundary), + typeof(DefraSourceDiscoveryMode), + typeof(DefraSourceDiscoveryStatus), + typeof(DefraSourceDiscoveryRequest), + typeof(DefraSourceDocumentCandidate), + typeof(DefraSourceDiscoveryIssue), + typeof(DefraSourceDiscoveryValidationResult), + typeof(DefraSourceDiscoveryResult), + typeof(DefraSourceDiscoveryBoundary), + typeof(IpccSourceDiscoveryMode), + typeof(IpccSourceDiscoveryStatus), + typeof(IpccSourceDiscoveryRequest), + typeof(IpccSourceDocumentCandidate), + typeof(IpccSourceDiscoveryIssue), + typeof(IpccSourceDiscoveryValidationResult), + typeof(IpccSourceDiscoveryResult), + typeof(IpccSourceDiscoveryBoundary), typeof(SourceDownloadArtifact), typeof(SourceDownloadArtifactBatch), typeof(SourceDownloadArtifactRegistry), @@ -20,6 +44,12 @@ public void RuntimePassiveSourceAcquisitionContractTypesArePublic() typeof(SourceAcquisitionRunRequest), typeof(SourceAcquisitionRunResult), typeof(SourceAcquisitionRunRegistry), + typeof(ISourceAcquisitionRunRepository), + typeof(SourceAcquisitionRunRepositoryPersistStatus), + typeof(SourceAcquisitionRunRepositoryIssue), + typeof(SourceAcquisitionRunRepositoryPersistResult), + typeof(SourceAcquisitionRunRepositoryValidationResult), + typeof(SourceAcquisitionRunRepositoryRegistry), }; Assert.Equal( @@ -27,6 +57,30 @@ public void RuntimePassiveSourceAcquisitionContractTypesArePublic() "SourceDiscoveryCandidate", "SourceDiscoveryCandidateBatch", "SourceDiscoveryCandidateRegistry", + "GhgSourceDiscoveryMode", + "GhgSourceDiscoveryStatus", + "GhgSourceDiscoveryRequest", + "GhgSourceDocumentCandidate", + "GhgSourceDiscoveryIssue", + "GhgSourceDiscoveryValidationResult", + "GhgSourceDiscoveryResult", + "GhgSourceDiscoveryBoundary", + "DefraSourceDiscoveryMode", + "DefraSourceDiscoveryStatus", + "DefraSourceDiscoveryRequest", + "DefraSourceDocumentCandidate", + "DefraSourceDiscoveryIssue", + "DefraSourceDiscoveryValidationResult", + "DefraSourceDiscoveryResult", + "DefraSourceDiscoveryBoundary", + "IpccSourceDiscoveryMode", + "IpccSourceDiscoveryStatus", + "IpccSourceDiscoveryRequest", + "IpccSourceDocumentCandidate", + "IpccSourceDiscoveryIssue", + "IpccSourceDiscoveryValidationResult", + "IpccSourceDiscoveryResult", + "IpccSourceDiscoveryBoundary", "SourceDownloadArtifact", "SourceDownloadArtifactBatch", "SourceDownloadArtifactRegistry", @@ -34,6 +88,12 @@ public void RuntimePassiveSourceAcquisitionContractTypesArePublic() "SourceAcquisitionRunRequest", "SourceAcquisitionRunResult", "SourceAcquisitionRunRegistry", + "ISourceAcquisitionRunRepository", + "SourceAcquisitionRunRepositoryPersistStatus", + "SourceAcquisitionRunRepositoryIssue", + "SourceAcquisitionRunRepositoryPersistResult", + "SourceAcquisitionRunRepositoryValidationResult", + "SourceAcquisitionRunRepositoryRegistry", ], publicContractTypes.Select(type => type.Name)); Assert.All(publicContractTypes, type => Assert.True(type.IsPublic, $"{type.Name} must be public.")); @@ -106,11 +166,16 @@ public void SourceAcquisitionRegistryPublicApiIsDeterministic() var artifacts = SourceDownloadArtifactRegistry.CreateDefaultArtifactBatch(); var requests = SourceAcquisitionRunRegistry.CreateDefaultRunRequests(); var results = SourceAcquisitionRunRegistry.CreateDefaultRunResults(); + var persistResult = SourceAcquisitionRunRepositoryRegistry.CreatePersistResult( + "in_memory", + results); Assert.Equal(3, candidates.CandidateCount); Assert.Equal(3, artifacts.ArtifactCount); Assert.Equal(3, requests.Count); Assert.Equal(3, results.Count); + Assert.Equal(3, persistResult.PersistedCount); + Assert.Equal(SourceAcquisitionRunRepositoryPersistStatus.Declared, persistResult.Status); Assert.Equal( ParserAdapterDescriptorRegistry.Descriptors.Select(descriptor => descriptor.SourceFamily), requests.Select(request => request.SourceFamily)); @@ -143,6 +208,7 @@ public void SourceAcquisitionContractPublicApiConstructionRemainsRuntimePassive( typeof(SourceDiscoveryCandidateRegistry), typeof(SourceDownloadArtifactRegistry), typeof(SourceAcquisitionRunRegistry), + typeof(SourceAcquisitionRunRepositoryRegistry), }; var publicMethodNames = sourceAcquisitionContractTypes .SelectMany(type => type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) @@ -156,6 +222,8 @@ public void SourceAcquisitionContractPublicApiConstructionRemainsRuntimePassive( "CreateDefaultArtifactBatch", "CreateDefaultRunRequests", "CreateDefaultRunResults", + "CreatePersistResult", + "ValidateInputs", ], publicMethodNames); Assert.DoesNotContain("Discover", publicMethodNames); @@ -209,4 +277,44 @@ public void SourceAcquisitionContractPublicApiDoesNotExposeDbHttpFileIoOrRuntime Assert.DoesNotContain("Parse", publicMembers); Assert.DoesNotContain("Execute", publicMembers); } + + [Fact] + public void SourceAcquisitionRunRepositoryContractPublicApiDoesNotExposeRuntimeDbHttpFileIoOrParserExecutionSurface() + { + var repositoryContractTypes = new[] + { + typeof(ISourceAcquisitionRunRepository), + typeof(SourceAcquisitionRunRepositoryIssue), + typeof(SourceAcquisitionRunRepositoryPersistResult), + typeof(SourceAcquisitionRunRepositoryValidationResult), + typeof(SourceAcquisitionRunRepositoryRegistry), + }; + var publicMembers = repositoryContractTypes + .SelectMany(type => type.GetMembers(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + .Select(member => member.Name) + .ToArray(); + var blockedTerms = new[] + { + "Db", + "Sql", + "Postgres", + "Http", + "Open", + "ReadFile", + "Write", + "StatFile", + "Exists", + "Fetch", + "Calculate", + "Factor", + }; + + foreach (var term in blockedTerms) + { + Assert.DoesNotContain(publicMembers, member => member.Contains(term, StringComparison.OrdinalIgnoreCase)); + } + + Assert.DoesNotContain("Parse", publicMembers); + Assert.DoesNotContain("Execute", publicMembers); + } } diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceAcquisitionRunRepositoryContractTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceAcquisitionRunRepositoryContractTests.cs new file mode 100644 index 00000000..6339ee32 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceAcquisitionRunRepositoryContractTests.cs @@ -0,0 +1,139 @@ +using System.Reflection; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class SourceAcquisitionRunRepositoryContractTests +{ + [Fact] + public void SourceAcquisitionRunRepositoryInterfaceSupportsMetadataOnlyPersistenceContract() + { + ISourceAcquisitionRunRepository repository = new InMemorySourceAcquisitionRunRepository(); + + var result = repository.PersistRuns(SourceAcquisitionRunRegistry.CreateDefaultRunResults()); + + Assert.Equal("in_memory", repository.ProviderName); + Assert.Equal(SourceAcquisitionRunRepositoryPersistStatus.Declared, result.Status); + Assert.Equal(3, result.PersistedCount); + Assert.Empty(result.Issues); + } + + [Fact] + public void SourceAcquisitionRunRepositoryValidationRequiresProviderName() + { + var validation = SourceAcquisitionRunRepositoryRegistry.ValidateInputs( + "", + SourceAcquisitionRunRegistry.CreateDefaultRunResults()); + + Assert.False(validation.IsValid); + Assert.Equal( + "SOURCE_ACQUISITION_RUN_REPOSITORY_MISSING_PROVIDER_NAME", + validation.Issues[0].Code); + Assert.Equal("ProviderName", validation.Issues[0].FieldName); + } + + [Fact] + public void SourceAcquisitionRunRepositoryValidationRejectsNullRuns() + { + var validation = SourceAcquisitionRunRepositoryRegistry.ValidateInputs( + "in_memory", + [null]); + + Assert.False(validation.IsValid); + Assert.Equal("SOURCE_ACQUISITION_RUN_REPOSITORY_INVALID_RUN", validation.Issues[0].Code); + Assert.Equal("Runs[0]", validation.Issues[0].FieldName); + } + + [Fact] + public void SourceAcquisitionRunRepositoryPersistResultReportsValidationFailure() + { + var result = SourceAcquisitionRunRepositoryRegistry.CreatePersistResult( + "", + [null]); + + Assert.Equal(SourceAcquisitionRunRepositoryPersistStatus.FailedValidation, result.Status); + Assert.Equal(0, result.PersistedCount); + Assert.Equal(2, result.Issues.Count); + } + + [Fact] + public void SourceAcquisitionRunRepositoryPersistResultSnapshotsIssueCollections() + { + var issues = new List + { + new("CUSTOM_REPOSITORY_WARNING", "custom issue", "Runs", "warning"), + }; + + var result = SourceAcquisitionRunRepositoryRegistry.CreatePersistResult( + "in_memory", + SourceAcquisitionRunRegistry.CreateDefaultRunResults(), + issues); + issues.Clear(); + + Assert.Equal(SourceAcquisitionRunRepositoryPersistStatus.FailedValidation, result.Status); + Assert.Equal(0, result.PersistedCount); + Assert.Single(result.Issues); + Assert.Equal("CUSTOM_REPOSITORY_WARNING", result.Issues[0].Code); + } + + [Fact] + public void SourceAcquisitionRunRepositoryPersistStatusValuesAreDeterministic() + { + Assert.Equal( + [ + SourceAcquisitionRunRepositoryPersistStatus.Declared, + SourceAcquisitionRunRepositoryPersistStatus.FailedValidation, + ], + Enum.GetValues()); + } + + [Fact] + public void SourceAcquisitionRunRepositoryContractRemainsRuntimePassive() + { + var publicMembers = new[] + { + typeof(ISourceAcquisitionRunRepository), + typeof(SourceAcquisitionRunRepositoryIssue), + typeof(SourceAcquisitionRunRepositoryPersistResult), + typeof(SourceAcquisitionRunRepositoryRegistry), + typeof(SourceAcquisitionRunRepositoryValidationResult), + } + .SelectMany(type => type.GetMembers(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + .Select(member => member.Name) + .ToArray(); + var blockedTerms = new[] + { + "Db", + "Sql", + "Postgres", + "Http", + "Open", + "ReadFile", + "Write", + "StatFile", + "Exists", + "Fetch", + "Calculate", + "Factor", + }; + + foreach (var term in blockedTerms) + { + Assert.DoesNotContain(publicMembers, member => member.Contains(term, StringComparison.OrdinalIgnoreCase)); + } + + Assert.DoesNotContain("Parse", publicMembers); + Assert.DoesNotContain("Execute", publicMembers); + } + + private sealed class InMemorySourceAcquisitionRunRepository : ISourceAcquisitionRunRepository + { + public string ProviderName => "in_memory"; + + public SourceAcquisitionRunRepositoryPersistResult PersistRuns( + IEnumerable runs) + { + return SourceAcquisitionRunRepositoryRegistry.CreatePersistResult(ProviderName, runs); + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceCycleOrchestratorTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceCycleOrchestratorTests.cs new file mode 100644 index 00000000..c0fff473 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceCycleOrchestratorTests.cs @@ -0,0 +1,264 @@ +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class SourceCycleOrchestratorTests +{ + [Fact] + public async Task NoSuccessfulYearTargetsDefaultInitialYearForAllSourceFamilies() + { + var session = new RecordingYearStateSession(); + var result = await CreateOrchestrator(session).PreviewAsync(); + + Assert.Equal(3, result.RunCount); + Assert.Equal( + SourceFamilyRegistry.SupportedFamilies.OrderBy(family => family.ToWireName(), StringComparer.Ordinal), + result.Runs.Select(run => run.SourceFamily)); + Assert.All(result.Runs, run => + { + Assert.Null(run.LatestSuccessfulYear); + Assert.Equal(2024, run.TargetYear); + Assert.Equal(SourceCycleRunStatus.NoAvailableSourceYear, run.Status); + Assert.Equal(SourceCycleRunStatus.PersistenceNotImplemented, run.PersistenceStatus); + }); + Assert.Equal(3, session.LatestReadCount); + Assert.Equal(0, session.RecordWriteCount); + Assert.False(result.RecordsInserted); + Assert.False(result.YearStateAdvanced); + } + + [Fact] + public async Task LatestSuccessfulYearTargetsNextYearForAllSourceFamilies() + { + var session = new RecordingYearStateSession( + SourceFamilyRegistry.SupportedFamilies.ToDictionary(family => family, _ => (int?)2024)); + var result = await CreateOrchestrator(session).PreviewAsync(); + + Assert.All(result.Runs, run => + { + Assert.Equal(2024, run.LatestSuccessfulYear); + Assert.Equal(2025, run.TargetYear); + }); + Assert.Equal(0, session.RecordWriteCount); + } + + [Fact] + public async Task UnavailableTargetYearReturnsNoAvailableSourceYear() + { + var result = await CreateOrchestrator(new RecordingYearStateSession()).PreviewAsync(); + + Assert.All(result.Runs, run => + { + Assert.Equal(SourceCycleRunStatus.NoAvailableSourceYear, run.Status); + Assert.Equal(SourceCycleRunStatus.NoAvailableSourceYear, run.ParserStatus); + Assert.Equal(0, run.ArtifactCount); + Assert.Equal(0, run.ParsedRowCount); + }); + } + + [Fact] + public async Task ParserHandoffOutputIsRepresentedForConfiguredLocalArtifact() + { + var artifactPath = FixturePath("ghg_protocol", "ghg_protocol_sample_factors.csv"); + var orchestrator = CreateOrchestrator( + new RecordingYearStateSession(), + artifacts: + [ + new SourceCycleArtifact( + SourceFamily.GhgProtocol, + 2024, + artifactPath, + "text/csv", + ".csv", + "v1"), + ]); + + var result = await orchestrator.PreviewAsync(); + var run = result.Runs.Single(item => item.SourceFamily == SourceFamily.GhgProtocol); + + Assert.Equal(SourceCycleRunStatus.PersistenceNotImplemented, run.Status); + Assert.Equal(SourceCycleRunStatus.Parsed, run.ParserStatus); + Assert.Equal(SourceCycleRunStatus.PersistenceNotImplemented, run.PersistenceStatus); + Assert.Equal(ParserRunStatus.Completed, run.ParserRunStatus); + Assert.Equal(1, run.ArtifactCount); + Assert.Equal(2, run.ParsedRowCount); + Assert.Equal(1, run.ParserIssueCount); + Assert.Equal(artifactPath, run.ArtifactReference); + } + + [Fact] + public async Task YearStateIsNotAdvancedByParsingAloneAndNoDbInsertIsAttempted() + { + var session = new RecordingYearStateSession(); + var artifactPath = FixturePath("defra_desnz", "defra_desnz_normalized_factors.csv"); + var result = await CreateOrchestrator( + session, + enabledFamilies: [SourceFamily.DefraDesnz], + artifacts: + [ + new SourceCycleArtifact(SourceFamily.DefraDesnz, 2024, artifactPath, "text/csv", ".csv"), + ]).PreviewAsync(); + + Assert.False(result.PostgreSQLConnectionOpened); + Assert.False(result.PostgreSQLSqlExecuted); + Assert.False(result.RecordsInserted); + Assert.False(result.YearStateAdvanced); + Assert.Equal(0, session.RecordWriteCount); + Assert.Equal(SourceCycleRunStatus.PersistenceNotImplemented, result.Runs[0].Status); + } + + [Fact] + public async Task UnsupportedArtifactShapeFailsClosedWithoutNetworkAccess() + { + var result = await CreateOrchestrator( + new RecordingYearStateSession(), + enabledFamilies: [SourceFamily.IpccEfdb], + artifacts: + [ + new SourceCycleArtifact(SourceFamily.IpccEfdb, 2024, "https://example.invalid/ipcc.csv", "text/csv", ".csv"), + ]).PreviewAsync(); + + var run = result.Runs.Single(); + Assert.Equal(SourceCycleRunStatus.ParserNotAvailable, run.Status); + Assert.Equal(SourceCycleRunStatus.ParserNotAvailable, run.ParserStatus); + Assert.False(result.NetworkAccessAttempted); + Assert.Equal(0, run.ParsedRowCount); + } + + [Fact] + public void SourceCycleConfigurationLoadsEnabledFamiliesAndArtifactsFromConfig() + { + var artifactPath = FixturePath("ipcc_efdb", "ipcc_efdb_sample_factors.csv"); + var configPath = WriteSourceCycleConfig( + $$""" + { + "enabled_source_families": ["ipcc_efdb"], + "source_artifacts": { + "ipcc_efdb": { + "2024": { + "path": "{{JsonEscape(artifactPath)}}", + "content_type": "text/csv", + "extension": ".csv", + "version_label": "efdb-v2024" + } + } + } + } + """); + + var families = SourceCycleConfiguration.LoadEnabledSourceFamilies(configPath); + var artifacts = SourceCycleConfiguration.LoadArtifacts(configPath); + + Assert.Equal([SourceFamily.IpccEfdb], families); + var artifact = Assert.Single(artifacts); + Assert.Equal(SourceFamily.IpccEfdb, artifact.SourceFamily); + Assert.Equal(2024, artifact.ReportingYear); + Assert.Equal(artifactPath, artifact.ArtifactReference); + Assert.Equal("efdb-v2024", artifact.VersionLabel); + } + + [Fact] + public async Task SourceYearsArtifactUrlFileUriCanBeParsedWithoutNetworkAccess() + { + var artifactPath = FixturePath("defra_desnz", "defra_desnz_normalized_factors.csv"); + var artifactUri = new Uri(artifactPath).AbsoluteUri; + var configPath = WriteSourceCycleConfig( + $$""" + { + "enabled_source_families": ["defra_desnz"], + "source_years": { + "defra_desnz": { + "2024": { + "artifact_url": "{{artifactUri}}", + "content_type": "text/csv", + "format_hint": "csv", + "version_label": "conversion-factors-2024" + } + } + } + } + """); + + var result = await CreateOrchestrator( + new RecordingYearStateSession(), + SourceCycleConfiguration.LoadEnabledSourceFamilies(configPath), + SourceCycleConfiguration.LoadArtifacts(configPath)).PreviewAsync(); + + var run = Assert.Single(result.Runs); + Assert.Equal(SourceCycleRunStatus.PersistenceNotImplemented, run.Status); + Assert.Equal(SourceCycleRunStatus.Parsed, run.ParserStatus); + Assert.Equal(2, run.ParsedRowCount); + Assert.False(result.NetworkAccessAttempted); + } + + private static SourceCycleOrchestrator CreateOrchestrator( + RecordingYearStateSession session, + IEnumerable? enabledFamilies = null, + IEnumerable? artifacts = null) => + new( + new PostgreSQLSourceFamilyYearStateRepository(session), + enabledFamilies, + artifacts); + + private static string FixturePath(string familyDirectory, string fileName) + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixtureDirectory = Path.Combine( + directory.FullName, + "tests", + "fixtures", + "source_documents", + familyDirectory); + if (Directory.Exists(fixtureDirectory)) + { + return Path.Combine(fixtureDirectory, fileName); + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate source document fixture directory."); + } + + private static string WriteSourceCycleConfig(string json) + { + var path = Path.Combine(Path.GetTempPath(), $"carbonops-source-cycle-{Guid.NewGuid():N}.json"); + File.WriteAllText(path, json); + return path; + } + + private static string JsonEscape(string value) => value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal); + + private sealed class RecordingYearStateSession : IPostgreSQLSourceFamilyYearStateSession + { + private readonly IReadOnlyDictionary _latestYears; + + public RecordingYearStateSession(IReadOnlyDictionary? latestYears = null) + { + _latestYears = latestYears ?? new Dictionary(); + } + + public int LatestReadCount { get; private set; } + + public int RecordWriteCount { get; private set; } + + public Task LatestSuccessfulYearAsync( + SourceFamily sourceFamily, + CancellationToken cancellationToken = default) + { + LatestReadCount++; + return Task.FromResult(_latestYears.TryGetValue(sourceFamily, out var value) ? value : null); + } + + public Task RecordSuccessfulYearAsync( + SourceFamily sourceFamily, + int ingestedYear, + CancellationToken cancellationToken = default) + { + RecordWriteCount++; + return Task.CompletedTask; + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceDocumentRepositoryContractTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceDocumentRepositoryContractTests.cs new file mode 100644 index 00000000..17b0901b --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceDocumentRepositoryContractTests.cs @@ -0,0 +1,164 @@ +using System.Reflection; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class SourceDocumentRepositoryContractTests +{ + [Fact] + public void SourceDocumentRepositoryTypesArePublic() + { + var publicContractTypes = new[] + { + typeof(ISourceDocumentRepository), + typeof(SourceDocumentRepositoryPersistStatus), + typeof(SourceDocumentRepositoryIssue), + typeof(SourceDocumentRepositoryPersistResult), + typeof(SourceDocumentRepositoryValidationResult), + typeof(SourceDocumentRepositoryRegistry), + }; + + Assert.Equal( + [ + "ISourceDocumentRepository", + "SourceDocumentRepositoryPersistStatus", + "SourceDocumentRepositoryIssue", + "SourceDocumentRepositoryPersistResult", + "SourceDocumentRepositoryValidationResult", + "SourceDocumentRepositoryRegistry", + ], + publicContractTypes.Select(type => type.Name)); + Assert.All(publicContractTypes, type => Assert.True(type.IsPublic, $"{type.Name} must be public.")); + } + + [Fact] + public void SourceDocumentRepositoryInterfaceSupportsMetadataOnlyPersistenceContract() + { + ISourceDocumentRepository repository = new InMemorySourceDocumentRepository(); + + var result = repository.PersistSourceDocuments( + SourceDocumentPersistenceMapper.MapDefaultDryRunManifest().Records); + + Assert.Equal("in_memory", repository.ProviderName); + Assert.Equal(SourceDocumentRepositoryPersistStatus.Declared, result.Status); + Assert.Equal(3, result.PersistedCount); + Assert.Empty(result.Issues); + } + + [Fact] + public void SourceDocumentRepositoryValidationRequiresProviderName() + { + var validation = SourceDocumentRepositoryRegistry.ValidateInputs( + "", + SourceDocumentPersistenceMapper.MapDefaultDryRunManifest().Records); + + Assert.False(validation.IsValid); + Assert.Equal("SOURCE_DOCUMENT_REPOSITORY_MISSING_PROVIDER_NAME", validation.Issues[0].Code); + Assert.Equal("ProviderName", validation.Issues[0].FieldName); + } + + [Fact] + public void SourceDocumentRepositoryValidationRejectsNullRecords() + { + var validation = SourceDocumentRepositoryRegistry.ValidateInputs( + "in_memory", + [null]); + + Assert.False(validation.IsValid); + Assert.Equal("SOURCE_DOCUMENT_REPOSITORY_INVALID_RECORD", validation.Issues[0].Code); + Assert.Equal("Records[0]", validation.Issues[0].FieldName); + } + + [Fact] + public void SourceDocumentRepositoryPersistResultReportsValidationFailure() + { + var result = SourceDocumentRepositoryRegistry.CreatePersistResult( + "", + [null]); + + Assert.Equal(SourceDocumentRepositoryPersistStatus.FailedValidation, result.Status); + Assert.Equal(0, result.PersistedCount); + Assert.Equal(2, result.Issues.Count); + } + + [Fact] + public void SourceDocumentRepositoryPersistResultSnapshotsIssueCollections() + { + var issues = new List + { + new("CUSTOM_SOURCE_DOCUMENT_REPOSITORY_WARNING", "custom issue", "Records", "warning"), + }; + + var result = SourceDocumentRepositoryRegistry.CreatePersistResult( + "in_memory", + SourceDocumentPersistenceMapper.MapDefaultDryRunManifest().Records, + issues); + issues.Clear(); + + Assert.Equal(SourceDocumentRepositoryPersistStatus.FailedValidation, result.Status); + Assert.Equal(0, result.PersistedCount); + Assert.Single(result.Issues); + Assert.Equal("CUSTOM_SOURCE_DOCUMENT_REPOSITORY_WARNING", result.Issues[0].Code); + } + + [Fact] + public void SourceDocumentRepositoryPersistStatusValuesAreDeterministic() + { + Assert.Equal( + [ + SourceDocumentRepositoryPersistStatus.Declared, + SourceDocumentRepositoryPersistStatus.FailedValidation, + ], + Enum.GetValues()); + } + + [Fact] + public void SourceDocumentRepositoryContractRemainsRuntimePassive() + { + var publicMembers = new[] + { + typeof(ISourceDocumentRepository), + typeof(SourceDocumentRepositoryIssue), + typeof(SourceDocumentRepositoryPersistResult), + typeof(SourceDocumentRepositoryRegistry), + typeof(SourceDocumentRepositoryValidationResult), + } + .SelectMany(type => type.GetMembers(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + .Select(member => member.Name) + .ToArray(); + var blockedTerms = new[] + { + "Db", + "Sql", + "Postgres", + "Http", + "Open", + "ReadFile", + "Write", + "StatFile", + "Exists", + "Fetch", + "Calculate", + "Factor", + }; + + foreach (var term in blockedTerms) + { + Assert.DoesNotContain(publicMembers, member => member.Contains(term, StringComparison.OrdinalIgnoreCase)); + } + + Assert.DoesNotContain("Parse", publicMembers); + Assert.DoesNotContain("Execute", publicMembers); + } + + private sealed class InMemorySourceDocumentRepository : ISourceDocumentRepository + { + public string ProviderName => "in_memory"; + + public SourceDocumentRepositoryPersistResult PersistSourceDocuments( + IEnumerable records) + { + return SourceDocumentRepositoryRegistry.CreatePersistResult(ProviderName, records); + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceFamilyRepositoryContractTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceFamilyRepositoryContractTests.cs new file mode 100644 index 00000000..d5430abb --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceFamilyRepositoryContractTests.cs @@ -0,0 +1,282 @@ +using System.Reflection; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class SourceFamilyRepositoryContractTests +{ + [Fact] + public void SourceFamilyRepositoryTypesArePublic() + { + var publicContractTypes = new[] + { + typeof(ISourceFamilyRepository), + typeof(SourceFamilyMasterRecord), + typeof(SourceFamilyDetailRecord), + typeof(SourceFamilyRepositoryPersistStatus), + typeof(SourceFamilyRepositoryIssue), + typeof(SourceFamilyRepositoryPersistResult), + typeof(SourceFamilyRepositoryValidationResult), + typeof(SourceFamilyRepositoryTableNames), + typeof(SourceFamilyRepositoryRegistry), + }; + + Assert.Equal( + [ + "ISourceFamilyRepository", + "SourceFamilyMasterRecord", + "SourceFamilyDetailRecord", + "SourceFamilyRepositoryPersistStatus", + "SourceFamilyRepositoryIssue", + "SourceFamilyRepositoryPersistResult", + "SourceFamilyRepositoryValidationResult", + "SourceFamilyRepositoryTableNames", + "SourceFamilyRepositoryRegistry", + ], + publicContractTypes.Select(type => type.Name)); + Assert.All(publicContractTypes, type => Assert.True(type.IsPublic, $"{type.Name} must be public.")); + } + + [Fact] + public void SourceFamilyRepositoryInterfaceSupportsMetadataOnlyPersistenceContract() + { + ISourceFamilyRepository repository = new InMemorySourceFamilyRepository(); + + var result = repository.PersistSourceFamilyRecords( + [CreateMasterRecord()], + [CreateDetailRecord()]); + + Assert.Equal("in_memory", repository.ProviderName); + Assert.Equal(SourceFamilyRepositoryPersistStatus.Declared, result.Status); + Assert.Equal(1, result.PersistedMasterCount); + Assert.Equal(1, result.PersistedDetailCount); + Assert.Empty(result.Issues); + } + + [Fact] + public void SourceFamilyRepositoryValidationRequiresProviderName() + { + var validation = SourceFamilyRepositoryRegistry.ValidateInputs( + "", + [CreateMasterRecord()], + [CreateDetailRecord()]); + + Assert.False(validation.IsValid); + Assert.Equal("SOURCE_FAMILY_REPOSITORY_MISSING_PROVIDER_NAME", validation.Issues[0].Code); + Assert.Equal("ProviderName", validation.Issues[0].FieldName); + } + + [Fact] + public void SourceFamilyRepositoryValidationRejectsNullRecords() + { + var validation = SourceFamilyRepositoryRegistry.ValidateInputs( + "in_memory", + [null], + [null]); + + Assert.False(validation.IsValid); + Assert.Equal("SOURCE_FAMILY_REPOSITORY_INVALID_MASTER_RECORD", validation.Issues[0].Code); + Assert.Equal("MasterRecords[0]", validation.Issues[0].FieldName); + Assert.Equal("SOURCE_FAMILY_REPOSITORY_INVALID_DETAIL_RECORD", validation.Issues[1].Code); + Assert.Equal("DetailRecords[0]", validation.Issues[1].FieldName); + } + + [Fact] + public void SourceFamilyRepositoryValidationRejectsMissingRequiredFields() + { + var validation = SourceFamilyRepositoryRegistry.ValidateInputs( + "in_memory", + [CreateMasterRecord(sourceFamilyMasterId: "")], + []); + + Assert.False(validation.IsValid); + Assert.Equal("SOURCE_FAMILY_REPOSITORY_MISSING_REQUIRED_FIELD", validation.Issues[0].Code); + Assert.Equal("MasterRecords[0].SourceFamilyMasterId", validation.Issues[0].FieldName); + } + + [Fact] + public void SourceFamilyRepositoryValidationRequiresDetailMasterReference() + { + var validation = SourceFamilyRepositoryRegistry.ValidateInputs( + "in_memory", + [CreateMasterRecord()], + [CreateDetailRecord(sourceFamilyMasterId: "missing_master")]); + + Assert.False(validation.IsValid); + Assert.Equal("SOURCE_FAMILY_REPOSITORY_DETAIL_MASTER_NOT_DECLARED", validation.Issues[0].Code); + Assert.Equal("DetailRecords[0].SourceFamilyMasterId", validation.Issues[0].FieldName); + } + + [Fact] + public void SourceFamilyRepositoryValidationRequiresDetailMasterReferenceForSameSourceFamily() + { + var validation = SourceFamilyRepositoryRegistry.ValidateInputs( + "in_memory", + [CreateMasterRecord(SourceFamily.DefraDesnz, sourceFamilyMasterId: "shared_master")], + [CreateDetailRecord(SourceFamily.IpccEfdb, sourceFamilyMasterId: "shared_master")]); + + Assert.False(validation.IsValid); + Assert.Equal("SOURCE_FAMILY_REPOSITORY_DETAIL_MASTER_NOT_DECLARED", validation.Issues[0].Code); + } + + [Fact] + public void SourceFamilyRepositoryPersistResultReportsValidationFailure() + { + var result = SourceFamilyRepositoryRegistry.CreatePersistResult( + "", + [null], + [null]); + + Assert.Equal(SourceFamilyRepositoryPersistStatus.FailedValidation, result.Status); + Assert.Equal(0, result.PersistedMasterCount); + Assert.Equal(0, result.PersistedDetailCount); + Assert.Equal(3, result.Issues.Count); + } + + [Fact] + public void SourceFamilyRepositoryPersistResultSnapshotsInputAndIssueCollections() + { + var masterRecords = new List { CreateMasterRecord() }; + var detailRecords = new List { CreateDetailRecord() }; + var issues = new List + { + new("CUSTOM_SOURCE_FAMILY_REPOSITORY_WARNING", "custom issue", "MasterRecords", "warning"), + }; + + var result = SourceFamilyRepositoryRegistry.CreatePersistResult( + "in_memory", + masterRecords, + detailRecords, + issues); + masterRecords.Clear(); + detailRecords.Clear(); + issues.Clear(); + + Assert.Equal(SourceFamilyRepositoryPersistStatus.FailedValidation, result.Status); + Assert.Equal(0, result.PersistedMasterCount); + Assert.Equal(0, result.PersistedDetailCount); + Assert.Single(result.Issues); + Assert.Equal("CUSTOM_SOURCE_FAMILY_REPOSITORY_WARNING", result.Issues[0].Code); + } + + [Fact] + public void SourceFamilyRepositoryExposesCatalogTableNames() + { + Assert.Equal( + new SourceFamilyRepositoryTableNames( + "ghg_emission_factor_masters", + "ghg_emission_factor_details"), + SourceFamilyRepositoryRegistry.GetTableNames(SourceFamily.GhgProtocol)); + Assert.Equal( + new SourceFamilyRepositoryTableNames( + "defra_emission_factor_masters", + "defra_emission_factor_details"), + SourceFamilyRepositoryRegistry.GetTableNames(SourceFamily.DefraDesnz)); + Assert.Equal( + new SourceFamilyRepositoryTableNames( + "ipcc_emission_factor_masters", + "ipcc_emission_factor_details"), + SourceFamilyRepositoryRegistry.GetTableNames(SourceFamily.IpccEfdb)); + } + + [Fact] + public void SourceFamilyRepositoryPersistStatusValuesAreDeterministic() + { + Assert.Equal( + [ + SourceFamilyRepositoryPersistStatus.Declared, + SourceFamilyRepositoryPersistStatus.FailedValidation, + ], + Enum.GetValues()); + } + + [Fact] + public void SourceFamilyRepositoryContractRemainsRuntimePassive() + { + var publicMembers = new[] + { + typeof(ISourceFamilyRepository), + typeof(SourceFamilyMasterRecord), + typeof(SourceFamilyDetailRecord), + typeof(SourceFamilyRepositoryIssue), + typeof(SourceFamilyRepositoryPersistResult), + typeof(SourceFamilyRepositoryRegistry), + typeof(SourceFamilyRepositoryValidationResult), + typeof(SourceFamilyRepositoryTableNames), + } + .SelectMany(type => type.GetMembers(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + .Select(member => member.Name) + .ToArray(); + var blockedTerms = new[] + { + "Db", + "Sql", + "Postgres", + "Http", + "Open", + "ReadFile", + "Write", + "StatFile", + "Exists", + "Fetch", + "Calculate", + }; + + foreach (var term in blockedTerms) + { + Assert.DoesNotContain(publicMembers, member => member.Contains(term, StringComparison.OrdinalIgnoreCase)); + } + + Assert.DoesNotContain("Parse", publicMembers); + Assert.DoesNotContain("Execute", publicMembers); + } + + private static SourceFamilyMasterRecord CreateMasterRecord( + SourceFamily sourceFamily = SourceFamily.DefraDesnz, + string sourceFamilyMasterId = "defra_master_001") + { + return new SourceFamilyMasterRecord( + sourceFamily, + sourceFamilyMasterId, + "source_document_001", + "defra_2025_publication", + "declared", + null, + null, + "checksum_master_001", + "dry_run_timestamp_unavailable", + "dry_run_timestamp_unavailable"); + } + + private static SourceFamilyDetailRecord CreateDetailRecord( + SourceFamily sourceFamily = SourceFamily.DefraDesnz, + string sourceFamilyMasterId = "defra_master_001") + { + return new SourceFamilyDetailRecord( + sourceFamily, + "defra_detail_001", + sourceFamilyMasterId, + "defra_row_001", + "1.25", + "kgco2e", + "declared", + "checksum_detail_001", + "dry_run_timestamp_unavailable", + "dry_run_timestamp_unavailable"); + } + + private sealed class InMemorySourceFamilyRepository : ISourceFamilyRepository + { + public string ProviderName => "in_memory"; + + public SourceFamilyRepositoryPersistResult PersistSourceFamilyRecords( + IEnumerable masterRecords, + IEnumerable detailRecords) + { + return SourceFamilyRepositoryRegistry.CreatePersistResult( + ProviderName, + masterRecords, + detailRecords); + } + } +} diff --git a/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceOnboardingRegistryContractTests.cs b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceOnboardingRegistryContractTests.cs new file mode 100644 index 00000000..ef9971d1 --- /dev/null +++ b/tests/dotnet/CarbonOps.Parser.Contracts.Tests/SourceOnboardingRegistryContractTests.cs @@ -0,0 +1,452 @@ +using System.Reflection; +using System.Text.Json; +using CarbonOps.Parser.Contracts; + +namespace CarbonOps.Parser.Contracts.Tests; + +public sealed class SourceOnboardingRegistryContractTests +{ + [Fact] + public void Phase2SourceOnboardingRegistryContainsPhaseOneSourceFamilies() + { + var registry = SourceOnboardingRegistry.CreatePhase2SourceOnboardingRegistry(); + + Assert.Equal(3, registry.EntryCount); + Assert.Equal( + [ + "ghg_protocol", + "defra_desnz", + "ipcc_efdb", + ], + registry.Entries.Select(entry => entry.SourceFamily)); + Assert.Equal(["ghg_protocol", "defra_desnz", "ipcc_efdb"], SourceOnboardingRegistry.Phase2OnboardingSourceFamilies); + Assert.Equal(["ghg_protocol", "defra_desnz", "ipcc_efdb"], registry.Entries.Select(entry => entry.SourceId)); + Assert.All(registry.Entries, entry => Assert.True(entry.Enabled)); + } + + [Fact] + public void Phase2SourceOnboardingRegistryRepresentsFutureOnboardingMetadata() + { + var registry = SourceOnboardingRegistry.CreatePhase2SourceOnboardingRegistry(); + + foreach (var entry in registry.Entries) + { + Assert.Equal(SourceOnboardingDiscoveryStrategy.DeclaredReference, entry.DiscoveryStrategy); + Assert.Equal(SourceOnboardingUpdateCadence.Unknown, entry.UpdateCadence); + Assert.Equal(ExpectedParserKey(entry.SourceFamily), entry.ParserCapability.ParserKey); + Assert.Equal(ParserSourceFormat.DiscoveryReference, entry.ParserCapability.ParserSourceFormat); + Assert.Equal( + ["document_id", "display_name", "source_reference", "expected_format"], + entry.ValidationExpectations.RequiredDocumentFields); + Assert.StartsWith($"discovery://{entry.SourceId}/", entry.Documents[0].SourceReference, StringComparison.Ordinal); + } + } + + [Fact] + public void Phase2SourceOnboardingRegistryMatchesSharedParityExpectations() + { + using var expectations = LoadParityExpectations(); + var root = expectations.RootElement; + var registry = SourceOnboardingRegistry.CreatePhase2SourceOnboardingRegistry(); + var expectedFamilies = JsonStringArray(root.GetProperty("phase2_source_families")); + var expectedEntries = root.GetProperty("entries").EnumerateArray().ToArray(); + + Assert.Equal(expectedFamilies, SourceOnboardingRegistry.Phase2OnboardingSourceFamilies); + Assert.Equal(expectedEntries.Length, registry.Entries.Count); + + for (var index = 0; index < expectedEntries.Length; index++) + { + var expected = expectedEntries[index]; + var actual = registry.Entries[index]; + + Assert.Equal(expected.GetProperty("source_id").GetString(), actual.SourceId); + Assert.Equal(expected.GetProperty("source_family").GetString(), actual.SourceFamily); + Assert.Equal(expected.GetProperty("display_name").GetString(), actual.DisplayName); + Assert.Equal(expected.GetProperty("discovery_strategy").GetString(), DiscoveryStrategyWireName(actual.DiscoveryStrategy)); + Assert.Equal(expected.GetProperty("update_cadence").GetString(), UpdateCadenceWireName(actual.UpdateCadence)); + Assert.Equal(expected.GetProperty("enabled").GetBoolean(), actual.Enabled); + + var expectedDocument = expected.GetProperty("documents")[0]; + Assert.Equal(expectedDocument.GetProperty("document_id").GetString(), actual.Documents[0].DocumentId); + Assert.Equal(expectedDocument.GetProperty("display_name").GetString(), actual.Documents[0].DisplayName); + Assert.Equal(expectedDocument.GetProperty("source_reference").GetString(), actual.Documents[0].SourceReference); + Assert.Equal(expectedDocument.GetProperty("expected_format").GetString(), actual.Documents[0].ExpectedFormat); + Assert.Equal(expectedDocument.GetProperty("required").GetBoolean(), actual.Documents[0].Required); + + var expectedCapability = expected.GetProperty("parser_capability"); + Assert.Equal(expectedCapability.GetProperty("parser_key").GetString(), actual.ParserCapability.ParserKey.Value); + Assert.Equal(expectedCapability.GetProperty("parser_source_format").GetString(), actual.ParserCapability.ParserSourceFormat.ToWireName()); + Assert.Equal(expectedCapability.GetProperty("supports_parser_execution").GetBoolean(), actual.ParserCapability.SupportsParserExecution); + Assert.Equal(expectedCapability.GetProperty("capability_notes").GetString(), actual.ParserCapability.CapabilityNotes); + + var expectedValidation = expected.GetProperty("validation_expectations"); + Assert.Equal(JsonStringArray(expectedValidation.GetProperty("required_document_fields")), actual.ValidationExpectations.RequiredDocumentFields); + Assert.Equal(expectedValidation.GetProperty("checksum_required").GetBoolean(), actual.ValidationExpectations.ChecksumRequired); + Assert.Equal(expectedValidation.GetProperty("schema_validation_required").GetBoolean(), actual.ValidationExpectations.SchemaValidationRequired); + Assert.Equal(expectedValidation.GetProperty("validation_notes").GetString(), actual.ValidationExpectations.ValidationNotes); + + var expectedSafety = expected.GetProperty("runtime_safety"); + Assert.Equal(expectedSafety.GetProperty("allows_network_calls").GetBoolean(), actual.RuntimeSafety.AllowsNetworkCalls); + Assert.Equal(expectedSafety.GetProperty("allows_file_reads").GetBoolean(), actual.RuntimeSafety.AllowsFileReads); + Assert.Equal(expectedSafety.GetProperty("allows_database_writes").GetBoolean(), actual.RuntimeSafety.AllowsDatabaseWrites); + Assert.Equal(expectedSafety.GetProperty("requires_credentials").GetBoolean(), actual.RuntimeSafety.RequiresCredentials); + Assert.Equal(expectedSafety.GetProperty("safety_notes").GetString(), actual.RuntimeSafety.SafetyNotes); + } + + Assert.Equal( + [ + "Python validation raises TypeError or ValueError for invalid registries; .NET validation returns ContractValidationResult errors and lookup helpers throw ArgumentException when an invalid custom registry is supplied.", + ], + JsonStringArray(root.GetProperty("accepted_asymmetries"))); + } + + [Fact] + public void Phase2SourceOnboardingParserKeysAlignWithPhaseOneDescriptorRegistry() + { + var registry = SourceOnboardingRegistry.CreatePhase2SourceOnboardingRegistry(); + + foreach (var entry in registry.Entries) + { + Assert.True(ParserAdapterDescriptorRegistry.TryGetBySourceFamily(ParseSourceFamily(entry.SourceFamily), out var descriptor)); + Assert.NotNull(descriptor); + Assert.Equal(descriptor!.ParserKey, entry.ParserCapability.ParserKey); + } + } + + [Fact] + public void Phase2SourceOnboardingRegistryIsRuntimeSafeByDefault() + { + var registry = SourceOnboardingRegistry.CreatePhase2SourceOnboardingRegistry(); + + foreach (var entry in registry.Entries) + { + Assert.False(entry.ParserCapability.SupportsParserExecution); + Assert.False(entry.ValidationExpectations.ChecksumRequired); + Assert.False(entry.ValidationExpectations.SchemaValidationRequired); + Assert.False(entry.RuntimeSafety.AllowsNetworkCalls); + Assert.False(entry.RuntimeSafety.AllowsFileReads); + Assert.False(entry.RuntimeSafety.AllowsDatabaseWrites); + Assert.False(entry.RuntimeSafety.RequiresCredentials); + } + } + + [Fact] + public void ValidRegistryEntryPassesValidation() + { + var registry = new SourceOnboardingRegistry([ValidEntry("new_registry_source")]); + + var result = registry.Validate(); + + Assert.True(result.IsValid); + Assert.Empty(result.Errors); + } + + [Fact] + public void InvalidEntryValuesFailValidation() + { + var registry = new SourceOnboardingRegistry( + [ + ValidEntry( + "invalid_registry_source", + " ", + discoveryStrategy: (SourceOnboardingDiscoveryStrategy)999, + updateCadence: (SourceOnboardingUpdateCadence)999), + ]); + + var result = registry.Validate(); + + Assert.False(result.IsValid); + Assert.Contains("SourceFamily is required.", result.Errors); + Assert.Contains("DiscoveryStrategy must be a defined source onboarding discovery strategy.", result.Errors); + Assert.Contains("UpdateCadence must be a defined source onboarding update cadence.", result.Errors); + } + + [Fact] + public void DuplicateIdentifiersFailValidation() + { + var sourceIdRegistry = new SourceOnboardingRegistry( + [ + ValidEntry("duplicate_registry_source", "duplicate_registry_source"), + ValidEntry("duplicate_registry_source", "duplicate_registry_source_two"), + ]); + var sourceFamilyRegistry = new SourceOnboardingRegistry( + [ + ValidEntry("duplicate_registry_family_one", "duplicate_registry_family"), + ValidEntry("duplicate_registry_family_two", "duplicate_registry_family"), + ]); + var documentRegistry = new SourceOnboardingRegistry( + [ + ValidEntry( + "duplicate_registry_document", + "duplicate_registry_document", + documents: + [ + ValidDocument("duplicate_document"), + ValidDocument("duplicate_document"), + ]), + ]); + + Assert.Contains("Duplicate SourceId found: duplicate_registry_source", sourceIdRegistry.Validate().Errors); + Assert.Contains("Duplicate SourceFamily found: duplicate_registry_family", sourceFamilyRegistry.Validate().Errors); + Assert.Contains("Duplicate DocumentId found: duplicate_document", documentRegistry.Validate().Errors); + } + + [Fact] + public void MissingRequiredFieldsFailValidation() + { + var registry = new SourceOnboardingRegistry( + [ + new SourceOnboardingRegistryEntry( + "", + "", + " ", + [], + SourceOnboardingDiscoveryStrategy.SourceSpecificDiscovery, + new SourceOnboardingParserCapability( + new ParserKey(" "), + (ParserSourceFormat)999, + supportsParserExecution: false, + ""), + new SourceOnboardingValidationExpectations( + [], + checksumRequired: true, + schemaValidationRequired: true, + ""), + SourceOnboardingUpdateCadence.Periodic, + new SourceOnboardingRuntimeSafety( + allowsNetworkCalls: false, + allowsFileReads: false, + allowsDatabaseWrites: false, + requiresCredentials: false, + "")), + ]); + + var result = registry.Validate(); + + Assert.False(result.IsValid); + Assert.Contains("SourceId is required.", result.Errors); + Assert.Contains("DisplayName is required.", result.Errors); + Assert.Contains("Documents must include at least one document for SourceId ''.", result.Errors); + Assert.Contains("ParserCapability.ParserKey is required.", result.Errors); + Assert.Contains("ParserCapability.ParserSourceFormat must be a defined parser source format.", result.Errors); + Assert.Contains("ParserCapability.CapabilityNotes is required.", result.Errors); + Assert.Contains("RequiredDocumentFields must not be empty for SourceId ''.", result.Errors); + Assert.Contains("ValidationNotes is required.", result.Errors); + Assert.Contains("SafetyNotes is required.", result.Errors); + } + + [Fact] + public void DeterministicOrderingIsEnforced() + { + var registry = SourceOnboardingRegistry.CreatePhase2SourceOnboardingRegistry(); + var reordered = new SourceOnboardingRegistry( + [ + registry.Entries[1], + registry.Entries[0], + registry.Entries[2], + ]); + var unorderedDocuments = new SourceOnboardingRegistry( + [ + ValidEntry( + "unordered_documents", + "unordered_documents", + documents: + [ + ValidDocument("z_document"), + ValidDocument("a_document"), + ]), + ]); + var mixedSourceIds = new SourceOnboardingRegistry( + [ + ValidEntry("z_custom_source_id", "ghg_protocol"), + ValidEntry("a_custom_source_id", "custom_source_family"), + ]); + var mixedSourceIdsReordered = new SourceOnboardingRegistry(mixedSourceIds.Entries.Reverse()); + + Assert.Equal(SourceOnboardingRegistry.Phase2OnboardingSourceFamilies, registry.Entries.Select(entry => entry.SourceFamily)); + Assert.Equal(registry.Entries, SourceOnboardingRegistry.ListEntries(registry)); + Assert.True(mixedSourceIds.Validate().IsValid); + Assert.Contains( + "Entries must follow Phase 1 source order, then SourceId order.", + reordered.Validate().Errors); + Assert.Contains( + "Entries must follow Phase 1 source order, then SourceId order.", + mixedSourceIdsReordered.Validate().Errors); + Assert.Contains( + "Documents must be ordered by DocumentId for SourceId 'unordered_documents'.", + unorderedDocuments.Validate().Errors); + } + + [Fact] + public void LookupIsDeterministicAndReturnsDeclaredEntries() + { + var registry = SourceOnboardingRegistry.CreatePhase2SourceOnboardingRegistry(); + + foreach (var sourceFamily in SourceFamilyRegistry.SupportedFamilies) + { + Assert.True(SourceOnboardingRegistry.TryGetBySourceFamily(sourceFamily, out var entry, registry)); + Assert.NotNull(entry); + Assert.Equal(sourceFamily.ToWireName(), entry!.SourceFamily); + Assert.Equal(sourceFamily.ToWireName(), entry.SourceId); + } + + Assert.False(SourceOnboardingRegistry.TryGetBySourceFamily("unknown_source", out var missing, registry)); + Assert.Null(missing); + } + + [Fact] + public void LookupRejectsInvalidCustomRegistry() + { + var entry = ValidEntry("duplicate_registry_source"); + var registry = new SourceOnboardingRegistry( + [ + entry, + ValidEntry("duplicate_registry_source", "duplicate_registry_source_two"), + ]); + + var exception = Assert.Throws(() => + SourceOnboardingRegistry.TryGetBySourceFamily("duplicate_registry_source", out _, registry)); + + Assert.Contains("Duplicate SourceId found: duplicate_registry_source", exception.Message); + } + + [Fact] + public void RegistrySnapshotsInputCollections() + { + var entries = new List + { + ValidEntry("snapshot_source"), + }; + var registry = new SourceOnboardingRegistry(entries); + + entries.Clear(); + + Assert.Single(registry.Entries); + Assert.Equal("snapshot_source", registry.Entries[0].SourceId); + } + + [Fact] + public void ContractDoesNotIntroduceRuntimeDiscoveryParserDatabaseOrDownloadSurface() + { + var publicMembers = typeof(SourceOnboardingRegistry) + .GetMembers(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly) + .Concat(typeof(SourceOnboardingRegistryEntry) + .GetMembers(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + .Select(member => member.Name) + .ToArray(); + var blockedTerms = new[] + { + "Db", + "Sql", + "Postgres", + "Open", + "Write", + "Download", + "Fetch", + "Persist", + }; + + foreach (var term in blockedTerms) + { + Assert.DoesNotContain(publicMembers, member => member.Contains(term, StringComparison.OrdinalIgnoreCase)); + } + + Assert.DoesNotContain("Parse", publicMembers); + Assert.DoesNotContain("Execute", publicMembers); + } + + private static SourceOnboardingRegistryEntry ValidEntry( + string sourceId, + string? sourceFamily = null, + SourceOnboardingDiscoveryStrategy discoveryStrategy = SourceOnboardingDiscoveryStrategy.SourceSpecificDiscovery, + SourceOnboardingUpdateCadence updateCadence = SourceOnboardingUpdateCadence.Periodic, + IReadOnlyList? documents = null) => + new( + sourceId, + sourceFamily ?? sourceId, + "New Registry Source", + documents ?? [ValidDocument($"{sourceId}_document")], + discoveryStrategy, + new SourceOnboardingParserCapability( + new ParserKey($"{sourceId}_parser"), + ParserSourceFormat.DiscoveryReference, + supportsParserExecution: false, + "metadata only"), + new SourceOnboardingValidationExpectations( + ["document_id", "source_reference"], + checksumRequired: true, + schemaValidationRequired: true, + "shape validation only"), + updateCadence, + new SourceOnboardingRuntimeSafety( + allowsNetworkCalls: false, + allowsFileReads: false, + allowsDatabaseWrites: false, + requiresCredentials: false, + "contract metadata only")); + + private static SourceOnboardingDocument ValidDocument(string documentId) => + new( + documentId, + "Declared document", + $"discovery://{documentId}", + "discovery"); + + private static JsonDocument LoadParityExpectations() => + JsonDocument.Parse(File.ReadAllText(Path.Combine(ParityFixtureDirectory(), "source_onboarding_registry_expectations.json"))); + + private static IReadOnlyList JsonStringArray(JsonElement array) => + array.EnumerateArray().Select(item => item.GetString() ?? string.Empty).ToArray(); + + private static string DiscoveryStrategyWireName(SourceOnboardingDiscoveryStrategy strategy) => + strategy switch + { + SourceOnboardingDiscoveryStrategy.DeclaredReference => "declared_reference", + SourceOnboardingDiscoveryStrategy.SourceSpecificDiscovery => "source_specific_discovery", + _ => throw new ArgumentOutOfRangeException(nameof(strategy), strategy, "Unknown discovery strategy."), + }; + + private static string UpdateCadenceWireName(SourceOnboardingUpdateCadence cadence) => + cadence switch + { + SourceOnboardingUpdateCadence.Unknown => "unknown", + SourceOnboardingUpdateCadence.Annual => "annual", + SourceOnboardingUpdateCadence.Periodic => "periodic", + _ => throw new ArgumentOutOfRangeException(nameof(cadence), cadence, "Unknown update cadence."), + }; + + private static SourceFamily ParseSourceFamily(string sourceFamily) => + sourceFamily switch + { + "ghg_protocol" => SourceFamily.GhgProtocol, + "defra_desnz" => SourceFamily.DefraDesnz, + "ipcc_efdb" => SourceFamily.IpccEfdb, + _ => throw new ArgumentOutOfRangeException(nameof(sourceFamily), sourceFamily, "Unknown source family."), + }; + + private static ParserKey ExpectedParserKey(string sourceFamily) => + sourceFamily switch + { + "ghg_protocol" => ParserSelectionRegistry.GetParserKey(SourceFamily.GhgProtocol), + "defra_desnz" => ParserSelectionRegistry.GetParserKey(SourceFamily.DefraDesnz), + "ipcc_efdb" => ParserSelectionRegistry.GetParserKey(SourceFamily.IpccEfdb), + _ => throw new ArgumentOutOfRangeException(nameof(sourceFamily), sourceFamily, "Unknown source family."), + }; + + private static string ParityFixtureDirectory() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var fixtureDirectory = Path.Combine(directory.FullName, "tests", "fixtures", "parity"); + if (Directory.Exists(fixtureDirectory)) + { + return fixtureDirectory; + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate parity fixture directory."); + } +} diff --git a/tests/fixtures/parity/defra_desnz_normalized_output_expectations.json b/tests/fixtures/parity/defra_desnz_normalized_output_expectations.json new file mode 100644 index 00000000..9128533b --- /dev/null +++ b/tests/fixtures/parity/defra_desnz_normalized_output_expectations.json @@ -0,0 +1,104 @@ +{ + "header": [ + "source_year", + "source_version", + "category", + "subcategory", + "activity", + "factor_id", + "factor_name", + "factor_value", + "unit", + "greenhouse_gas", + "provenance" + ], + "sample_fixture": "tests/fixtures/source_documents/defra_desnz/defra_desnz_normalized_factors.csv", + "sample_status": { + "python": "success", + "dotnet": "Completed" + }, + "sample_issue_codes": [], + "sample_rows": [ + { + "row_identifier": "defra_desnz_2024_conversion-factors-2024_DEFRA-2024-ELEC_row_2", + "source_row_number": 2, + "reporting_year": 2024, + "fields": [ + ["source_family", "defra_desnz"], + ["source_year", "2024"], + ["source_version", "conversion-factors-2024"], + ["factor_id", "DEFRA-2024-ELEC"], + ["factor_name", "Electricity generated"], + ["factor_value", "0.20705"], + ["unit", "kWh"], + ["category", "Energy"], + ["subcategory", "Electricity"], + ["activity", "Generated"], + ["greenhouse_gas", "CO2e"], + ["provenance_artifact_reference", "tests/fixtures/source_documents/defra_desnz/defra_desnz_normalized_factors.csv"], + ["provenance_checksum_algorithm", "sha256"], + ["provenance_checksum_value", "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"], + ["provenance_row_number", "2"], + ["provenance", "worksheet:UK electricity row 10"], + ["source_family_master_id", "defra_master_2024_conversion-factors-2024_DEFRA-2024-ELEC"], + ["source_family_detail_id", "defra_detail_2024_conversion-factors-2024_DEFRA-2024-ELEC"], + ["master_external_key", "2024:conversion-factors-2024:DEFRA-2024-ELEC"], + ["detail_external_key", "DEFRA-2024-ELEC:kWh:CO2e"] + ] + }, + { + "row_identifier": "defra_desnz_2024_conversion-factors-2024_DEFRA-2024-VAN_row_3", + "source_row_number": 3, + "reporting_year": 2024, + "fields": [ + ["source_family", "defra_desnz"], + ["source_year", "2024"], + ["source_version", "conversion-factors-2024"], + ["factor_id", "DEFRA-2024-VAN"], + ["factor_name", "Average van distance"], + ["factor_value", "0.17031"], + ["unit", "km"], + ["category", "Transport"], + ["subcategory", "Vans"], + ["activity", "Average van"], + ["greenhouse_gas", "CO2e"], + ["provenance_artifact_reference", "tests/fixtures/source_documents/defra_desnz/defra_desnz_normalized_factors.csv"], + ["provenance_checksum_algorithm", "sha256"], + ["provenance_checksum_value", "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"], + ["provenance_row_number", "3"], + ["provenance", "worksheet:Business travel land row 24"], + ["source_family_master_id", "defra_master_2024_conversion-factors-2024_DEFRA-2024-VAN"], + ["source_family_detail_id", "defra_detail_2024_conversion-factors-2024_DEFRA-2024-VAN"], + ["master_external_key", "2024:conversion-factors-2024:DEFRA-2024-VAN"], + ["detail_external_key", "DEFRA-2024-VAN:km:CO2e"] + ] + } + ], + "malformed_fixture": "tests/fixtures/source_documents/defra_desnz/defra_desnz_malformed_factors.csv", + "malformed_status": { + "python": "failed", + "dotnet": "Failed" + }, + "malformed_issues": [ + { + "code": "DEFRA_DESNZ_CONTENT_INVALID_FACTOR_VALUE", + "field_key": "factor_value", + "python_location": "row[2].factor_value", + "source_row_number": 2 + }, + { + "code": "DEFRA_DESNZ_CONTENT_MISSING_REQUIRED_FIELD", + "field_key": "unit", + "python_location": "row[3].unit", + "source_row_number": 3 + } + ], + "empty_status": { + "python": "no_records", + "dotnet": "Completed", + "intentional_difference": "Python exposes a no_records status for valid DEFRA/DESNZ content with no parsed rows; .NET keeps the adapter run completed and carries DEFRA_DESNZ_CONTENT_NO_RECORDS as a warning." + }, + "empty_issue_codes": [ + "DEFRA_DESNZ_CONTENT_NO_RECORDS" + ] +} diff --git a/tests/fixtures/parity/ghg_protocol_normalized_output_expectations.json b/tests/fixtures/parity/ghg_protocol_normalized_output_expectations.json new file mode 100644 index 00000000..439bf85c --- /dev/null +++ b/tests/fixtures/parity/ghg_protocol_normalized_output_expectations.json @@ -0,0 +1,108 @@ +{ + "header": [ + "record_type", + "source_year", + "source_version", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", + "subcategory", + "scope", + "gas", + "provenance_note" + ], + "sample_fixture": "tests/fixtures/source_documents/ghg_protocol/ghg_protocol_sample_factors.csv", + "sample_status": { + "python": "success", + "dotnet": "Completed" + }, + "sample_issue_codes": [ + "GHG_PROTOCOL_CONTENT_UNSUPPORTED_ROW_SKIPPED" + ], + "sample_rows": [ + { + "row_identifier": "ghg_protocol_2024_v1_GHG-ELEC-001_row_2", + "source_row_number": 2, + "reporting_year": 2024, + "fields": [ + ["source_family", "ghg_protocol"], + ["source_year", "2024"], + ["source_version", "v1"], + ["factor_id", "GHG-ELEC-001"], + ["factor_name", "Grid electricity"], + ["factor_value", "0.233"], + ["unit", "kg CO2e/kWh"], + ["category", "Stationary combustion"], + ["subcategory", "Electricity"], + ["scope", "Scope 2"], + ["gas", "CO2e"], + ["provenance_artifact_reference", "tests/fixtures/source_documents/ghg_protocol/ghg_protocol_sample_factors.csv"], + ["provenance_checksum_algorithm", "sha256"], + ["provenance_checksum_value", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"], + ["provenance_row_number", "2"], + ["provenance_note", "fixture row 1"], + ["source_family_master_id", "ghg_master_2024_v1_GHG-ELEC-001"], + ["source_family_detail_id", "ghg_detail_2024_v1_GHG-ELEC-001"], + ["master_external_key", "2024:v1:GHG-ELEC-001"], + ["detail_external_key", "GHG-ELEC-001:kg CO2e/kWh"] + ] + }, + { + "row_identifier": "ghg_protocol_2024_v1_GHG-GAS-001_row_4", + "source_row_number": 4, + "reporting_year": 2024, + "fields": [ + ["source_family", "ghg_protocol"], + ["source_year", "2024"], + ["source_version", "v1"], + ["factor_id", "GHG-GAS-001"], + ["factor_name", "Natural gas"], + ["factor_value", "0.184"], + ["unit", "kg CO2e/kWh"], + ["category", "Stationary combustion"], + ["subcategory", "Natural gas"], + ["scope", "Scope 1"], + ["gas", "CO2e"], + ["provenance_artifact_reference", "tests/fixtures/source_documents/ghg_protocol/ghg_protocol_sample_factors.csv"], + ["provenance_checksum_algorithm", "sha256"], + ["provenance_checksum_value", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"], + ["provenance_row_number", "4"], + ["provenance_note", "fixture row 3"], + ["source_family_master_id", "ghg_master_2024_v1_GHG-GAS-001"], + ["source_family_detail_id", "ghg_detail_2024_v1_GHG-GAS-001"], + ["master_external_key", "2024:v1:GHG-GAS-001"], + ["detail_external_key", "GHG-GAS-001:kg CO2e/kWh"] + ] + } + ], + "malformed_fixture": "tests/fixtures/source_documents/ghg_protocol/ghg_protocol_malformed_factors.csv", + "malformed_status": { + "python": "failed", + "dotnet": "Failed" + }, + "malformed_issues": [ + { + "code": "GHG_PROTOCOL_CONTENT_INVALID_FACTOR_VALUE", + "field_key": "factor_value", + "python_location": "row[2].factor_value", + "source_row_number": 2 + }, + { + "code": "GHG_PROTOCOL_CONTENT_INVALID_SOURCE_YEAR", + "field_key": "source_year", + "python_location": "row[3].source_year", + "source_row_number": 3 + } + ], + "unsupported_only_status": { + "python": "no_records", + "dotnet": "Completed", + "intentional_difference": "Python exposes a no_records status when all content rows are skipped; .NET keeps the adapter run completed and carries GHG_PROTOCOL_CONTENT_NO_RECORDS as a warning." + }, + "unsupported_only_issue_codes": [ + "GHG_PROTOCOL_CONTENT_UNSUPPORTED_ROW_SKIPPED", + "GHG_PROTOCOL_CONTENT_NO_RECORDS" + ] +} diff --git a/tests/fixtures/parity/ipcc_efdb_normalized_output_expectations.json b/tests/fixtures/parity/ipcc_efdb_normalized_output_expectations.json new file mode 100644 index 00000000..31f3e521 --- /dev/null +++ b/tests/fixtures/parity/ipcc_efdb_normalized_output_expectations.json @@ -0,0 +1,120 @@ +{ + "header": [ + "record_type", + "source_year", + "source_version", + "factor_id", + "factor_name", + "factor_value", + "unit", + "category", + "subcategory", + "ipcc_sector", + "gas", + "region", + "technology", + "provenance" + ], + "sample_fixture": "tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_sample_factors.csv", + "sample_status": { + "python": "success", + "dotnet": "Completed" + }, + "sample_issue_codes": [ + "IPCC_EFDB_CONTENT_UNSUPPORTED_ROW_SKIPPED" + ], + "sample_rows": [ + { + "row_identifier": "ipcc_efdb_2006_efdb-v2024_IPCC-ENERGY-CO2_row_2", + "source_row_number": 2, + "reporting_year": 2006, + "fields": [ + ["source_family", "ipcc_efdb"], + ["source_year", "2006"], + ["source_version", "efdb-v2024"], + ["factor_id", "IPCC-ENERGY-CO2"], + ["factor_name", "Stationary combustion CO2"], + ["factor_value", "56.1"], + ["unit", "t CO2/TJ"], + ["category", "Energy"], + ["subcategory", "Stationary combustion"], + ["ipcc_sector", "1A"], + ["gas", "CO2"], + ["region", "Global"], + ["technology", "Default"], + ["provenance_artifact_reference", "tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_sample_factors.csv"], + ["provenance_checksum_algorithm", "sha256"], + ["provenance_checksum_value", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"], + ["provenance_row_number", "2"], + ["provenance", "worksheet:EFDB row 12"], + ["source_family_master_id", "ipcc_master_2006_efdb-v2024_IPCC-ENERGY-CO2"], + ["source_family_detail_id", "ipcc_detail_2006_efdb-v2024_IPCC-ENERGY-CO2"], + ["master_external_key", "2006:efdb-v2024:IPCC-ENERGY-CO2"], + ["detail_external_key", "IPCC-ENERGY-CO2:t CO2/TJ:CO2:1A"] + ] + }, + { + "row_identifier": "ipcc_efdb_2019_efdb-v2024_IPCC-WASTE-CH4_row_4", + "source_row_number": 4, + "reporting_year": 2019, + "fields": [ + ["source_family", "ipcc_efdb"], + ["source_year", "2019"], + ["source_version", "efdb-v2024"], + ["factor_id", "IPCC-WASTE-CH4"], + ["factor_name", "Wastewater methane"], + ["factor_value", "0.25"], + ["unit", "kg CH4/kg COD"], + ["category", "Waste"], + ["subcategory", "Wastewater"], + ["ipcc_sector", "4D"], + ["gas", "CH4"], + ["region", "Global"], + ["technology", "Default"], + ["provenance_artifact_reference", "tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_sample_factors.csv"], + ["provenance_checksum_algorithm", "sha256"], + ["provenance_checksum_value", "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"], + ["provenance_row_number", "4"], + ["provenance", "worksheet:EFDB row 44"], + ["source_family_master_id", "ipcc_master_2019_efdb-v2024_IPCC-WASTE-CH4"], + ["source_family_detail_id", "ipcc_detail_2019_efdb-v2024_IPCC-WASTE-CH4"], + ["master_external_key", "2019:efdb-v2024:IPCC-WASTE-CH4"], + ["detail_external_key", "IPCC-WASTE-CH4:kg CH4/kg COD:CH4:4D"] + ] + } + ], + "malformed_fixture": "tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_malformed_factors.csv", + "malformed_status": { + "python": "failed", + "dotnet": "Failed" + }, + "malformed_issues": [ + { + "code": "IPCC_EFDB_CONTENT_INVALID_SOURCE_YEAR", + "field_key": "source_year", + "python_location": "row[2].source_year", + "source_row_number": 2 + }, + { + "code": "IPCC_EFDB_CONTENT_INVALID_FACTOR_VALUE", + "field_key": "factor_value", + "python_location": "row[3].factor_value", + "source_row_number": 3 + }, + { + "code": "IPCC_EFDB_CONTENT_MISSING_REQUIRED_FIELD", + "field_key": "factor_id", + "python_location": "row[4].factor_id", + "source_row_number": 4 + } + ], + "unsupported_only_status": { + "python": "no_records", + "dotnet": "Completed", + "intentional_difference": "Python exposes a no_records status when all IPCC EFDB content rows are skipped; .NET keeps the adapter run completed and carries IPCC_EFDB_CONTENT_NO_RECORDS as a warning." + }, + "unsupported_only_issue_codes": [ + "IPCC_EFDB_CONTENT_UNSUPPORTED_ROW_SKIPPED", + "IPCC_EFDB_CONTENT_NO_RECORDS" + ] +} diff --git a/tests/fixtures/parity/ipcc_source_download_execution_expectations.json b/tests/fixtures/parity/ipcc_source_download_execution_expectations.json new file mode 100644 index 00000000..8f53aa6d --- /dev/null +++ b/tests/fixtures/parity/ipcc_source_download_execution_expectations.json @@ -0,0 +1,68 @@ +{ + "source_family": "ipcc_efdb", + "source_key": "ipcc_efdb", + "candidate_id": "ipcc_source_discovery_candidate_001_ipcc_efdb", + "candidate_title": "IPCC EFDB", + "source_reference_uri": "mock://ipcc_efdb/efdb.xlsx", + "artifact_kind": "xlsx", + "target_relative_path": "ipcc/efdb.xlsx", + "content_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "extension": ".xlsx", + "document_year": 2006, + "reporting_year": 2024, + "version_label": "pt053_fixture_download", + "successful_download": { + "payload_text": "deterministic ipcc source bytes", + "status": "downloaded", + "downloaded": true, + "already_known": false, + "reused_existing": false, + "checksum_sha256": "ab52d433d6f9e29ba16eb12a7d77cfb67c6778c61368ac52efbc40ff983fd5d1", + "size_bytes": 31, + "final_uri": "mock://ipcc_efdb/final.xlsx", + "issue_codes": [] + }, + "existing_known_document": { + "payload_text": "existing ipcc bytes", + "transport_called": false, + "checksum_sha256": "360694d307907c42592d1af4b00f15120568a9b2ed538f857dce1d45e239474d", + "size_bytes": 19, + "issue_codes": [], + "python": { + "status": "downloaded", + "downloaded": true, + "reused_existing": true + }, + "dotnet": { + "status": "already_known", + "downloaded": false, + "already_known": true + }, + "intentional_difference": ".NET exposes an explicit already_known status; Python keeps the existing public downloaded status and marks the artifact reused_existing=true." + }, + "checksum_mismatch": { + "payload_text": "unexpected", + "expected_checksum_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "status": "failed", + "artifact": null, + "issue_codes": [ + "IPCC_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH" + ] + }, + "default_candidate_blocked": { + "status": "blocked", + "artifact": null, + "issue_codes": [ + "IPCC_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE", + "IPCC_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE" + ] + }, + "blank_response_metadata": { + "status": "failed", + "artifact": null, + "issue_codes": [ + "IPCC_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "IPCC_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI" + ] + } +} diff --git a/tests/fixtures/parity/parsed_factor_persistence_writer_expectations.json b/tests/fixtures/parity/parsed_factor_persistence_writer_expectations.json new file mode 100644 index 00000000..9f94d5dc --- /dev/null +++ b/tests/fixtures/parity/parsed_factor_persistence_writer_expectations.json @@ -0,0 +1,49 @@ +{ + "normalized_row": { + "source_family": "ghg_protocol", + "source_key": "ghg_protocol", + "parser_key": "ghg_protocol_phase1_parser", + "artifact_reference": "artifact://ghg/factors.csv", + "row_id": "ghg-row-001", + "source_row_number": 2, + "reporting_year": 2024, + "fields": [ + ["source_year", "2024"], + ["source_version", "fixture-v1"], + ["factor_id", "GHG-001"], + ["factor_value", "1.25"], + ["factor_unit", "kgco2e"], + ["gas", "CO2e"], + ["provenance_artifact_reference", "artifact://ghg/factors.csv"], + ["provenance_checksum_value", "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"] + ] + }, + "expected_command": { + "status": "declared", + "master_count": 1, + "detail_count": 1, + "skipped_duplicate_count": 0, + "master_record": { + "source_family": "ghg", + "source_family_master_id": "ghg_master_2fc8ecfc2a09b71e", + "source_document_id": "source_document_d979c281dce2fa107de70d92", + "master_external_key": "2024:fixture-v1:artifact://ghg/factors.csv:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "lifecycle_status": "active", + "record_checksum_sha256": "25d5fa4ed709861fb9eb7823c1ad1323ccb81690b54828aab891d9e2ac63ae66", + "created_at": "dry_run_timestamp_unavailable", + "updated_at": "dry_run_timestamp_unavailable" + }, + "detail_record": { + "source_family": "ghg", + "source_family_detail_id": "ghg_detail_321df2543171bdb9", + "source_family_master_id": "ghg_master_2fc8ecfc2a09b71e", + "detail_external_key": "GHG-001:kgco2e:CO2e", + "factor_value": "1.25", + "factor_unit": "kgco2e", + "lifecycle_status": "active", + "record_checksum_sha256": "c33fe99c57afcfff4692950c3412b7aaae13e37e783f2c7dfe3136bf028f2809", + "created_at": "dry_run_timestamp_unavailable", + "updated_at": "dry_run_timestamp_unavailable" + } + } +} diff --git a/tests/fixtures/parity/phase1_operational_diagnostics_expectations.json b/tests/fixtures/parity/phase1_operational_diagnostics_expectations.json new file mode 100644 index 00000000..fe552e7e --- /dev/null +++ b/tests/fixtures/parity/phase1_operational_diagnostics_expectations.json @@ -0,0 +1,71 @@ +{ + "request_keys": [ + "correlation_id", + "execution_mode", + "max_degree_of_parallelism", + "run_id", + "source_families" + ], + "family_keys": [ + "correlation_id", + "documents", + "failures", + "parser", + "persistence", + "run_id", + "source_family", + "source_key", + "status" + ], + "document_keys": [ + "checksum_sha256", + "document_id", + "source_family", + "source_key" + ], + "parser_keys": [ + "accepted_row_count", + "failure_count", + "result_status", + "run_id", + "validation_issue_count" + ], + "failure_keys": [ + "code", + "field_name", + "message", + "severity", + "source_family", + "source_key", + "stage" + ], + "summary_keys": [ + "completed_family_count", + "failed_family_count", + "failure_count", + "parsed_factor_row_count", + "parser_run_count", + "persisted_detail_count", + "persisted_master_count", + "persisted_parser_run_count", + "persisted_source_document_count", + "persisted_source_run_count", + "requested_family_count", + "source_artifact_count", + "source_candidate_count" + ], + "orchestrator_event_names": [ + "phase1_ingestion_orchestrator_started", + "phase1_source_family_completed", + "phase1_ingestion_orchestrator_completed" + ], + "service_host_event_names": [ + "phase1_service_host_starting", + "phase1_service_host_started", + "phase1_service_host_scheduled_run_started", + "phase1_service_host_scheduled_run_completed", + "phase1_service_host_scheduled_run_skipped" + ], + "redacted": "", + "intentional_status_difference": ".NET Phase1IngestionFamilyRunStatus is intentionally coarser than Python; shared diagnostics preserve stage and failure code for troubleshooting." +} diff --git a/tests/fixtures/parity/source_onboarding_registry_expectations.json b/tests/fixtures/parity/source_onboarding_registry_expectations.json new file mode 100644 index 00000000..099ad12e --- /dev/null +++ b/tests/fixtures/parity/source_onboarding_registry_expectations.json @@ -0,0 +1,135 @@ +{ + "phase2_source_families": [ + "ghg_protocol", + "defra_desnz", + "ipcc_efdb" + ], + "entries": [ + { + "source_id": "ghg_protocol", + "source_family": "ghg_protocol", + "display_name": "GHG Protocol", + "documents": [ + { + "document_id": "ghg_protocol_declared_reference", + "display_name": "GHG Protocol declared reference", + "source_reference": "discovery://ghg_protocol/onboarding", + "expected_format": "discovery", + "required": true + } + ], + "discovery_strategy": "declared_reference", + "parser_capability": { + "parser_key": "ghg_protocol_phase1_parser", + "parser_source_format": "discovery_reference", + "supports_parser_execution": false, + "capability_notes": "Registry metadata only; parser execution is outside this onboarding contract." + }, + "validation_expectations": { + "required_document_fields": [ + "document_id", + "display_name", + "source_reference", + "expected_format" + ], + "checksum_required": false, + "schema_validation_required": false, + "validation_notes": "Declared discovery references are validated for contract shape only." + }, + "update_cadence": "unknown", + "runtime_safety": { + "allows_network_calls": false, + "allows_file_reads": false, + "allows_database_writes": false, + "requires_credentials": false, + "safety_notes": "Default onboarding registry is runtime-passive and local-only." + }, + "enabled": true + }, + { + "source_id": "defra_desnz", + "source_family": "defra_desnz", + "display_name": "DEFRA/DESNZ", + "documents": [ + { + "document_id": "defra_desnz_declared_reference", + "display_name": "DEFRA/DESNZ declared reference", + "source_reference": "discovery://defra_desnz/onboarding", + "expected_format": "discovery", + "required": true + } + ], + "discovery_strategy": "declared_reference", + "parser_capability": { + "parser_key": "defra_desnz_phase1_parser", + "parser_source_format": "discovery_reference", + "supports_parser_execution": false, + "capability_notes": "Registry metadata only; parser execution is outside this onboarding contract." + }, + "validation_expectations": { + "required_document_fields": [ + "document_id", + "display_name", + "source_reference", + "expected_format" + ], + "checksum_required": false, + "schema_validation_required": false, + "validation_notes": "Declared discovery references are validated for contract shape only." + }, + "update_cadence": "unknown", + "runtime_safety": { + "allows_network_calls": false, + "allows_file_reads": false, + "allows_database_writes": false, + "requires_credentials": false, + "safety_notes": "Default onboarding registry is runtime-passive and local-only." + }, + "enabled": true + }, + { + "source_id": "ipcc_efdb", + "source_family": "ipcc_efdb", + "display_name": "IPCC EFDB", + "documents": [ + { + "document_id": "ipcc_efdb_declared_reference", + "display_name": "IPCC EFDB declared reference", + "source_reference": "discovery://ipcc_efdb/onboarding", + "expected_format": "discovery", + "required": true + } + ], + "discovery_strategy": "declared_reference", + "parser_capability": { + "parser_key": "ipcc_efdb_phase1_parser", + "parser_source_format": "discovery_reference", + "supports_parser_execution": false, + "capability_notes": "Registry metadata only; parser execution is outside this onboarding contract." + }, + "validation_expectations": { + "required_document_fields": [ + "document_id", + "display_name", + "source_reference", + "expected_format" + ], + "checksum_required": false, + "schema_validation_required": false, + "validation_notes": "Declared discovery references are validated for contract shape only." + }, + "update_cadence": "unknown", + "runtime_safety": { + "allows_network_calls": false, + "allows_file_reads": false, + "allows_database_writes": false, + "requires_credentials": false, + "safety_notes": "Default onboarding registry is runtime-passive and local-only." + }, + "enabled": true + } + ], + "accepted_asymmetries": [ + "Python validation raises TypeError or ValueError for invalid registries; .NET validation returns ContractValidationResult errors and lookup helpers throw ArgumentException when an invalid custom registry is supplied." + ] +} diff --git a/tests/fixtures/postgresql_phase1_schema_ddl.sql b/tests/fixtures/postgresql_phase1_schema_ddl.sql index 2525ab8e..2cc1a135 100644 --- a/tests/fixtures/postgresql_phase1_schema_ddl.sql +++ b/tests/fixtures/postgresql_phase1_schema_ddl.sql @@ -1,4 +1,4 @@ -CREATE TABLE ingestion_runs ( +CREATE TABLE IF NOT EXISTS ingestion_runs ( ingestion_run_id uuid NOT NULL, run_status text NOT NULL, created_at timestamp with time zone NOT NULL, @@ -6,9 +6,9 @@ CREATE TABLE ingestion_runs ( CONSTRAINT pk_ingestion_runs PRIMARY KEY (ingestion_run_id) ); -CREATE INDEX idx_ingestion_runs_run_status ON ingestion_runs (run_status); +CREATE INDEX IF NOT EXISTS idx_ingestion_runs_run_status ON ingestion_runs (run_status); -CREATE TABLE source_documents ( +CREATE TABLE IF NOT EXISTS source_documents ( source_document_id uuid NOT NULL, ingestion_run_id uuid NOT NULL, source_family text NOT NULL, @@ -23,9 +23,9 @@ CREATE TABLE source_documents ( CONSTRAINT fk_source_documents_ingestion_run_id FOREIGN KEY (ingestion_run_id) REFERENCES ingestion_runs (ingestion_run_id) ); -CREATE INDEX idx_source_documents_ingestion_run_id ON source_documents (ingestion_run_id); +CREATE INDEX IF NOT EXISTS idx_source_documents_ingestion_run_id ON source_documents (ingestion_run_id); -CREATE TABLE parser_runs ( +CREATE TABLE IF NOT EXISTS parser_runs ( parser_run_id uuid NOT NULL, source_document_id uuid NOT NULL, parser_status text NOT NULL, @@ -36,9 +36,9 @@ CREATE TABLE parser_runs ( CONSTRAINT fk_parser_runs_source_document_id FOREIGN KEY (source_document_id) REFERENCES source_documents (source_document_id) ); -CREATE INDEX idx_parser_runs_source_document_id ON parser_runs (source_document_id); +CREATE INDEX IF NOT EXISTS idx_parser_runs_source_document_id ON parser_runs (source_document_id); -CREATE TABLE schema_bootstrap_states ( +CREATE TABLE IF NOT EXISTS schema_bootstrap_states ( schema_bootstrap_state_id uuid NOT NULL, schema_contract_version text NOT NULL, bootstrap_status text NOT NULL, @@ -48,29 +48,160 @@ CREATE TABLE schema_bootstrap_states ( CONSTRAINT uq_schema_bootstrap_states_contract_version UNIQUE (schema_contract_version) ); -CREATE TABLE ghg_emission_factor_masters ( +CREATE TABLE IF NOT EXISTS parser_ingestion_runs ( + run_id text NOT NULL, + started_at timestamp with time zone NOT NULL, + finished_at timestamp with time zone, + status text NOT NULL, + trigger_type text DEFAULT 'operator' NOT NULL, + config_hash text, + enabled_source_families jsonb DEFAULT '[]'::jsonb NOT NULL, + initial_year integer, + cycle_count integer, + total_parsed_rows integer DEFAULT 0 NOT NULL, + total_inserted_count integer DEFAULT 0 NOT NULL, + total_skipped_duplicate_count integer DEFAULT 0 NOT NULL, + failure_count integer DEFAULT 0 NOT NULL, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT pk_parser_ingestion_runs PRIMARY KEY (run_id) +); + +CREATE TABLE IF NOT EXISTS parser_ingestion_source_results ( + parser_ingestion_source_result_id uuid NOT NULL, + run_id text NOT NULL, + source_family text NOT NULL, + target_year integer, + latest_year integer, + status text NOT NULL, + download_status text, + parse_status text, + validation_status text, + insert_status text, + parsed_rows integer DEFAULT 0 NOT NULL, + master_inserted integer DEFAULT 0 NOT NULL, + master_skipped integer DEFAULT 0 NOT NULL, + detail_inserted integer DEFAULT 0 NOT NULL, + detail_skipped integer DEFAULT 0 NOT NULL, + issue_count integer DEFAULT 0 NOT NULL, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT pk_parser_ingestion_source_results PRIMARY KEY (parser_ingestion_source_result_id), + CONSTRAINT uq_parser_ingestion_source_results_run_family_year UNIQUE (run_id, source_family, target_year), + CONSTRAINT fk_parser_ingestion_source_results_run_id FOREIGN KEY (run_id) REFERENCES parser_ingestion_runs (run_id) +); + +CREATE TABLE IF NOT EXISTS parser_ingestion_issues ( + parser_ingestion_issue_id uuid NOT NULL, + run_id text NOT NULL, + source_family text, + target_year integer, + stage text NOT NULL, + code text NOT NULL, + severity text DEFAULT 'error' NOT NULL, + field_name text, + message text NOT NULL, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT pk_parser_ingestion_issues PRIMARY KEY (parser_ingestion_issue_id), + CONSTRAINT fk_parser_ingestion_issues_run_id FOREIGN KEY (run_id) REFERENCES parser_ingestion_runs (run_id) +); + +CREATE INDEX IF NOT EXISTS idx_parser_ingestion_issues_run_id ON parser_ingestion_issues (run_id); + +CREATE INDEX IF NOT EXISTS idx_parser_ingestion_issues_family_year ON parser_ingestion_issues (source_family, target_year); + +CREATE INDEX IF NOT EXISTS idx_parser_ingestion_issues_code ON parser_ingestion_issues (code); + +CREATE TABLE IF NOT EXISTS source_family_year_states ( + source_family_year_state_id uuid NOT NULL, + source_family text NOT NULL, + ingested_year integer NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + CONSTRAINT pk_source_family_year_states PRIMARY KEY (source_family_year_state_id), + CONSTRAINT uq_source_family_year_states_family_year UNIQUE (source_family, ingested_year) +); + +CREATE INDEX IF NOT EXISTS idx_source_family_year_states_family_year ON source_family_year_states (source_family, ingested_year); + +CREATE TABLE IF NOT EXISTS normalized_factor_records ( + normalized_factor_record_id text NOT NULL, + idempotency_key_sha256 text NOT NULL, + source_family text NOT NULL, + source_id text NOT NULL, + source_year integer, + source_version text, + record_id text NOT NULL, + source_row_number integer, + source_document_reference text, + source_artifact_reference text, + source_checksum_sha256 text, + factor_id text, + factor_name text, + factor_value numeric NOT NULL, + factor_unit text NOT NULL, + validation_status text NOT NULL, + run_id text, + parser_key text NOT NULL, + metadata jsonb NOT NULL, + normalized_fields jsonb NOT NULL, + warnings jsonb NOT NULL, + errors jsonb NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + CONSTRAINT pk_normalized_factor_records PRIMARY KEY (normalized_factor_record_id), + CONSTRAINT uq_normalized_factor_records_idempotency_key UNIQUE (idempotency_key_sha256) +); + +CREATE INDEX IF NOT EXISTS idx_normalized_factor_records_source_year ON normalized_factor_records (source_family, source_id, source_year); + +CREATE TABLE IF NOT EXISTS ghg_emission_factor_masters ( ghg_emission_factor_master_id uuid NOT NULL, + source_family text NOT NULL, + source_year integer NOT NULL, + source_version text NOT NULL, + source_release text, source_document_id uuid NOT NULL, + ingestion_run_id uuid, + run_id text, master_external_key text NOT NULL, - lifecycle_status text NOT NULL, + status text NOT NULL, + artifact_reference text, + artifact_checksum_sha256 text, + archive_reference text, + archive_checksum_sha256 text, effective_from timestamp with time zone, effective_to timestamp with time zone, record_checksum_sha256 text NOT NULL, + metadata jsonb NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, CONSTRAINT pk_ghg_emission_factor_masters PRIMARY KEY (ghg_emission_factor_master_id), - CONSTRAINT uq_ghg_emission_factor_masters_external_key UNIQUE (master_external_key), - CONSTRAINT fk_ghg_emission_factor_masters_source_document_id FOREIGN KEY (source_document_id) REFERENCES source_documents (source_document_id) + CONSTRAINT uq_ghg_emission_factor_masters_family_year_version_key UNIQUE (source_family, source_year, source_version, master_external_key), + CONSTRAINT fk_ghg_emission_factor_masters_source_document_id FOREIGN KEY (source_document_id) REFERENCES source_documents (source_document_id), + CONSTRAINT fk_ghg_emission_factor_masters_ingestion_run_id FOREIGN KEY (ingestion_run_id) REFERENCES ingestion_runs (ingestion_run_id) ); -CREATE TABLE ghg_emission_factor_details ( +CREATE INDEX IF NOT EXISTS idx_ghg_emission_factor_masters_source_year ON ghg_emission_factor_masters (source_family, source_year, source_version); + +CREATE INDEX IF NOT EXISTS idx_ghg_emission_factor_masters_ingestion_run_id ON ghg_emission_factor_masters (ingestion_run_id); + +CREATE TABLE IF NOT EXISTS ghg_emission_factor_details ( ghg_emission_factor_detail_id uuid NOT NULL, ghg_emission_factor_master_id uuid NOT NULL, detail_external_key text NOT NULL, + source_row_number integer, + factor_id text, + factor_name text, factor_value numeric NOT NULL, factor_unit text NOT NULL, - lifecycle_status text NOT NULL, + status text NOT NULL, record_checksum_sha256 text NOT NULL, + raw_fields jsonb NOT NULL, + normalized_fields jsonb NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, CONSTRAINT pk_ghg_emission_factor_details PRIMARY KEY (ghg_emission_factor_detail_id), @@ -78,31 +209,52 @@ CREATE TABLE ghg_emission_factor_details ( CONSTRAINT fk_ghg_emission_factor_details_ghg_emission_factor_master_id FOREIGN KEY (ghg_emission_factor_master_id) REFERENCES ghg_emission_factor_masters (ghg_emission_factor_master_id) ); -CREATE INDEX idx_ghg_emission_factor_details_ghg_emission_factor_master_id ON ghg_emission_factor_details (ghg_emission_factor_master_id); +CREATE INDEX IF NOT EXISTS idx_ghg_emission_factor_details_ghg_emission_factor_master_id ON ghg_emission_factor_details (ghg_emission_factor_master_id); -CREATE TABLE defra_emission_factor_masters ( +CREATE TABLE IF NOT EXISTS defra_emission_factor_masters ( defra_emission_factor_master_id uuid NOT NULL, + source_family text NOT NULL, + source_year integer NOT NULL, + source_version text NOT NULL, + source_release text, source_document_id uuid NOT NULL, + ingestion_run_id uuid, + run_id text, master_external_key text NOT NULL, - lifecycle_status text NOT NULL, + status text NOT NULL, + artifact_reference text, + artifact_checksum_sha256 text, + archive_reference text, + archive_checksum_sha256 text, effective_from timestamp with time zone, effective_to timestamp with time zone, record_checksum_sha256 text NOT NULL, + metadata jsonb NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, CONSTRAINT pk_defra_emission_factor_masters PRIMARY KEY (defra_emission_factor_master_id), - CONSTRAINT uq_defra_emission_factor_masters_external_key UNIQUE (master_external_key), - CONSTRAINT fk_defra_emission_factor_masters_source_document_id FOREIGN KEY (source_document_id) REFERENCES source_documents (source_document_id) + CONSTRAINT uq_defra_emission_factor_masters_family_year_version_key UNIQUE (source_family, source_year, source_version, master_external_key), + CONSTRAINT fk_defra_emission_factor_masters_source_document_id FOREIGN KEY (source_document_id) REFERENCES source_documents (source_document_id), + CONSTRAINT fk_defra_emission_factor_masters_ingestion_run_id FOREIGN KEY (ingestion_run_id) REFERENCES ingestion_runs (ingestion_run_id) ); -CREATE TABLE defra_emission_factor_details ( +CREATE INDEX IF NOT EXISTS idx_defra_emission_factor_masters_source_year ON defra_emission_factor_masters (source_family, source_year, source_version); + +CREATE INDEX IF NOT EXISTS idx_defra_emission_factor_masters_ingestion_run_id ON defra_emission_factor_masters (ingestion_run_id); + +CREATE TABLE IF NOT EXISTS defra_emission_factor_details ( defra_emission_factor_detail_id uuid NOT NULL, defra_emission_factor_master_id uuid NOT NULL, detail_external_key text NOT NULL, + source_row_number integer, + factor_id text, + factor_name text, factor_value numeric NOT NULL, factor_unit text NOT NULL, - lifecycle_status text NOT NULL, + status text NOT NULL, record_checksum_sha256 text NOT NULL, + raw_fields jsonb NOT NULL, + normalized_fields jsonb NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, CONSTRAINT pk_defra_emission_factor_details PRIMARY KEY (defra_emission_factor_detail_id), @@ -110,31 +262,52 @@ CREATE TABLE defra_emission_factor_details ( CONSTRAINT fk_defra_emission_factor_details_defra_emission_fa_98fe08fa20f4 FOREIGN KEY (defra_emission_factor_master_id) REFERENCES defra_emission_factor_masters (defra_emission_factor_master_id) ); -CREATE INDEX idx_defra_emission_factor_details_defra_emission_f_532bf4e61faf ON defra_emission_factor_details (defra_emission_factor_master_id); +CREATE INDEX IF NOT EXISTS idx_defra_emission_factor_details_defra_emission_f_532bf4e61faf ON defra_emission_factor_details (defra_emission_factor_master_id); -CREATE TABLE ipcc_emission_factor_masters ( +CREATE TABLE IF NOT EXISTS ipcc_emission_factor_masters ( ipcc_emission_factor_master_id uuid NOT NULL, + source_family text NOT NULL, + source_year integer NOT NULL, + source_version text NOT NULL, + source_release text, source_document_id uuid NOT NULL, + ingestion_run_id uuid, + run_id text, master_external_key text NOT NULL, - lifecycle_status text NOT NULL, + status text NOT NULL, + artifact_reference text, + artifact_checksum_sha256 text, + archive_reference text, + archive_checksum_sha256 text, effective_from timestamp with time zone, effective_to timestamp with time zone, record_checksum_sha256 text NOT NULL, + metadata jsonb NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, CONSTRAINT pk_ipcc_emission_factor_masters PRIMARY KEY (ipcc_emission_factor_master_id), - CONSTRAINT uq_ipcc_emission_factor_masters_external_key UNIQUE (master_external_key), - CONSTRAINT fk_ipcc_emission_factor_masters_source_document_id FOREIGN KEY (source_document_id) REFERENCES source_documents (source_document_id) + CONSTRAINT uq_ipcc_emission_factor_masters_family_year_version_key UNIQUE (source_family, source_year, source_version, master_external_key), + CONSTRAINT fk_ipcc_emission_factor_masters_source_document_id FOREIGN KEY (source_document_id) REFERENCES source_documents (source_document_id), + CONSTRAINT fk_ipcc_emission_factor_masters_ingestion_run_id FOREIGN KEY (ingestion_run_id) REFERENCES ingestion_runs (ingestion_run_id) ); -CREATE TABLE ipcc_emission_factor_details ( +CREATE INDEX IF NOT EXISTS idx_ipcc_emission_factor_masters_source_year ON ipcc_emission_factor_masters (source_family, source_year, source_version); + +CREATE INDEX IF NOT EXISTS idx_ipcc_emission_factor_masters_ingestion_run_id ON ipcc_emission_factor_masters (ingestion_run_id); + +CREATE TABLE IF NOT EXISTS ipcc_emission_factor_details ( ipcc_emission_factor_detail_id uuid NOT NULL, ipcc_emission_factor_master_id uuid NOT NULL, detail_external_key text NOT NULL, + source_row_number integer, + factor_id text, + factor_name text, factor_value numeric NOT NULL, factor_unit text NOT NULL, - lifecycle_status text NOT NULL, + status text NOT NULL, record_checksum_sha256 text NOT NULL, + raw_fields jsonb NOT NULL, + normalized_fields jsonb NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, CONSTRAINT pk_ipcc_emission_factor_details PRIMARY KEY (ipcc_emission_factor_detail_id), @@ -142,4 +315,4 @@ CREATE TABLE ipcc_emission_factor_details ( CONSTRAINT fk_ipcc_emission_factor_details_ipcc_emission_factor_master_id FOREIGN KEY (ipcc_emission_factor_master_id) REFERENCES ipcc_emission_factor_masters (ipcc_emission_factor_master_id) ); -CREATE INDEX idx_ipcc_emission_factor_details_ipcc_emission_factor_master_id ON ipcc_emission_factor_details (ipcc_emission_factor_master_id); +CREATE INDEX IF NOT EXISTS idx_ipcc_emission_factor_details_ipcc_emission_factor_master_id ON ipcc_emission_factor_details (ipcc_emission_factor_master_id); diff --git a/tests/fixtures/source_documents/defra_desnz/defra_desnz_malformed_factors.csv b/tests/fixtures/source_documents/defra_desnz/defra_desnz_malformed_factors.csv new file mode 100644 index 00000000..d20c0acc --- /dev/null +++ b/tests/fixtures/source_documents/defra_desnz/defra_desnz_malformed_factors.csv @@ -0,0 +1,3 @@ +source_year,source_version,category,subcategory,activity,factor_id,factor_name,factor_value,unit,greenhouse_gas,provenance +2024,conversion-factors-2024,Energy,Electricity,Generated,DEFRA-2024-ELEC,Electricity generated,not-a-number,kWh,CO2e,worksheet:UK electricity row 10 +2024,conversion-factors-2024,Transport,Vans,Average van,DEFRA-2024-VAN,Average van distance,0.17031,,CO2e,worksheet:Business travel land row 24 diff --git a/tests/fixtures/source_documents/defra_desnz/defra_desnz_normalized_factors.csv b/tests/fixtures/source_documents/defra_desnz/defra_desnz_normalized_factors.csv new file mode 100644 index 00000000..7a602daa --- /dev/null +++ b/tests/fixtures/source_documents/defra_desnz/defra_desnz_normalized_factors.csv @@ -0,0 +1,3 @@ +source_year,source_version,category,subcategory,activity,factor_id,factor_name,factor_value,unit,greenhouse_gas,provenance +2024,conversion-factors-2024,Energy,Electricity,Generated,DEFRA-2024-ELEC,Electricity generated,0.20705,kWh,CO2e,worksheet:UK electricity row 10 +2024,conversion-factors-2024,Transport,Vans,Average van,DEFRA-2024-VAN,Average van distance,0.17031,km,CO2e,worksheet:Business travel land row 24 diff --git a/tests/fixtures/source_documents/ghg_protocol/ghg_protocol_malformed_factors.csv b/tests/fixtures/source_documents/ghg_protocol/ghg_protocol_malformed_factors.csv new file mode 100644 index 00000000..1dc53478 --- /dev/null +++ b/tests/fixtures/source_documents/ghg_protocol/ghg_protocol_malformed_factors.csv @@ -0,0 +1,3 @@ +record_type,source_year,source_version,factor_id,factor_name,factor_value,unit,category,subcategory,scope,gas,provenance_note +emission_factor,2024,v1,GHG-ELEC-001,Grid electricity,not-a-number,kg CO2e/kWh,Stationary combustion,Electricity,Scope 2,CO2e,fixture row 1 +emission_factor,missing-year,v1,GHG-GAS-001,Natural gas,0.184,kg CO2e/kWh,Stationary combustion,Natural gas,Scope 1,CO2e,fixture row 2 diff --git a/tests/fixtures/source_documents/ghg_protocol/ghg_protocol_sample_factors.csv b/tests/fixtures/source_documents/ghg_protocol/ghg_protocol_sample_factors.csv new file mode 100644 index 00000000..5e5bcf33 --- /dev/null +++ b/tests/fixtures/source_documents/ghg_protocol/ghg_protocol_sample_factors.csv @@ -0,0 +1,4 @@ +record_type,source_year,source_version,factor_id,factor_name,factor_value,unit,category,subcategory,scope,gas,provenance_note +emission_factor,2024,v1,GHG-ELEC-001,Grid electricity,0.233,kg CO2e/kWh,Stationary combustion,Electricity,Scope 2,CO2e,fixture row 1 +metadata,2024,v1,NOTE-001,Workbook note,0,not applicable,Notes,,,,unsupported row +emission_factor,2024,v1,GHG-GAS-001,Natural gas,0.184,kg CO2e/kWh,Stationary combustion,Natural gas,Scope 1,CO2e,fixture row 3 diff --git a/tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_malformed_factors.csv b/tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_malformed_factors.csv new file mode 100644 index 00000000..7f624512 --- /dev/null +++ b/tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_malformed_factors.csv @@ -0,0 +1,4 @@ +record_type,source_year,source_version,factor_id,factor_name,factor_value,unit,category,subcategory,ipcc_sector,gas,region,technology,provenance +emission_factor,year,efdb-v2024,IPCC-BAD-YEAR,Bad year,1.2,kg CO2/TJ,Energy,,1A,CO2,Global,Default,worksheet:bad year +emission_factor,2006,efdb-v2024,IPCC-BAD-VALUE,Bad value,not-a-number,kg CO2/TJ,Energy,,1A,CO2,Global,Default,worksheet:bad value +emission_factor,2006,efdb-v2024,,Missing factor id,1.2,kg CO2/TJ,Energy,,1A,CO2,Global,Default,worksheet:missing factor id diff --git a/tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_sample_factors.csv b/tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_sample_factors.csv new file mode 100644 index 00000000..134944dc --- /dev/null +++ b/tests/fixtures/source_documents/ipcc_efdb/ipcc_efdb_sample_factors.csv @@ -0,0 +1,4 @@ +record_type,source_year,source_version,factor_id,factor_name,factor_value,unit,category,subcategory,ipcc_sector,gas,region,technology,provenance +emission_factor,2006,efdb-v2024,IPCC-ENERGY-CO2,Stationary combustion CO2,56.1,t CO2/TJ,Energy,Stationary combustion,1A,CO2,Global,Default,worksheet:EFDB row 12 +metadata,2006,efdb-v2024,IPCC-NOTE-001,Workbook note,0,none,Notes,,metadata,CO2,,,skip +emission_factor,2019,efdb-v2024,IPCC-WASTE-CH4,Wastewater methane,0.25,kg CH4/kg COD,Waste,Wastewater,4D,CH4,Global,Default,worksheet:EFDB row 44 diff --git a/tests/test_agent_dispatch_handoff_reporter.py b/tests/test_agent_dispatch_handoff_reporter.py index 664e62e6..13bda9a3 100644 --- a/tests/test_agent_dispatch_handoff_reporter.py +++ b/tests/test_agent_dispatch_handoff_reporter.py @@ -200,7 +200,12 @@ def runner(command): def _assert_no_forbidden_commands(calls: list[tuple[str, ...]]) -> None: for call in calls: - joined = " ".join(call).lower() + command_parts = tuple( + part + for index, part in enumerate(call) + if part != "--body" and (index == 0 or call[index - 1] != "--body") + ) + joined = " ".join(command_parts).lower() assert call[1:3] not in { ("pr", "merge"), ("pr", "review"), @@ -470,6 +475,14 @@ def test_claim_mode_emits_expected_issue_edit_command(tmp_path: Path) -> None: "--add-label", "status:in-progress", ) in calls + body_edit_calls = [ + call + for call in calls + if call[1:3] == ("issue", "edit") and "--body" in call + ] + assert len(body_edit_calls) == 1 + assert "Status: in-progress" in body_edit_calls[0][-1] + assert body_edit_calls[0][-1].splitlines().count("Status: in-progress") == 1 assert "## Label Mutation Performed" in report assert "- Removed label: `status:ready`" in report assert "- Added label: `status:in-progress`" in report @@ -577,9 +590,13 @@ def test_claim_mode_claims_exactly_one_issue_by_lane_priority_and_number(tmp_pat ) issue_edit_calls = [call for call in calls if call[1:3] == ("issue", "edit")] + label_edit_calls = [call for call in issue_edit_calls if "--body" not in call] + body_edit_calls = [call for call in issue_edit_calls if "--body" in call] assert exit_code == 0 - assert len(issue_edit_calls) == 1 - assert issue_edit_calls[0][3] == "407" + assert len(label_edit_calls) == 1 + assert label_edit_calls[0][3] == "407" + assert len(body_edit_calls) == 1 + assert body_edit_calls[0][3] == "407" assert "Selected issue number: #407" in report @@ -1464,7 +1481,7 @@ def test_run_once_claims_one_ready_task_only_with_explicit_opt_in(tmp_path: Path assert exit_code == 0 assert "claimed_task_created" in report assert "Task claimed: yes" in report - assert len(issue_edit_calls) == 1 + assert len(issue_edit_calls) == 2 assert issue_edit_calls[0] == ( "gh", "issue", @@ -1477,6 +1494,16 @@ def test_run_once_claims_one_ready_task_only_with_explicit_opt_in(tmp_path: Path "--add-label", "status:in-progress", ) + assert issue_edit_calls[1][:6] == ( + "gh", + "issue", + "edit", + "415", + "--repo", + "example/repo", + ) + assert issue_edit_calls[1][-2] == "--body" + assert "Status: in-progress" in issue_edit_calls[1][-1] assert (tmp_path / "OPS-020-415-prompt.md").exists() _assert_no_forbidden_commands(calls) diff --git a/tests/test_agent_task_watcher.py b/tests/test_agent_task_watcher.py new file mode 100644 index 00000000..037506f7 --- /dev/null +++ b/tests/test_agent_task_watcher.py @@ -0,0 +1,323 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "agent_task_watcher.py" +SPEC = importlib.util.spec_from_file_location("agent_task_watcher", SCRIPT_PATH) +assert SPEC is not None +watcher = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = watcher +SPEC.loader.exec_module(watcher) + + +Issue = watcher.Issue +PullRequest = watcher.PullRequest +WatcherError = watcher.WatcherError + + +class FakeClient: + def __init__(self, *, issues: tuple[Issue, ...], pull_request: PullRequest) -> None: + self.issues = {issue.number: issue for issue in issues} + self.pull_request = pull_request + self.comments: list[tuple[int, str]] = [] + self.find_calls: list[str] = [] + self.add_calls: list[tuple[int, str]] = [] + self.remove_calls: list[tuple[int, str]] = [] + self.edit_body_calls: list[tuple[int, str]] = [] + + def view_pr(self, pr_number: int) -> PullRequest: + assert pr_number == self.pull_request.number + return self.pull_request + + def view_issue(self, issue_number: int) -> Issue: + return self.issues[issue_number] + + def find_issue_for_task(self, task_id: str) -> int | None: + self.find_calls.append(task_id) + matches = [ + issue.number + for issue in self.issues.values() + if issue.title.startswith(f"[{task_id}]") + ] + return matches[0] if len(matches) == 1 else None + + def add_label(self, issue_number: int, label: str) -> None: + self.add_calls.append((issue_number, label)) + issue = self.issues[issue_number] + if label not in issue.labels: + self.issues[issue_number] = Issue( + number=issue.number, + title=issue.title, + body=issue.body, + labels=issue.labels + (label,), + state=issue.state, + ) + + def remove_label(self, issue_number: int, label: str) -> None: + self.remove_calls.append((issue_number, label)) + issue = self.issues[issue_number] + self.issues[issue_number] = Issue( + number=issue.number, + title=issue.title, + body=issue.body, + labels=tuple(existing for existing in issue.labels if existing != label), + state=issue.state, + ) + + def edit_body(self, issue_number: int, body: str) -> None: + self.edit_body_calls.append((issue_number, body)) + issue = self.issues[issue_number] + self.issues[issue_number] = Issue( + number=issue.number, + title=issue.title, + body=body, + labels=issue.labels, + state=issue.state, + ) + + def comment(self, issue_number: int, body: str) -> None: + self.comments.append((issue_number, body)) + + +def _issue( + number: int, + task_id: str, + labels: tuple[str, ...], + *, + body: str | None = None, +) -> Issue: + body = body if body is not None else f"Task ID: {task_id}\nDepends on: none\nUnblocks: none" + return Issue( + number=number, + title=f"[{task_id}] Example task", + body=body, + labels=labels, + ) + + +def _pr(body: str, *, title: str = "[OPS-031] Example task") -> PullRequest: + return PullRequest(number=505, title=title, body=body, merged=True) + + +def test_merged_transition_removes_other_status_labels_and_adds_merged() -> None: + issue = _issue( + 505, + "OPS-031", + ("status:ready", "status:in-progress", "status:blocked", "lane:ops"), + ) + client = FakeClient(issues=(issue,), pull_request=_pr("Task-ID: OPS-031\nTask-Issue: #505")) + + replacement = watcher.replace_status_label(client, issue, "status:merged") + + assert replacement.old_statuses == ("status:ready", "status:in-progress", "status:blocked") + assert client.issues[505].labels == ("lane:ops", "status:merged") + assert client.add_calls == [(505, "status:merged")] + assert client.remove_calls == [ + (505, "status:ready"), + (505, "status:in-progress"), + (505, "status:blocked"), + ] + assert client.issues[505].body.startswith("Task ID: OPS-031\nStatus: merged\n") + + +def test_ready_transition_removes_blocked_in_progress_merged_and_adds_ready() -> None: + issue = _issue( + 506, + "OPS-032", + ("status:blocked", "status:in-progress", "status:merged", "agent:ops"), + ) + client = FakeClient(issues=(issue,), pull_request=_pr("Task-ID: OPS-031\nTask-Issue: #505")) + + replacement = watcher.replace_status_label(client, issue, "status:ready") + + assert replacement.old_statuses == ("status:blocked", "status:in-progress", "status:merged") + assert client.issues[506].labels == ("agent:ops", "status:ready") + assert client.issues[506].body.startswith("Task ID: OPS-032\nStatus: ready\n") + + +def test_blocked_transition_updates_body_status() -> None: + issue = _issue( + 508, + "OPS-034", + ("status:ready", "lane:ops"), + body="Task ID: OPS-034\nLane: ops\nStatus: ready\nDepends on: OPS-033\nUnblocks: none", + ) + client = FakeClient(issues=(issue,), pull_request=_pr("Task-ID: OPS-034\nTask-Issue: #508")) + + watcher.replace_status_label(client, issue, "status:blocked") + + assert client.issues[508].labels == ("lane:ops", "status:blocked") + assert client.issues[508].body.splitlines().count("Status: blocked") == 1 + assert "Status: ready" not in client.issues[508].body + + +def test_in_progress_transition_updates_existing_body_status() -> None: + issue = _issue( + 507, + "OPS-033", + ("status:ready", "lane:ops"), + body="Task ID: OPS-033\nLane: ops\nStatus: ready\nDepends on: none\nUnblocks: none", + ) + client = FakeClient(issues=(issue,), pull_request=_pr("Task-ID: OPS-033\nTask-Issue: #507")) + + watcher.replace_status_label(client, issue, "status:in-progress") + + assert client.issues[507].labels == ("lane:ops", "status:in-progress") + assert client.issues[507].body.splitlines().count("Status: in-progress") == 1 + assert "Status: ready" not in client.issues[507].body + + +def test_repeated_watcher_run_is_idempotent() -> None: + source = _issue( + 505, + "OPS-031", + ("status:ready",), + body="Task ID: OPS-031\nDepends on: OPS-030\nUnblocks: none", + ) + dependency = _issue(504, "OPS-030", ("status:merged",)) + client = FakeClient( + issues=(source, dependency), + pull_request=_pr("Task-ID: OPS-031\nTask-Issue: #505"), + ) + + watcher.process_merged_pr(client, 505) + first_labels = client.issues[505].labels + first_body = client.issues[505].body + watcher.process_merged_pr(client, 505) + + assert first_labels == ("status:merged",) + assert client.issues[505].labels == first_labels + assert client.issues[505].body == first_body + assert client.issues[505].body.splitlines().count("Status: merged") == 1 + assert client.add_calls == [(505, "status:merged")] + assert client.edit_body_calls == [(505, first_body)] + + +def test_pr_footer_task_issue_mapping_is_preferred_over_title_parsing() -> None: + issue = _issue(700, "OPS-031", ("status:ready",)) + client = FakeClient( + issues=(issue,), + pull_request=_pr("Task-ID: OPS-031\nTask-Issue: #700", title="[OPS-999] Wrong title"), + ) + + result = watcher.process_merged_pr(client, 505) + + assert result is not None + assert result.task_issue_number == 700 + assert client.find_calls == [] + assert client.issues[700].labels == ("status:merged",) + + +def test_task_issue_footer_uses_resolved_issue_task_id_when_pr_task_id_is_missing() -> None: + issue = _issue(700, "OPS-031", ("status:ready",)) + client = FakeClient( + issues=(issue,), + pull_request=_pr("Task-Issue: #700", title="[OPS-999] Wrong title"), + ) + + result = watcher.process_merged_pr(client, 505) + + assert result is not None + assert result.task_id == "OPS-031" + assert result.task_issue_number == 700 + assert client.find_calls == [] + + +def test_downstream_dependency_ready_update_uses_status_replacement() -> None: + source = _issue( + 505, + "OPS-031", + ("status:in-progress",), + body="Task ID: OPS-031\nDepends on: none\nUnblocks: OPS-032", + ) + dependent = _issue( + 506, + "OPS-032", + ("status:blocked", "status:in-progress", "lane:ops"), + body="Task ID: OPS-032\nDepends on: OPS-031\nUnblocks: none", + ) + client = FakeClient( + issues=(source, dependent), + pull_request=_pr("Task-ID: OPS-031\nTask-Issue: #505"), + ) + + result = watcher.process_merged_pr(client, 505) + + assert result is not None + assert client.issues[505].labels == ("status:merged",) + assert client.issues[506].labels == ("lane:ops", "status:ready") + assert "Status: merged" in client.issues[505].body + assert "Status: ready" in client.issues[506].body + assert (506, "status:ready") in client.add_calls + assert (506, "status:blocked") in client.remove_calls + assert (506, "status:in-progress") in client.remove_calls + + +def test_downstream_missing_depends_on_becomes_needs_attention() -> None: + source = _issue( + 505, + "OPS-031", + ("status:ready",), + body="Task ID: OPS-031\nDepends on: none\nUnblocks: OPS-032", + ) + dependent = _issue( + 506, + "OPS-032", + ("status:blocked",), + body="Task ID: OPS-032\nUnblocks: none", + ) + client = FakeClient( + issues=(source, dependent), + pull_request=_pr("Task-ID: OPS-031\nTask-Issue: #505"), + ) + + result = watcher.process_merged_pr(client, 505) + + assert result is not None + assert client.issues[506].labels == ("status:needs-attention",) + assert "Status: needs-attention" in client.issues[506].body + assert "`OPS-032` (#506) needs attention: missing `Depends on:` metadata." in result.skipped_downstream + + +def test_downstream_unresolved_dependency_becomes_needs_attention() -> None: + source = _issue( + 505, + "OPS-031", + ("status:ready",), + body="Task ID: OPS-031\nDepends on: none\nUnblocks: OPS-032", + ) + dependent = _issue( + 506, + "OPS-032", + ("status:blocked",), + body="Task ID: OPS-032\nDepends on: OPS-999\nUnblocks: none", + ) + client = FakeClient( + issues=(source, dependent), + pull_request=_pr("Task-ID: OPS-031\nTask-Issue: #505"), + ) + + result = watcher.process_merged_pr(client, 505) + + assert result is not None + assert client.issues[506].labels == ("status:needs-attention",) + assert ( + "`OPS-032` (#506) needs attention: dependencies did not resolve to unique issues: `OPS-999`." + in result.skipped_downstream + ) + + +def test_missing_task_mapping_produces_clear_diagnostic() -> None: + client = FakeClient( + issues=(), + pull_request=_pr("Summary only", title="No task id here"), + ) + + with pytest.raises(WatcherError, match="no Task-Issue footer and no Task-ID/title task id"): + watcher.process_merged_pr(client, 505) diff --git a/tests/test_configured_cycle_runner.py b/tests/test_configured_cycle_runner.py new file mode 100644 index 00000000..5effcd1d --- /dev/null +++ b/tests/test_configured_cycle_runner.py @@ -0,0 +1,795 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from carbonfactor_parser.persistence.postgresql_runtime_config import ( + load_postgresql_runtime_config, +) +from carbonfactor_parser.persistence.postgresql_runtime_schema_bootstrap import ( + PostgreSQLRuntimeSchemaBootstrapResult, +) +from carbonfactor_parser.persistence.postgresql_year_state_repository import ( + PostgreSQLSourceFamilyYearStateRepository, +) +from carbonfactor_parser.persistence.postgresql_runtime import ( + PostgreSQLRuntimeStartupResult, +) +from carbonfactor_parser.persistence.postgresql_source_family_repository import ( + PostgreSQLSourceFamilyRuntimeRepository, +) +from carbonfactor_parser.pipeline.configured_cycle_config import ( + ConfiguredCycleRunnerConfig as ExtractedConfiguredCycleRunnerConfig, + ConfiguredSourceYearArtifact as ExtractedConfiguredSourceYearArtifact, + load_configured_cycle_runner_config as extracted_load_configured_cycle_runner_config, +) +from carbonfactor_parser.pipeline.configured_cycle_models import ( + ConfiguredCycleResult as ExtractedConfiguredCycleResult, +) +from carbonfactor_parser.pipeline.configured_cycle_summary import ( + emit_configured_cycle_summary as extracted_emit_configured_cycle_summary, +) +from carbonfactor_parser.pipeline.configured_cycle_dependencies import ( + ConfiguredCycleValidationBoundary as ExtractedConfiguredCycleValidationBoundary, + build_configured_cycle_dependencies, +) +from carbonfactor_parser.pipeline.configured_cycle_runner import ( + ConfiguredCycleResult, + ConfiguredCycleRunnerConfig, + ConfiguredCycleRunnerStatus, + ConfiguredCycleValidationBoundary, + ConfiguredSourceYearArtifact, + emit_configured_cycle_summary, + load_configured_cycle_runner_config, + run_configured_cycle_runner, +) +from carbonfactor_parser.pipeline import source_artifact_transport +from carbonfactor_parser.pipeline.defra_desnz_production_e2e import ( + DefraDesnzProductionParserBoundary, + DefraDesnzProductionSourceAdapter, +) +from carbonfactor_parser.pipeline.ghg_protocol_production_e2e import ( + GHGProtocolProductionParserBoundary, + GHGProtocolProductionSourceAdapter, +) +from carbonfactor_parser.pipeline.ipcc_efdb_production_e2e import ( + IpccEfdbProductionParserBoundary, + IpccEfdbProductionSourceAdapter, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EYearOrchestratorDependencies, +) +from carbonfactor_parser.pipeline.source_artifact_transport import ( + build_configured_artifact_transport, +) + + +def test_configured_cycle_runner_imports_remain_backward_compatible() -> None: + assert ConfiguredCycleRunnerConfig is ExtractedConfiguredCycleRunnerConfig + assert ConfiguredSourceYearArtifact is ExtractedConfiguredSourceYearArtifact + assert load_configured_cycle_runner_config is extracted_load_configured_cycle_runner_config + assert ConfiguredCycleValidationBoundary is ExtractedConfiguredCycleValidationBoundary + assert ConfiguredCycleResult is ExtractedConfiguredCycleResult + assert emit_configured_cycle_summary is extracted_emit_configured_cycle_summary + + +def test_build_configured_cycle_dependencies_wires_configured_runtime( + tmp_path: Path, +) -> None: + connection = _FakeConnection() + startup = _startup(connection) + config = ConfiguredCycleRunnerConfig( + postgresql_config_result=load_postgresql_runtime_config( + {"CARBONOPS_POSTGRESQL_DSN": "postgresql://user:pass@localhost/db"}, + ), + archive_root=tmp_path / "archive", + enabled_source_families=("ghg_protocol", "defra_desnz", "ipcc_efdb"), + initial_year=2024, + cycle_interval_seconds=0, + max_cycles=1, + source_years=_source_years(tmp_path), + allow_live_source_access=False, + ) + + dependencies = build_configured_cycle_dependencies(config, startup) + + assert isinstance(dependencies, ProductionE2EYearOrchestratorDependencies) + assert dependencies.year_state_repository is startup.year_state_repository + assert dependencies.source_adapters.keys() == { + "ghg_protocol", + "defra_desnz", + "ipcc_efdb", + } + assert isinstance( + dependencies.source_adapters["ghg_protocol"], + GHGProtocolProductionSourceAdapter, + ) + assert isinstance( + dependencies.source_adapters["defra_desnz"], + DefraDesnzProductionSourceAdapter, + ) + assert isinstance( + dependencies.source_adapters["ipcc_efdb"], + IpccEfdbProductionSourceAdapter, + ) + assert dependencies.parser_boundaries.keys() == { + "ghg_protocol", + "defra_desnz", + "ipcc_efdb", + } + assert isinstance( + dependencies.parser_boundaries["ghg_protocol"], + GHGProtocolProductionParserBoundary, + ) + assert isinstance( + dependencies.parser_boundaries["defra_desnz"], + DefraDesnzProductionParserBoundary, + ) + assert isinstance( + dependencies.parser_boundaries["ipcc_efdb"], + IpccEfdbProductionParserBoundary, + ) + assert isinstance( + dependencies.validation_boundary, + ExtractedConfiguredCycleValidationBoundary, + ) + assert isinstance( + dependencies.insert_repository, + PostgreSQLSourceFamilyRuntimeRepository, + ) + assert dependencies.insert_repository._connection is connection + assert ( + dependencies.source_adapters["ghg_protocol"]._source_years[2024].year == 2024 + ) + assert ( + dependencies.source_adapters["defra_desnz"]._source_years[2024].year == 2024 + ) + assert dependencies.source_adapters["ipcc_efdb"]._source_years[2024].year == 2024 + + +def test_source_artifact_transport_reads_local_file_path(tmp_path: Path) -> None: + artifact_path = tmp_path / "artifact.csv" + artifact_path.write_bytes(b"configured artifact") + + transport = build_configured_artifact_transport(allow_live_source_access=False) + + assert transport(str(artifact_path)) == b"configured artifact" + assert transport(f"local:{artifact_path}") == b"configured artifact" + assert transport(artifact_path.as_uri()) == b"configured artifact" + + +def test_source_artifact_transport_blocks_https_without_live_opt_in() -> None: + transport = build_configured_artifact_transport(allow_live_source_access=False) + + with pytest.raises(ValueError, match="Live HTTPS source access requires explicit"): + transport("https://example.invalid/factors.csv") + + +def test_source_artifact_transport_uses_configured_https_policy( + monkeypatch: pytest.MonkeyPatch, +) -> None: + requests = [] + + class _Response: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return None + + def read(self) -> bytes: + return b"live artifact" + + def fake_urlopen(request, *, timeout): + requests.append((request, timeout)) + return _Response() + + monkeypatch.setattr(source_artifact_transport, "urlopen", fake_urlopen) + + transport = build_configured_artifact_transport(allow_live_source_access=True) + + assert transport("https://example.invalid/factors.csv") == b"live artifact" + request, timeout = requests[0] + assert request.full_url == "https://example.invalid/factors.csv" + assert request.get_header("User-agent") == "carbonops-parser/0.1" + assert timeout == 60 + + +def test_configured_cycle_runner_loads_json_config(tmp_path: Path) -> None: + config_path = tmp_path / "carbonops-cycle.json" + config_path.write_text( + json.dumps( + { + "postgresql": {"dsn": "postgresql://user:pass@localhost/db"}, + "archive_root": str(tmp_path / "archive"), + "enabled_source_families": ["ghg_protocol", "defra_desnz"], + "initial_year": 2024, + "cycle": {"interval_seconds": 0, "max_cycles": 2}, + "real_source_smoke": {"allow_live_source_access": True}, + "source_years": { + "ghg_protocol": { + "2024": { + "artifact_url": str(tmp_path / "ghg-2024.csv"), + "version_label": "v2024", + }, + }, + }, + }, + ), + encoding="utf-8", + ) + + config = load_configured_cycle_runner_config(config_path) + + assert config.postgresql_config_result.is_ready + assert config.archive_root == tmp_path / "archive" + assert config.enabled_source_families == ("ghg_protocol", "defra_desnz") + assert config.initial_year == 2024 + assert config.max_cycles == 2 + assert config.allow_live_source_access is True + assert config.source_years is not None + assert config.source_years["ghg_protocol"][2024].version_label == "v2024" + + +def test_configured_cycle_runner_rejects_conflicting_live_source_access_aliases( + tmp_path: Path, +) -> None: + config_path = tmp_path / "carbonops-cycle.json" + config_path.write_text( + json.dumps( + { + "postgresql": {"dsn": "postgresql://user:pass@localhost/db"}, + "allow_live_source_access": False, + "real_source_smoke": {"allow_live_source_access": True}, + }, + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="Conflicting live source access settings"): + load_configured_cycle_runner_config(config_path) + + +def test_configured_cycle_runner_keeps_explicit_false_live_source_access_aliases( + tmp_path: Path, +) -> None: + config_path = tmp_path / "carbonops-cycle.json" + config_path.write_text( + json.dumps( + { + "postgresql": {"dsn": "postgresql://user:pass@localhost/db"}, + "allow_live_source_access": False, + "real_source_smoke": {"allow_live_source_access": False}, + }, + ), + encoding="utf-8", + ) + + config = load_configured_cycle_runner_config(config_path) + + assert config.allow_live_source_access is False + + +def test_configured_cycle_runner_loads_top_level_live_source_access_true( + tmp_path: Path, +) -> None: + config_path = tmp_path / "carbonops-cycle.json" + config_path.write_text( + json.dumps( + { + "postgresql": {"dsn": "postgresql://user:pass@localhost/db"}, + "allow_live_source_access": True, + }, + ), + encoding="utf-8", + ) + + config = load_configured_cycle_runner_config(config_path) + + assert config.allow_live_source_access is True + + +def test_configured_cycle_runner_runs_2024_to_2027_and_is_idempotent( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + connection = _FakeConnection() + startup = _startup(connection) + config = ConfiguredCycleRunnerConfig( + postgresql_config_result=load_postgresql_runtime_config( + {"CARBONOPS_POSTGRESQL_DSN": "postgresql://user:pass@localhost/db"}, + ), + archive_root=tmp_path / "archive", + enabled_source_families=("ghg_protocol", "defra_desnz", "ipcc_efdb"), + initial_year=2024, + cycle_interval_seconds=0, + max_cycles=4, + source_years=_source_years(tmp_path), + ) + + result = run_configured_cycle_runner(config, startup=startup, sleep=lambda _: None) + + captured = capsys.readouterr() + assert result.status is ConfiguredCycleRunnerStatus.COMPLETED + assert tuple( + family.year_state.target_year + for cycle in result.cycles[:3] + for family in cycle.result.family_results + ) == (2024, 2024, 2024, 2025, 2025, 2025, 2026, 2026, 2026) + assert tuple( + family.status.value for family in result.cycles[-1].result.family_results + ) == ( + "no_available_source_year", + "no_available_source_year", + "no_available_source_year", + ) + assert "target_year=2027" in captured.out + assert "status=no_available_source_year" in captured.out + assert connection.latest_years == { + "ghg_protocol": 2026, + "defra_desnz": 2026, + "ipcc_efdb": 2026, + } + assert all( + connection.table_counts[table_name] > 0 + for table_name in ( + "ghg_emission_factor_masters", + "ghg_emission_factor_details", + "defra_emission_factor_masters", + "defra_emission_factor_details", + "ipcc_emission_factor_masters", + "ipcc_emission_factor_details", + ) + ) + + second = run_configured_cycle_runner(config, startup=startup, sleep=lambda _: None) + + assert second.cycles[0].result.family_results[0].year_state.target_year == 2027 + assert connection.latest_years == { + "ghg_protocol": 2026, + "defra_desnz": 2026, + "ipcc_efdb": 2026, + } + + +def test_configured_cycle_runner_blocks_https_without_live_opt_in( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + connection = _FakeConnection() + config = ConfiguredCycleRunnerConfig( + postgresql_config_result=load_postgresql_runtime_config( + {"CARBONOPS_POSTGRESQL_DSN": "postgresql://user:pass@localhost/db"}, + ), + archive_root=tmp_path / "archive", + enabled_source_families=("ghg_protocol",), + initial_year=2024, + cycle_interval_seconds=0, + max_cycles=1, + source_years={ + "ghg_protocol": { + 2024: ConfiguredSourceYearArtifact( + year=2024, + artifact_url="https://example.invalid/factors.csv?token=secret", + publication_url="https://example.invalid/publication", + title="configured live artifact", + version_label="live-2024", + content_type="text/csv", + format_hint="csv", + ) + } + }, + allow_live_source_access=False, + ) + + result = run_configured_cycle_runner( + config, + startup=_startup(connection), + sleep=lambda _: None, + ) + + captured = capsys.readouterr() + family = result.cycles[0].result.family_results[0] + assert result.status is ConfiguredCycleRunnerStatus.COMPLETED_WITH_FAILURES + assert family.status.value == "failed" + assert family.failures[0].code == "GHG_PROTOCOL_PRODUCTION_DOWNLOAD_FAILED" + assert "Live HTTPS source access requires explicit real-source smoke opt-in" in ( + family.failures[0].message + ) + assert "download_status=failed" in captured.out + assert "parse_status=not_run" in captured.out + assert "secret" not in family.failures[0].message + assert "token=secret" not in captured.out + + +def test_invalid_archive_root_type_fails_clearly(tmp_path: Path) -> None: + config_path = tmp_path / "bad.json" + config_path.write_text( + json.dumps({"postgresql": {"dsn": "postgresql://u:p@h/db"}, "archive_root": 12}), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="archive_root must be a string path"): + load_configured_cycle_runner_config(config_path) + + +def test_unsupported_enabled_source_family_fails_clearly(tmp_path: Path) -> None: + config_path = tmp_path / "bad-family.json" + config_path.write_text( + json.dumps( + {"postgresql": {"dsn": "postgresql://u:p@h/db"}, "enabled_source_families": ["unknown"]}, + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="Unsupported enabled source family"): + load_configured_cycle_runner_config(config_path) + + +@pytest.mark.parametrize("value,field", [("0", "initial_year"), ("x", "initial_year")]) +def test_invalid_initial_year_fails_clearly(tmp_path: Path, value: str, field: str) -> None: + config_path = tmp_path / "bad-year.json" + config_path.write_text( + json.dumps({"postgresql": {"dsn": "postgresql://u:p@h/db"}, "initial_year": value}), + encoding="utf-8", + ) + with pytest.raises(ValueError, match=field): + load_configured_cycle_runner_config(config_path) + + +def test_invalid_cycle_interval_fails_clearly(tmp_path: Path) -> None: + config_path = tmp_path / "bad-interval.json" + config_path.write_text( + json.dumps({"postgresql": {"dsn": "postgresql://u:p@h/db"}, "cycle_interval_seconds": -1}), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="cycle_interval_seconds"): + load_configured_cycle_runner_config(config_path) + + +def test_invalid_max_cycles_fails_clearly(tmp_path: Path) -> None: + config_path = tmp_path / "bad-max-cycles.json" + config_path.write_text( + json.dumps({"postgresql": {"dsn": "postgresql://u:p@h/db"}, "max_cycles": 0}), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="max_cycles"): + load_configured_cycle_runner_config(config_path) + + +def test_invalid_source_artifact_definition_fails_clearly(tmp_path: Path) -> None: + config_path = tmp_path / "bad-source.json" + config_path.write_text( + json.dumps( + { + "postgresql": {"dsn": "postgresql://u:p@h/db"}, + "source_years": {"ghg_protocol": {"2024": {"title": "missing artifact"}}}, + }, + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="requires artifact_url"): + load_configured_cycle_runner_config(config_path) + + +def _source_years( + tmp_path: Path, +) -> dict[str, dict[int, ConfiguredSourceYearArtifact]]: + source_years: dict[str, dict[int, ConfiguredSourceYearArtifact]] = { + "ghg_protocol": {}, + "defra_desnz": {}, + "ipcc_efdb": {}, + } + for year in (2024, 2025, 2026): + ghg_path = tmp_path / f"ghg-{year}.csv" + defra_path = tmp_path / f"defra-{year}.csv" + ipcc_path = tmp_path / f"ipcc-{year}.csv" + ghg_path.write_text(_ghg_csv(year), encoding="utf-8") + defra_path.write_text(_defra_csv(year), encoding="utf-8") + ipcc_path.write_text(_ipcc_csv(year), encoding="utf-8") + source_years["ghg_protocol"][year] = _artifact(year, ghg_path, f"v{year}") + source_years["defra_desnz"][year] = _artifact( + year, + defra_path, + f"conversion-factors-{year}", + ) + source_years["ipcc_efdb"][year] = _artifact( + year, + ipcc_path, + f"efdb-v{year}", + ) + return source_years + + +def _artifact( + year: int, + path: Path, + version_label: str, +) -> ConfiguredSourceYearArtifact: + return ConfiguredSourceYearArtifact( + year=year, + artifact_url=str(path), + publication_url=str(path), + title=f"fixture {year}", + version_label=version_label, + content_type="text/csv", + format_hint="csv", + ) + + +def _ghg_csv(year: int) -> str: + return ( + "record_type,source_year,source_version,factor_id,factor_name," + "factor_value,unit,category,subcategory,scope,gas,provenance_note\n" + f"emission_factor,{year},v{year},GHG-{year}-ELEC,Grid electricity," + "0.233,kg CO2e/kWh,Stationary combustion,Electricity,Scope 2,CO2e," + "fixture row 1\n" + ) + + +def _defra_csv(year: int) -> str: + return ( + "source_year,source_version,category,subcategory,activity,factor_id," + "factor_name,factor_value,unit,greenhouse_gas,provenance\n" + f"{year},conversion-factors-{year},Energy,Electricity,Generated," + f"DEFRA-{year}-ELEC,Electricity generated,0.20705,kWh,CO2e," + "worksheet:UK electricity row 10\n" + ) + + +def _ipcc_csv(year: int) -> str: + return ( + "record_type,source_year,source_version,factor_id,factor_name," + "factor_value,unit,category,subcategory,ipcc_sector,gas,region," + "technology,provenance\n" + f"emission_factor,{year},efdb-v{year},IPCC-{year}-ENERGY-CO2," + "Stationary combustion CO2,56.1,t CO2/TJ,Energy," + "Stationary combustion,1A,CO2,Global,Default,worksheet:EFDB row 12\n" + ) + + +def _startup(connection: "_FakeConnection") -> PostgreSQLRuntimeStartupResult: + return PostgreSQLRuntimeStartupResult( + connection=connection, + schema_bootstrap=PostgreSQLRuntimeSchemaBootstrapResult( + required_table_names=(), + present_table_names=(), + missing_table_names=(), + created_table_names=( + "ghg_emission_factor_masters", + "ghg_emission_factor_details", + ), + statement_count=0, + ), + year_state_repository=PostgreSQLSourceFamilyYearStateRepository(connection), + ) + + +class _FakeCursor: + def __init__(self, row: tuple[object, ...] | None = None) -> None: + self._row = row + + def fetchone(self) -> tuple[object, ...] | None: + return self._row + + +class _FakeConnection: + def __init__(self) -> None: + self.latest_years: dict[str, int] = {} + self.master_keys: set[tuple[object, ...]] = set() + self.detail_keys: set[tuple[object, ...]] = set() + self.table_counts = { + "ghg_emission_factor_masters": 0, + "ghg_emission_factor_details": 0, + "defra_emission_factor_masters": 0, + "defra_emission_factor_details": 0, + "ipcc_emission_factor_masters": 0, + "ipcc_emission_factor_details": 0, + } + + def execute(self, statement: str, parameters: object | None = None) -> _FakeCursor: + normalized = " ".join(statement.split()).lower() + if "select max(ingested_year)" in normalized: + assert isinstance(parameters, tuple) + return _FakeCursor((self.latest_years.get(str(parameters[0])),)) + if "insert into source_family_year_states" in normalized: + assert isinstance(parameters, tuple) + self.latest_years[str(parameters[1])] = int(parameters[2]) + return _FakeCursor() + if "_emission_factor_masters" in normalized and normalized.startswith("insert"): + assert isinstance(parameters, tuple) + table = _table_name(normalized) + key = (parameters[1], parameters[2], parameters[3], parameters[8]) + if key in self.master_keys: + return _FakeCursor() + self.master_keys.add(key) + self.table_counts[table] += 1 + return _FakeCursor((parameters[0],)) + if "_emission_factor_details" in normalized and normalized.startswith("insert"): + assert isinstance(parameters, tuple) + table = _table_name(normalized) + key = (parameters[1], parameters[2]) + if key in self.detail_keys: + return _FakeCursor() + self.detail_keys.add(key) + self.table_counts[table] += 1 + return _FakeCursor((parameters[0],)) + return _FakeCursor() + + def commit(self) -> None: + pass + + def rollback(self) -> None: + pass + + +def _table_name(normalized_statement: str) -> str: + parts = normalized_statement.split() + return parts[2] + + +def test_configured_cycle_runner_persists_history_once_per_cycle( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + connection = _FakeConnection() + repository = _FakeHistoryRepository() + config = ConfiguredCycleRunnerConfig( + postgresql_config_result=load_postgresql_runtime_config( + {"CARBONOPS_POSTGRESQL_DSN": "postgresql://user:pass@localhost/db"}, + ), + archive_root=tmp_path / "archive", + enabled_source_families=("ghg_protocol",), + initial_year=2024, + cycle_interval_seconds=0, + max_cycles=1, + source_years={"ghg_protocol": {2024: _artifact(2024, tmp_path / "ghg.csv", "v2024")}}, + ) + (tmp_path / "ghg.csv").write_text(_ghg_csv(2024), encoding="utf-8") + + result = run_configured_cycle_runner( + config, + startup=_startup(connection), + sleep=lambda _: None, + run_history_repository=repository, + ) + + captured = capsys.readouterr() + assert result.status is ConfiguredCycleRunnerStatus.COMPLETED + assert len(repository.commands) == 1 + command = repository.commands[0] + assert command.run.run_id == result.cycles[0].run_id + assert command.run.status == "completed" + assert command.source_results[0].source_family == "ghg_protocol" + assert command.source_results[0].target_year == 2024 + assert result.cycles[0].history_persistence_status == "declared" + assert result.cycles[0].history_persistence_issue_count == 0 + assert "cycle=1" in captured.out + assert "history_persistence status=declared" in captured.out + + +def test_configured_cycle_runner_history_failure_does_not_fail_ingestion( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + connection = _FakeConnection() + repository = _FakeHistoryRepository( + status="failed_database", + issue_message="database failed dsn=postgresql://user:secret@localhost/db token=abc", + ) + config = ConfiguredCycleRunnerConfig( + postgresql_config_result=load_postgresql_runtime_config( + {"CARBONOPS_POSTGRESQL_DSN": "postgresql://user:pass@localhost/db"}, + ), + archive_root=tmp_path / "archive", + enabled_source_families=("ghg_protocol",), + initial_year=2024, + cycle_interval_seconds=0, + max_cycles=1, + source_years={"ghg_protocol": {2024: _artifact(2024, tmp_path / "ghg.csv", "v2024")}}, + ) + (tmp_path / "ghg.csv").write_text(_ghg_csv(2024), encoding="utf-8") + + result = run_configured_cycle_runner( + config, + startup=_startup(connection), + sleep=lambda _: None, + run_history_repository=repository, + ) + + captured = capsys.readouterr() + assert result.status is ConfiguredCycleRunnerStatus.COMPLETED + assert result.cycles[0].result.status.value == "completed" + assert result.cycles[0].history_persistence_status == "failed" + assert result.cycles[0].history_persistence_issue_count == 1 + assert "cycle=1" in captured.out + assert "history_persistence status=failed" in captured.out + assert "POSTGRESQL_INGESTION_RUN_HISTORY_DATABASE_ERROR" in captured.out + assert "secret" not in captured.out + assert "token=abc" not in captured.out + + +def test_configured_cycle_runner_history_exception_is_sanitized_and_non_fatal( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + connection = _FakeConnection() + repository = _ExceptionHistoryRepository() + config = ConfiguredCycleRunnerConfig( + postgresql_config_result=load_postgresql_runtime_config( + {"CARBONOPS_POSTGRESQL_DSN": "postgresql://user:pass@localhost/db"}, + ), + archive_root=tmp_path / "archive", + enabled_source_families=("ghg_protocol",), + initial_year=2024, + cycle_interval_seconds=0, + max_cycles=1, + source_years={"ghg_protocol": {2024: _artifact(2024, tmp_path / "ghg.csv", "v2024")}}, + ) + (tmp_path / "ghg.csv").write_text(_ghg_csv(2024), encoding="utf-8") + + result = run_configured_cycle_runner( + config, + startup=_startup(connection), + sleep=lambda _: None, + run_history_repository=repository, + ) + + captured = capsys.readouterr() + assert result.status is ConfiguredCycleRunnerStatus.COMPLETED + assert result.cycles[0].result.status.value == "completed" + assert result.cycles[0].history_persistence_status == "failed" + assert result.cycles[0].history_persistence_issue_count == 1 + assert "INGESTION_RUN_HISTORY_PERSISTENCE_EXCEPTION" in captured.out + assert "secret" not in captured.out + assert "token=abc" not in captured.out + + +class _ExceptionHistoryRepository: + @property + def provider_name(self) -> str: + return "fake" + + def persist_ingestion_run_history(self, command): + raise RuntimeError( + "database failed dsn=postgresql://user:secret@localhost/db token=abc" + ) + + +class _FakeHistoryRepository: + def __init__(self, *, status: str = "declared", issue_message: str = "") -> None: + from carbonfactor_parser.persistence.ingestion_run_history import ( + ParserIngestionRunHistoryIssue, + ParserIngestionRunHistoryPersistResult, + ParserIngestionRunHistoryStatus, + ) + + self.commands = [] + self._result_status = ParserIngestionRunHistoryStatus(status) + self._issue = ( + ParserIngestionRunHistoryIssue( + code="POSTGRESQL_INGESTION_RUN_HISTORY_DATABASE_ERROR", + message=issue_message, + field_name="database", + ), + ) if issue_message else () + self._result_type = ParserIngestionRunHistoryPersistResult + + @property + def provider_name(self) -> str: + return "fake" + + def persist_ingestion_run_history(self, command): + self.commands.append(command) + return self._result_type( + provider_name=self.provider_name, + status=self._result_status, + persisted_run_count=1 if self._result_status.value == "declared" else 0, + persisted_source_result_count=( + len(command.source_results) if self._result_status.value == "declared" else 0 + ), + persisted_issue_count=( + len(command.issues) if self._result_status.value == "declared" else 0 + ), + issues=self._issue, + ) diff --git a/tests/test_defra_desnz_content_parser.py b/tests/test_defra_desnz_content_parser.py index 15049579..e5819c5b 100644 --- a/tests/test_defra_desnz_content_parser.py +++ b/tests/test_defra_desnz_content_parser.py @@ -1,9 +1,12 @@ import builtins +import json import sqlite3 import urllib.request +from decimal import Decimal from carbonfactor_parser.parsers import ( DEFRA_DESNZ_MINIMAL_CONTENT_HEADER, + DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER, ParserExecutionResult, ParserExecutionResultStatus, create_parser_file_content_input, @@ -11,10 +14,17 @@ ) +FIXTURE_DIR = "tests/fixtures/source_documents/defra_desnz" +PARITY_EXPECTATIONS = ( + "tests/fixtures/parity/defra_desnz_normalized_output_expectations.json" +) + + def _content_input( *, content: str | bytes = "factor_id,factor_name,unit\nF1,Electricity,kWh\n", - artifact_reference: str = "data/source-acquisition/defra_desnz/source.csv", + artifact_reference: str = f"{FIXTURE_DIR}/defra_desnz_normalized_factors.csv", + checksum_sha256: str = "c" * 64, ): return create_parser_file_content_input( source_family="defra_desnz", @@ -23,10 +33,30 @@ def _content_input( content_type="text/csv", format_hint="csv", artifact_reference=artifact_reference, - checksum_sha256="a" * 64, + checksum_sha256=checksum_sha256, ) +def _fixture_content(name: str) -> str: + with open(f"{FIXTURE_DIR}/{name}", encoding="utf-8") as fixture: + return fixture.read() + + +def _parity_expectations() -> dict[str, object]: + with open(PARITY_EXPECTATIONS, encoding="utf-8") as fixture: + return json.load(fixture) + + +def _canonical_fields(raw_fields: dict[str, object]) -> list[list[str | None]]: + expected_keys = [ + key for key, _ in _parity_expectations()["sample_rows"][0]["fields"] + ] + return [ + [key, None if raw_fields[key] is None else str(raw_fields[key])] + for key in expected_keys + ] + + def test_defra_desnz_minimal_content_header_is_deterministic() -> None: assert DEFRA_DESNZ_MINIMAL_CONTENT_HEADER == ( "factor_id", @@ -66,6 +96,87 @@ def test_valid_in_memory_defra_desnz_content_returns_success() -> None: assert result.raw_record_payload.records[0].row_number == 2 +def test_valid_normalized_defra_desnz_content_returns_success() -> None: + content_input = _content_input( + content=_fixture_content("defra_desnz_normalized_factors.csv"), + ) + + result = parse_defra_desnz_file_content(content_input) + + assert result.status == ParserExecutionResultStatus.SUCCESS + assert result.parsed_record_count == 2 + assert result.issues == () + assert result.parser_metadata == { + "parser_kind": "defra_desnz_normalized_csv_extraction", + "is_real_source_parser": True, + "normalization_executed": True, + } + assert result.raw_record_payload is not None + record = result.raw_record_payload.records[0] + assert record.raw_fields == { + "source_family": "defra_desnz", + "source_id": "defra_desnz", + "source_year": 2024, + "source_version": "conversion-factors-2024", + "factor_id": "DEFRA-2024-ELEC", + "factor_name": "Electricity generated", + "factor_value": Decimal("0.20705"), + "unit": "kWh", + "category": "Energy", + "subcategory": "Electricity", + "activity": "Generated", + "greenhouse_gas": "CO2e", + "provenance_artifact_reference": ( + "tests/fixtures/source_documents/defra_desnz/" + "defra_desnz_normalized_factors.csv" + ), + "provenance_checksum_algorithm": "sha256", + "provenance_checksum_value": "c" * 64, + "provenance_row_number": 2, + "provenance": "worksheet:UK electricity row 10", + "source_family_master_id": ( + "defra_master_2024_conversion-factors-2024_DEFRA-2024-ELEC" + ), + "source_family_detail_id": ( + "defra_detail_2024_conversion-factors-2024_DEFRA-2024-ELEC" + ), + "master_external_key": "2024:conversion-factors-2024:DEFRA-2024-ELEC", + "detail_external_key": "DEFRA-2024-ELEC:kWh:CO2e", + } + assert record.source_context == { + "artifact_reference": ( + "tests/fixtures/source_documents/defra_desnz/" + "defra_desnz_normalized_factors.csv" + ), + "source_year": "2024", + "source_version": "conversion-factors-2024", + "provenance": "worksheet:UK electricity row 10", + } + + +def test_valid_defra_desnz_content_matches_shared_parity_expectations() -> None: + expectations = _parity_expectations() + + result = parse_defra_desnz_file_content( + _content_input(content=_fixture_content("defra_desnz_normalized_factors.csv")), + ) + + assert result.status.value == expectations["sample_status"]["python"] + assert tuple(issue.code for issue in result.issues) == tuple( + expectations["sample_issue_codes"], + ) + assert result.raw_record_payload is not None + assert result.parsed_record_count == len(expectations["sample_rows"]) + + for record, expected_row in zip( + result.raw_record_payload.records, + expectations["sample_rows"], + strict=True, + ): + assert record.row_number == expected_row["source_row_number"] + assert _canonical_fields(record.raw_fields) == expected_row["fields"] + + def test_parsed_record_count_is_deterministic() -> None: content_input = _content_input( content=( @@ -129,7 +240,9 @@ def test_invalid_header_returns_failed_issue() -> None: def test_invalid_row_returns_failed_issue() -> None: result = parse_defra_desnz_file_content( - _content_input(content="factor_id,factor_name,unit\nF1,Electricity,kWh,extra\n"), + _content_input( + content="factor_id,factor_name,unit\nF1,Electricity,kWh,extra\n", + ), ) assert result.status == ParserExecutionResultStatus.FAILED @@ -137,6 +250,43 @@ def test_invalid_row_returns_failed_issue() -> None: assert result.issues[0].code == "DEFRA_DESNZ_CONTENT_INVALID_ROW" +def test_normalized_content_malformed_row_returns_structured_issue() -> None: + expectations = _parity_expectations() + result = parse_defra_desnz_file_content( + _content_input(content=_fixture_content("defra_desnz_malformed_factors.csv")), + ) + + assert result.status.value == expectations["malformed_status"]["python"] + assert result.parsed_record_count == 0 + assert result.raw_record_payload is None + assert tuple(issue.code for issue in result.issues) == tuple( + issue["code"] for issue in expectations["malformed_issues"] + ) + assert tuple(issue.location for issue in result.issues) == tuple( + issue["python_location"] for issue in expectations["malformed_issues"] + ) + assert result.issues[0].context == { + "row_number": 2, + "field_name": "factor_value", + "raw_value": "not-a-number", + } + + +def test_empty_normalized_defra_desnz_content_matches_documented_parity_status() -> None: + expectations = _parity_expectations() + result = parse_defra_desnz_file_content( + _content_input( + content=",".join(DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER) + "\n", + ), + ) + + assert result.status.value == expectations["empty_status"]["python"] + assert result.parsed_record_count == 0 + assert tuple(issue.code for issue in result.issues) == tuple( + expectations["empty_issue_codes"], + ) + + def test_invalid_content_identity_returns_failed_issue() -> None: content_input = create_parser_file_content_input( source_family=" ", @@ -181,7 +331,7 @@ def test_helper_preserves_metadata_without_reading_artifact_reference(tmp_path) assert result.status == ParserExecutionResultStatus.SUCCESS assert result.parser_input.artifact_reference == str(missing_artifact) - assert result.parser_input.checksum_sha256 == "a" * 64 + assert result.parser_input.checksum_sha256 == "c" * 64 assert result.raw_record_payload is not None assert result.raw_record_payload.source_context == { "artifact_reference": str(missing_artifact), diff --git a/tests/test_defra_desnz_fixture_manifest.py b/tests/test_defra_desnz_fixture_manifest.py index cf3fccbc..f7037867 100644 --- a/tests/test_defra_desnz_fixture_manifest.py +++ b/tests/test_defra_desnz_fixture_manifest.py @@ -14,6 +14,17 @@ Path(__file__).resolve().parents[0] / "fixtures" / "source_documents" / "defra_desnz" ) +EXPECTED_FIXTURE_FILE_NAMES = [ + "defra_desnz_malformed_factors.csv", + "defra_desnz_metadata.json", + "defra_desnz_normalized_factors.csv", + "defra_desnz_sample_factors.csv", +] + +EXPECTED_FIXTURE_SOURCE_NAMES = [ + f"defra_desnz:{file_name}" for file_name in EXPECTED_FIXTURE_FILE_NAMES +] + def test_manifest_can_be_created_from_no_documents() -> None: manifest = build_defra_desnz_fixture_manifest(()) @@ -31,11 +42,8 @@ def test_manifest_can_be_created_from_defra_desnz_fixture_documents() -> None: manifest = build_defra_desnz_fixture_manifest(documents) - assert manifest.document_count == 2 - assert [entry.file_name for entry in manifest.entries] == [ - "defra_desnz_metadata.json", - "defra_desnz_sample_factors.csv", - ] + assert manifest.document_count == 4 + assert [entry.file_name for entry in manifest.entries] == EXPECTED_FIXTURE_FILE_NAMES def test_manifest_entry_ordering_is_deterministic() -> None: @@ -106,8 +114,9 @@ def test_manifest_uses_discovered_document_metadata() -> None: assert [entry.source_family for entry in manifest.entries] == [ SourceFamily.DEFRA_DESNZ, SourceFamily.DEFRA_DESNZ, + SourceFamily.DEFRA_DESNZ, + SourceFamily.DEFRA_DESNZ, ] - assert [entry.source_name for entry in manifest.entries] == [ - "defra_desnz:defra_desnz_metadata.json", - "defra_desnz:defra_desnz_sample_factors.csv", - ] + assert [entry.source_name for entry in manifest.entries] == ( + EXPECTED_FIXTURE_SOURCE_NAMES + ) diff --git a/tests/test_defra_desnz_fixture_manifest_example.py b/tests/test_defra_desnz_fixture_manifest_example.py index 1743a0f1..6c0e7ca5 100644 --- a/tests/test_defra_desnz_fixture_manifest_example.py +++ b/tests/test_defra_desnz_fixture_manifest_example.py @@ -39,7 +39,7 @@ def test_defra_desnz_manifest_example_uses_artificial_fixture_directory() -> Non def test_defra_desnz_manifest_example_has_expected_document_count() -> None: manifest = build_defra_desnz_fixture_manifest_example() - assert manifest["document_count"] == 2 + assert manifest["document_count"] == 4 assert manifest["warnings"] == () @@ -57,7 +57,9 @@ def test_defra_desnz_manifest_example_derives_file_names_and_extensions() -> Non (entry["file_name"], entry["file_extension"]) for entry in manifest["entries"] ] == [ + ("defra_desnz_malformed_factors.csv", ".csv"), ("defra_desnz_metadata.json", ".json"), + ("defra_desnz_normalized_factors.csv", ".csv"), ("defra_desnz_sample_factors.csv", ".csv"), ] @@ -68,7 +70,9 @@ def test_defra_desnz_manifest_example_keeps_source_metadata() -> None: assert manifest["source_family"] == SourceFamily.DEFRA_DESNZ.value assert manifest["source_name"] == "defra_desnz" assert [entry["source_name"] for entry in manifest["entries"]] == [ + "defra_desnz:defra_desnz_malformed_factors.csv", "defra_desnz:defra_desnz_metadata.json", + "defra_desnz:defra_desnz_normalized_factors.csv", "defra_desnz:defra_desnz_sample_factors.csv", ] diff --git a/tests/test_defra_desnz_normalization_mapper.py b/tests/test_defra_desnz_normalization_mapper.py index f7422154..58491bb9 100644 --- a/tests/test_defra_desnz_normalization_mapper.py +++ b/tests/test_defra_desnz_normalization_mapper.py @@ -4,6 +4,7 @@ from carbonfactor_parser.normalization import ( DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS, + DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS, DefraDesnzNormalizationMappingResult, DefraDesnzNormalizationMappingStatus, NormalizationInput, @@ -15,6 +16,7 @@ map_defra_desnz_normalization_input_record, ) from carbonfactor_parser.parsers import ( + DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER, ParserExecutionResultStatus, create_parser_file_content_input, parse_defra_desnz_file_content, @@ -92,6 +94,64 @@ def test_expected_raw_fields_are_copied_deterministically() -> None: assert "unused_raw_field" not in dict(record_fields) +def test_normalized_defra_desnz_record_maps_to_persistence_ready_fields() -> None: + normalized_input = _normalization_input( + ( + NormalizationInputRecord( + source_family="defra_desnz", + source_id="defra_desnz", + record_index=1, + row_number=2, + raw_fields={ + "source_year": "2024", + "source_version": "conversion-factors-2024", + "category": "Energy", + "subcategory": "Electricity", + "activity": "Generated", + "factor_id": "DEFRA-2024-ELEC", + "factor_name": "Electricity generated", + "factor_value": "0.20705", + "unit": "kWh", + "greenhouse_gas": "CO2e", + "provenance": "worksheet:UK electricity row 10", + }, + source_context={"artifact_reference": "memory://defra"}, + ), + ), + ) + + result = map_defra_desnz_normalization_input(normalized_input) + + assert result.status == DefraDesnzNormalizationMappingStatus.SUCCESS + record = result.normalization_result.records[0] + assert record.record_id == ( + "defra_desnz:defra_desnz:2024:" + "conversion-factors-2024:DEFRA-2024-ELEC" + ) + assert record.is_artificial is False + assert record.source_reference == "memory://defra" + assert DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS == tuple( + field_name for field_name, _ in record.fields + ) + assert record.fields == ( + ("source_family", "defra_desnz"), + ("source_id", "defra_desnz"), + ("source_year", "2024"), + ("source_version", "conversion-factors-2024"), + ("record_index", 1), + ("row_number", 2), + ("factor_id", "DEFRA-2024-ELEC"), + ("factor_name", "Electricity generated"), + ("factor_value", 0.20705), + ("unit", "kWh"), + ("category", "Energy"), + ("subcategory", "Electricity"), + ("activity", "Generated"), + ("greenhouse_gas", "CO2e"), + ("provenance", "worksheet:UK electricity row 10"), + ) + + def test_single_record_mapping_helper_returns_success_result() -> None: result = map_defra_desnz_normalization_input_record( _normalization_input().records[0], @@ -125,6 +185,36 @@ def test_missing_required_raw_field_returns_failed_result_with_issue() -> None: ) +def test_invalid_normalized_factor_value_returns_failed_result_with_issue() -> None: + bad_record = NormalizationInputRecord( + source_family="defra_desnz", + source_id="defra_desnz", + record_index=1, + row_number=2, + raw_fields={ + "source_year": "2024", + "source_version": "conversion-factors-2024", + "category": "Energy", + "factor_id": "DEFRA-2024-ELEC", + "factor_name": "Electricity generated", + "factor_value": "not-a-number", + "unit": "kWh", + "provenance": "worksheet:UK electricity row 10", + }, + ) + + result = map_defra_desnz_normalization_input(_normalization_input((bad_record,))) + + assert result.status == DefraDesnzNormalizationMappingStatus.FAILED + assert result.normalization_result.records == () + assert result.normalization_result.issues[0].code == ( + "DEFRA_DESNZ_NORMALIZATION_INVALID_FACTOR_VALUE" + ) + assert result.normalization_result.issues[0].location == ( + "records[1].raw_fields.factor_value" + ) + + def test_empty_input_returns_no_records_result_with_issue() -> None: result = map_defra_desnz_normalization_input(_normalization_input(records=())) @@ -239,3 +329,51 @@ def test_in_memory_parser_to_minimal_normalization_mapping_path() -> None: ("factor_name", "Natural gas"), ("unit", "m3"), ) + + +def test_in_memory_parser_to_normalized_defra_desnz_mapping_path() -> None: + parser_input = create_parser_file_content_input( + source_family="defra_desnz", + source_id="defra_desnz", + content=( + ",".join(DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER) + + "\n" + + "2024,conversion-factors-2024,Energy,Electricity,Generated," + + "DEFRA-2024-ELEC,Electricity generated,0.20705,kWh,CO2e," + + "worksheet:UK electricity row 10\n" + ), + content_type="text/csv", + artifact_reference="memory://defra-desnz-content", + ) + + parser_result = parse_defra_desnz_file_content(parser_input) + handoff_result = build_parser_execution_normalization_handoff(parser_result) + input_result = build_normalization_input_from_parser_execution_handoff( + handoff_result, + ) + + assert parser_result.status == ParserExecutionResultStatus.SUCCESS + assert input_result.normalization_input is not None + + mapping_result = map_defra_desnz_normalization_input( + input_result.normalization_input, + ) + + assert mapping_result.status == DefraDesnzNormalizationMappingStatus.SUCCESS + assert mapping_result.normalization_result.records[0].fields == ( + ("source_family", "defra_desnz"), + ("source_id", "defra_desnz"), + ("source_year", "2024"), + ("source_version", "conversion-factors-2024"), + ("record_index", 1), + ("row_number", 2), + ("factor_id", "DEFRA-2024-ELEC"), + ("factor_name", "Electricity generated"), + ("factor_value", 0.20705), + ("unit", "kWh"), + ("category", "Energy"), + ("subcategory", "Electricity"), + ("activity", "Generated"), + ("greenhouse_gas", "CO2e"), + ("provenance", "worksheet:UK electricity row 10"), + ) diff --git a/tests/test_defra_desnz_production_e2e.py b/tests/test_defra_desnz_production_e2e.py new file mode 100644 index 00000000..df803b59 --- /dev/null +++ b/tests/test_defra_desnz_production_e2e.py @@ -0,0 +1,567 @@ +from __future__ import annotations + +import os +from pathlib import Path +import uuid +from zipfile import ZipFile + +import pytest + +from carbonfactor_parser.persistence import ( + POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR, + POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR, +) +from carbonfactor_parser.persistence.postgresql_normalized_factor_repository import ( + PostgreSQLNormalizedFactorRuntimeRepository, +) +from carbonfactor_parser.persistence.postgresql_runtime_schema_bootstrap import ( + bootstrap_postgresql_phase1_schema, +) +from carbonfactor_parser.persistence.postgresql_year_state_repository import ( + PostgreSQLSourceFamilyYearStateRepository, +) +from carbonfactor_parser.pipeline.defra_desnz_production_e2e import ( + DEFRA_DESNZ_SOURCE_FAMILY, + DefraDesnzPhase2ValidationBoundary, + DefraDesnzProductionParserBoundary, + DefraDesnzProductionSourceAdapter, + DefraDesnzSourceYear, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2ESourceYearDiscoveryRequest, + ProductionE2ESourceYearDiscoveryStatus, + ProductionE2EYearFamilyStatus, + ProductionE2EYearOrchestratorDependencies, + ProductionE2EYearOrchestratorRequest, + ProductionE2EYearRunStatus, + run_production_e2e_year_orchestrator, +) + + +def test_defra_desnz_2024_first_run_downloads_parses_validates_and_inserts( + tmp_path: Path, +) -> None: + source_bytes = _flat_file_csv(year=2024) + adapter = _adapter(tmp_path, {2024: source_bytes}) + insert_repository = _RecordingInsertRepository() + year_state = _YearStateRepository() + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-014-defra-2024", + enabled_source_families=(DEFRA_DESNZ_SOURCE_FAMILY,), + ), + _dependencies(year_state, adapter, insert_repository), + ) + + family = result.family_results[0] + assert result.status is ProductionE2EYearRunStatus.COMPLETED + assert family.status is ProductionE2EYearFamilyStatus.COMPLETED + assert family.year_state.target_year == 2024 + assert family.download_result is not None + assert family.download_result.artifact is not None + assert Path(family.download_result.artifact.artifact_reference).exists() + assert Path( + f"{family.download_result.artifact.artifact_reference}.metadata.json", + ).exists() + assert family.parsed_row_count == 1 + assert family.validation_result is not None + assert family.validation_result.blocking_error_count == 0 + assert insert_repository.inserted_batches[0].rows[0].normalized_fields + assert year_state.recorded_years == (("defra", 2024),) + + +def test_defra_desnz_next_run_after_2024_targets_2025(tmp_path: Path) -> None: + adapter = _adapter(tmp_path, {2025: _flat_file_csv(year=2025)}) + year_state = _YearStateRepository({"defra": 2024}) + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-014-defra-2025", + enabled_source_families=(DEFRA_DESNZ_SOURCE_FAMILY,), + ), + _dependencies(year_state, adapter, _RecordingInsertRepository()), + ) + + family = result.family_results[0] + assert family.status is ProductionE2EYearFamilyStatus.COMPLETED + assert family.year_state.latest_year == 2024 + assert family.year_state.target_year == 2025 + assert year_state.recorded_years == (("defra", 2025),) + + +def test_defra_desnz_2026_and_2027_unavailable_noop_safely(tmp_path: Path) -> None: + adapter = _adapter(tmp_path, {}) + insert_repository = _RecordingInsertRepository() + year_state = _YearStateRepository({"defra": 2025}) + + first = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-014-defra-2026", + enabled_source_families=(DEFRA_DESNZ_SOURCE_FAMILY,), + ), + _dependencies(year_state, adapter, insert_repository), + ) + year_state.latest_years["defra"] = 2026 + second = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-014-defra-2027", + enabled_source_families=(DEFRA_DESNZ_SOURCE_FAMILY,), + ), + _dependencies(year_state, adapter, insert_repository), + ) + + assert first.family_results[0].status is ( + ProductionE2EYearFamilyStatus.NO_AVAILABLE_SOURCE_YEAR + ) + assert first.family_results[0].year_state.target_year == 2026 + assert second.family_results[0].status is ( + ProductionE2EYearFamilyStatus.NO_AVAILABLE_SOURCE_YEAR + ) + assert second.family_results[0].year_state.target_year == 2027 + assert insert_repository.inserted_batches == () + assert year_state.recorded_years == () + + +def test_defra_desnz_discovery_marks_unmapped_year_unavailable( + tmp_path: Path, +) -> None: + adapter = DefraDesnzProductionSourceAdapter( + target_root=tmp_path / "archive", + source_years={}, + transport=lambda _: pytest.fail("unavailable discovery must not download"), + ) + + result = adapter.discover_target_year( + ProductionE2ESourceYearDiscoveryRequest( + source_family=DEFRA_DESNZ_SOURCE_FAMILY, + target_year=2026, + run_id="ph-023-defra-unmapped", + ), + ) + + assert ( + result.status + is ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ) + assert result.artifact_reference is None + assert result.reason_code == "defra_desnz_target_year_not_in_availability_map" + assert result.metadata is not None + assert ( + result.metadata["availability_strategy"] + == "govuk_publication_flat_file_link" + ) + + +def test_defra_desnz_live_discovery_returns_govuk_flat_file_artifact( + tmp_path: Path, +) -> None: + page = ( + 'Conversion factors 2025: flat file' + "" + ).encode("utf-8") + adapter = DefraDesnzProductionSourceAdapter( + target_root=tmp_path / "archive", + source_years={ + 2025: DefraDesnzSourceYear( + year=2025, + publication_url="https://www.gov.uk/example/2025", + artifact_url="", + title="Conversion factors 2025: flat file", + version_label="2025-test", + ), + }, + transport=lambda _: page, + ) + + result = adapter.discover_target_year( + ProductionE2ESourceYearDiscoveryRequest( + source_family=DEFRA_DESNZ_SOURCE_FAMILY, + target_year=2025, + run_id="ph-023-defra-available", + ), + ) + + assert ( + result.status + is ProductionE2ESourceYearDiscoveryStatus.SOURCE_YEAR_AVAILABLE + ) + assert result.artifact_reference == ( + "https://assets.publishing.service.gov.uk/media/defra-2025-flat-file.xlsx" + ) + assert result.metadata is not None + assert ( + result.metadata["availability_strategy"] + == "govuk_publication_flat_file_link" + ) + assert result.metadata["format_hint"] == "xlsx" + + +def test_defra_desnz_live_discovery_failure_is_redacted_and_user_readable( + tmp_path: Path, +) -> None: + def transport(_: str) -> bytes: + raise RuntimeError("GET https://token@example.invalid/private failed") + + adapter = DefraDesnzProductionSourceAdapter( + target_root=tmp_path / "archive", + source_years={ + 2024: DefraDesnzSourceYear( + year=2024, + publication_url="https://www.gov.uk/example/2024", + artifact_url="", + title="Conversion factors 2024: flat file", + version_label="2024-test", + ), + }, + transport=transport, + ) + + result = adapter.discover_target_year( + ProductionE2ESourceYearDiscoveryRequest( + source_family=DEFRA_DESNZ_SOURCE_FAMILY, + target_year=2024, + run_id="ph-023-defra-failure", + ), + ) + + assert ( + result.status + is ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ) + assert result.reason_code == "defra_desnz_flat_file_link_not_found" + assert result.metadata is not None + assert result.metadata["discovery_error_type"] == "RuntimeError" + assert "token@" not in str(result.metadata["discovery_error_message"]) + assert "@" in str(result.metadata["discovery_error_message"]) + assert "user_message" in result.metadata + + +def test_defra_desnz_live_discovery_plain_url_failure_redacts_userinfo( + tmp_path: Path, +) -> None: + def transport(_: str) -> bytes: + raise RuntimeError("https://user:secret@example.invalid/private") + + adapter = DefraDesnzProductionSourceAdapter( + target_root=tmp_path / "archive", + source_years={ + 2024: DefraDesnzSourceYear( + year=2024, + publication_url="https://www.gov.uk/example/2024", + artifact_url="", + title="Conversion factors 2024: flat file", + version_label="2024-test", + ), + }, + transport=transport, + ) + + result = adapter.discover_target_year( + ProductionE2ESourceYearDiscoveryRequest( + source_family=DEFRA_DESNZ_SOURCE_FAMILY, + target_year=2024, + run_id="ph-023-defra-plain-url-failure", + ), + ) + + assert ( + result.status + is ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ) + assert result.metadata is not None + discovery_error_message = str(result.metadata["discovery_error_message"]) + assert discovery_error_message == "https://example.invalid/..." + assert "user" not in discovery_error_message + assert "secret" not in discovery_error_message + assert "user:secret@" not in discovery_error_message + + +def test_defra_desnz_repeated_run_is_insert_idempotent(tmp_path: Path) -> None: + adapter = _adapter(tmp_path, {2024: _flat_file_csv(year=2024)}) + insert_repository = _IdempotentInsertRepository() + + first = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-014-defra-idempotent-a", + enabled_source_families=(DEFRA_DESNZ_SOURCE_FAMILY,), + ), + _dependencies(_YearStateRepository(), adapter, insert_repository), + ) + second = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-014-defra-idempotent-b", + enabled_source_families=(DEFRA_DESNZ_SOURCE_FAMILY,), + ), + _dependencies(_YearStateRepository(), adapter, insert_repository), + ) + + assert first.family_results[0].insert_summary is not None + assert second.family_results[0].insert_summary is not None + assert first.family_results[0].insert_summary.inserted == 1 + assert second.family_results[0].insert_summary.inserted == 0 + assert second.family_results[0].insert_summary.skipped_duplicate == 1 + + +def test_defra_desnz_parser_reads_xlsx_flat_file(tmp_path: Path) -> None: + xlsx_path = tmp_path / "defra-flat.xlsx" + _write_minimal_xlsx(xlsx_path) + artifact = _downloaded_artifact(xlsx_path, 2024) + + batch = DefraDesnzProductionParserBoundary().parse(artifact) + + assert batch.row_count == 1 + fields = dict(batch.rows[0].normalized_fields) + assert fields["source_year"] == 2024 + assert fields["factor_value"] == "0.20705" + assert fields["unit"] == "kWh" + + +@pytest.mark.postgresql_integration +def test_docker_postgresql_defra_desnz_production_e2e_integration( + tmp_path: Path, +) -> None: + if os.getenv(POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR) != "1": + pytest.skip("PostgreSQL integration test opt-in is not enabled.") + dsn = os.getenv(POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR) + if not dsn: + pytest.skip("PostgreSQL integration test DSN was not provided.") + + import psycopg + + schema_name = f"carbonops_ph014_{uuid.uuid4().hex}" + with psycopg.connect(dsn) as connection: + connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") + connection.execute(f"SET search_path TO {schema_name}") + bootstrap_postgresql_phase1_schema(connection) + + adapter = _adapter(tmp_path, {2024: _flat_file_csv(year=2024)}) + dependencies = ProductionE2EYearOrchestratorDependencies( + year_state_repository=PostgreSQLSourceFamilyYearStateRepository( + connection, + ), + source_adapters={DEFRA_DESNZ_SOURCE_FAMILY: adapter}, + parser_boundaries={ + DEFRA_DESNZ_SOURCE_FAMILY: DefraDesnzProductionParserBoundary(), + }, + validation_boundary=DefraDesnzPhase2ValidationBoundary(), + insert_repository=PostgreSQLNormalizedFactorRuntimeRepository(connection), + ) + + first = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-014-defra-postgresql-a", + enabled_source_families=(DEFRA_DESNZ_SOURCE_FAMILY,), + ), + dependencies, + ) + connection.execute("DELETE FROM source_family_year_states") + second = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-014-defra-postgresql-b", + enabled_source_families=(DEFRA_DESNZ_SOURCE_FAMILY,), + ), + dependencies, + ) + count = connection.execute( + "SELECT COUNT(*) FROM normalized_factor_records", + ).fetchone()[0] + + assert first.family_results[0].insert_summary is not None + assert second.family_results[0].insert_summary is not None + assert first.family_results[0].insert_summary.inserted == 1 + assert second.family_results[0].insert_summary.inserted == 0 + assert second.family_results[0].insert_summary.skipped_duplicate == 1 + assert count == 1 + + +class _YearStateRepository: + def __init__(self, latest_years: dict[str, int] | None = None) -> None: + self.latest_years = dict(latest_years or {}) + self._recorded_years: list[tuple[str, int]] = [] + + @property + def recorded_years(self) -> tuple[tuple[str, int], ...]: + return tuple(self._recorded_years) + + def latest_ingested_year(self, source_family: str) -> int | None: + return self.latest_years.get(source_family) + + def record_ingested_year(self, source_family: str, ingested_year: int) -> None: + self.latest_years[source_family] = ingested_year + self._recorded_years.append((source_family, ingested_year)) + + +class _RecordingInsertRepository: + def __init__(self) -> None: + self._inserted_batches = [] + + @property + def inserted_batches(self): + return tuple(self._inserted_batches) + + def insert_normalized_factor_records(self, batch): + self._inserted_batches.append(batch) + return _InsertResult("inserted", batch.row_count, batch.row_count) + + +class _IdempotentInsertRepository: + def __init__(self) -> None: + self._seen = set() + + def insert_normalized_factor_records(self, batch): + inserted = 0 + skipped = 0 + for row in batch.rows: + if row.row_id in self._seen: + skipped += 1 + else: + inserted += 1 + self._seen.add(row.row_id) + return _InsertResult("inserted", batch.row_count, inserted, skipped) + + +class _InsertResult: + def __init__( + self, + status: str, + attempted: int, + inserted: int, + skipped_duplicate: int = 0, + ) -> None: + self.status = status + self.attempted = attempted + self.inserted = inserted + self.skipped_duplicate = skipped_duplicate + self.failed = 0 + self.validation_error_count = 0 + + +def _dependencies( + year_state: _YearStateRepository, + adapter: DefraDesnzProductionSourceAdapter, + insert_repository, +) -> ProductionE2EYearOrchestratorDependencies: + return ProductionE2EYearOrchestratorDependencies( + year_state_repository=year_state, + source_adapters={DEFRA_DESNZ_SOURCE_FAMILY: adapter}, + parser_boundaries={ + DEFRA_DESNZ_SOURCE_FAMILY: DefraDesnzProductionParserBoundary(), + }, + validation_boundary=DefraDesnzPhase2ValidationBoundary(), + insert_repository=insert_repository, + ) + + +def _adapter( + tmp_path: Path, + source_bytes_by_year: dict[int, bytes], +) -> DefraDesnzProductionSourceAdapter: + years = { + year: DefraDesnzSourceYear( + year=year, + publication_url=f"https://example.invalid/defra/{year}", + artifact_url=f"https://example.invalid/defra/{year}.csv", + title=f"Conversion factors {year}: flat file", + version_label=f"{year}-test", + content_type="text/csv", + format_hint="csv", + ) + for year in source_bytes_by_year + } + + def transport(uri: str) -> bytes: + for year, content in source_bytes_by_year.items(): + if uri.endswith(f"/{year}.csv"): + return content + raise FileNotFoundError(uri) + + return DefraDesnzProductionSourceAdapter( + target_root=tmp_path / "archive", + source_years=years, + transport=transport, + ) + + +def _flat_file_csv(*, year: int) -> bytes: + return ( + "factor_id,factor_name,category,factor_value,unit,source_year,row_number\n" + f"electricity-{year},Electricity generated,UK electricity,0.20705,kWh,{year},2\n" + ).encode("utf-8") + + +def _downloaded_artifact(path: Path, year: int): + from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EDownloadedArtifact, + ) + + return ProductionE2EDownloadedArtifact( + source_family=DEFRA_DESNZ_SOURCE_FAMILY, + source_year=year, + artifact_reference=str(path), + checksum_sha256="a" * 64, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + format_hint="xlsx", + metadata={"version_label": f"{year}-test"}, + ) + + +def _write_minimal_xlsx(path: Path) -> None: + with ZipFile(path, "w") as archive: + archive.writestr( + "[Content_Types].xml", + ( + '' + '' + '' + '' + "" + ), + ) + archive.writestr( + "xl/workbook.xml", + ( + '' + '' + "" + ), + ) + archive.writestr( + "xl/_rels/workbook.xml.rels", + ( + '' + '' + "" + ), + ) + archive.writestr( + "xl/worksheets/sheet1.xml", + ( + '' + "" + "" + 'factor_id' + 'factor_name' + 'category' + 'factor_value' + 'unit' + "" + "" + 'electricity-2024' + 'Electricity generated' + 'UK electricity' + '0.20705' + 'kWh' + "" + "" + "" + ), + ) diff --git a/tests/test_defra_desnz_source_adapter.py b/tests/test_defra_desnz_source_adapter.py index fdd1a1cf..d082b51e 100644 --- a/tests/test_defra_desnz_source_adapter.py +++ b/tests/test_defra_desnz_source_adapter.py @@ -12,6 +12,17 @@ Path(__file__).resolve().parents[0] / "fixtures" / "source_documents" / "defra_desnz" ) +EXPECTED_FIXTURE_FILE_NAMES = [ + "defra_desnz_malformed_factors.csv", + "defra_desnz_metadata.json", + "defra_desnz_normalized_factors.csv", + "defra_desnz_sample_factors.csv", +] + +EXPECTED_FIXTURE_SOURCE_NAMES = [ + f"defra_desnz:{file_name}" for file_name in EXPECTED_FIXTURE_FILE_NAMES +] + def test_defra_desnz_source_adapter_is_importable() -> None: adapter = DefraDesnzSourceAdapter(directory_path=FIXTURE_DIRECTORY) @@ -25,13 +36,11 @@ def test_defra_desnz_source_adapter_discovers_deterministic_fixture_documents() result = adapter.discover() assert result.warnings == () - assert [document.source_name for document in result.documents] == [ - "defra_desnz:defra_desnz_metadata.json", - "defra_desnz:defra_desnz_sample_factors.csv", - ] + assert [document.source_name for document in result.documents] == ( + EXPECTED_FIXTURE_SOURCE_NAMES + ) assert [Path(document.file_reference or "").name for document in result.documents] == [ - "defra_desnz_metadata.json", - "defra_desnz_sample_factors.csv", + *EXPECTED_FIXTURE_FILE_NAMES ] @@ -117,10 +126,10 @@ def test_defra_desnz_source_adapter_metadata_is_consistent() -> None: document = adapter.discover().documents[0] assert document.source_family == SourceFamily.DEFRA_DESNZ - assert document.source_name == "defra_desnz:defra_desnz_sample_factors.csv" + assert document.source_name == "defra_desnz:defra_desnz_malformed_factors.csv" assert document.source_url is None assert document.file_reference == ( - str(FIXTURE_DIRECTORY / "defra_desnz_sample_factors.csv") + str(FIXTURE_DIRECTORY / "defra_desnz_malformed_factors.csv") ) @@ -138,7 +147,7 @@ def test_defra_desnz_source_adapter_works_with_summary_helper() -> None: summary = summarize_source_adapter_result(adapter.discover()) - assert summary.document_count == 2 + assert summary.document_count == 4 assert summary.file_extensions == (".csv", ".json") assert summary.source_families == (SourceFamily.DEFRA_DESNZ,) assert summary.has_documents is True diff --git a/tests/test_defra_source_discovery_boundary.py b/tests/test_defra_source_discovery_boundary.py index ca969ad6..76c82fb3 100644 --- a/tests/test_defra_source_discovery_boundary.py +++ b/tests/test_defra_source_discovery_boundary.py @@ -256,6 +256,41 @@ def test_result_validation_rejects_side_effect_flags() -> None: ) +def test_result_validation_rejects_declared_results_with_issue_metadata() -> None: + result = replace( + create_defra_source_discovery_result(), + issues=( + DEFRASourceDiscoveryIssue( + code="DEFRA_SOURCE_DISCOVERY_TEST_ISSUE", + message="test issue", + field_name="test", + ), + ), + ) + + validation = validate_defra_source_discovery_result(result) + + assert validation.is_valid is False + assert _issue_codes(validation.issues) == ( + "DEFRA_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + "DEFRA_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + ) + + +def test_result_validation_rejects_undefined_status() -> None: + result = replace( + create_defra_source_discovery_result(), + status="declared", # type: ignore[arg-type] + ) + + validation = validate_defra_source_discovery_result(result) + + assert validation.is_valid is False + assert _issue_codes(validation.issues) == ( + "DEFRA_SOURCE_DISCOVERY_RESULT_INVALID_STATUS", + ) + + def test_defra_discovery_contract_dataclasses_are_immutable() -> None: request = create_defra_source_discovery_request() result = create_defra_source_discovery_result() diff --git a/tests/test_defra_source_download_execution_boundary.py b/tests/test_defra_source_download_execution_boundary.py index 53e27e90..b3e0ed02 100644 --- a/tests/test_defra_source_download_execution_boundary.py +++ b/tests/test_defra_source_download_execution_boundary.py @@ -171,6 +171,21 @@ def test_invalid_download_requests_fail_closed_before_transport( "s3://bucket/defra.xlsx", "DEFRA_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI", ), + ( + "source_reference_uri", + "defra/conversion-factors.xlsx", + "DEFRA_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME", + ), + ( + "source_reference_uri", + "://defra/conversion-factors.xlsx", + "DEFRA_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + ), + ( + "source_reference_uri", + "https:///defra.xlsx", + "DEFRA_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + ), ( "target_root", "relative/root", @@ -439,6 +454,43 @@ def test_transport_errors_and_empty_content_are_failed_results( ) +def test_transport_response_validation_fails_closed(tmp_path: Path) -> None: + missing_response = execute_defra_source_download( + _valid_request(tmp_path / "missing"), + lambda _: None, # type: ignore[return-value] + ) + missing_content_response = execute_defra_source_download( + _valid_request(tmp_path / "missing-content"), + lambda _: DEFRASourceDownloadTransportResponse( + content=None, # type: ignore[arg-type] + ), + ) + blank_metadata_response = execute_defra_source_download( + _valid_request(tmp_path / "blank-metadata"), + lambda _: DEFRASourceDownloadTransportResponse( + content=b"content", + content_type=" ", + final_uri=" ", + ), + ) + + assert missing_response.status is DEFRASourceDownloadExecutionStatus.FAILED + assert _issue_codes(missing_response.issues) == ( + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING", + ) + assert ( + missing_content_response.status is DEFRASourceDownloadExecutionStatus.FAILED + ) + assert _issue_codes(missing_content_response.issues) == ( + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT", + ) + assert blank_metadata_response.status is DEFRASourceDownloadExecutionStatus.FAILED + assert _issue_codes(blank_metadata_response.issues) == ( + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "DEFRA_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + ) + + def test_download_execution_dataclasses_are_immutable(tmp_path: Path) -> None: request = _valid_request(tmp_path) result = execute_defra_source_download( @@ -494,6 +546,27 @@ def test_result_validation_rejects_side_effect_flags(tmp_path: Path) -> None: ) +def test_result_validation_rejects_invalid_status(tmp_path: Path) -> None: + result = DEFRASourceDownloadExecutionResult( + status="unknown", # type: ignore[arg-type] + request=_valid_request(tmp_path), + issues=( + DEFRASourceDownloadExecutionIssue( + code="DEFRA_SOURCE_DOWNLOAD_TEST_ISSUE", + message="test issue.", + field_name="status", + ), + ), + ) + + validation = validate_defra_source_download_execution_result(result) + + assert validation.is_valid is False + assert "DEFRA_SOURCE_DOWNLOAD_RESULT_INVALID_STATUS" in _issue_codes( + validation.issues + ) + + def test_download_execution_import_is_runtime_passive( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_diagnostics_runtime_output.py b/tests/test_diagnostics_runtime_output.py new file mode 100644 index 00000000..c9f6a2ff --- /dev/null +++ b/tests/test_diagnostics_runtime_output.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +from carbonfactor_parser.diagnostics.ingestion_runtime_events import ( + build_configured_cycle_summary_payload, + build_configured_runner_summary_payload, +) +from carbonfactor_parser.diagnostics.redaction import redact_sensitive_text +from carbonfactor_parser.pipeline.configured_cycle_runner import ( + ConfiguredCycleResult, + ConfiguredCycleRunnerResult, + ConfiguredCycleRunnerStatus, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EFailureDetail, + ProductionE2EInsertSummary, + ProductionE2EYearFamilyResult, + ProductionE2EYearFamilyStatus, + ProductionE2EYearOrchestratorRequest, + ProductionE2EYearOrchestratorResult, + ProductionE2EYearRunStatus, + ProductionE2EYearRunSummary, + ProductionE2EYearSelectionStatus, + ProductionE2EYearState, +) + + +def test_redaction_sanitizes_postgresql_uri_userinfo() -> None: + message = ( + "connect postgresql://carbonops:super-secret@example.invalid:5432/db" + "?sslmode=require&token=abc123" + ) + + redacted = redact_sensitive_text(message) + + assert "super-secret" not in redacted + assert "carbonops:super-secret" not in redacted + assert "token=abc123" not in redacted + assert "postgresql://***@example.invalid:5432/db" in redacted + assert "token=***" in redacted + + +def test_redaction_sanitizes_sensitive_assignments() -> None: + message = ( + "password=hunter2 passwd=other pwd=tiny token=tok-value secret=sec-value " + "key=api dsn=postgresql://user:pass@example.invalid/db " + "connection_string=postgresql://user:pass@example.invalid/db" + ) + + redacted = redact_sensitive_text(message) + + assert "hunter2" not in redacted + assert "other" not in redacted + assert "tiny" not in redacted + assert "tok-value" not in redacted + assert "sec-value" not in redacted + assert "api" not in redacted + assert "user:pass" not in redacted + assert "password=***" in redacted + assert "token=***" in redacted + assert "dsn=***" in redacted + assert "connection_string=***" in redacted + + +def test_redaction_sanitizes_compound_sensitive_assignments() -> None: + message = ( + "api_key=abc123 apikey=abc123 access_key=abc123 accesskey=abc123 " + "private_key=abc123 privatekey=abc123 " + "connection_uri=postgresql://user:pass@example.invalid/db " + "connectionstring=postgresql://user:pass@example.invalid/db " + "database_url=postgresql://user:pass@example.invalid/db " + "databaseurl=postgresql://user:pass@example.invalid/db" + ) + + redacted = redact_sensitive_text(message) + + assert "abc123" not in redacted + assert "user:pass" not in redacted + assert "api_key=***" in redacted + assert "access_key=***" in redacted + assert "private_key=***" in redacted + assert "connection_uri=***" in redacted + assert "database_url=***" in redacted + + +def test_redaction_sanitizes_compound_sensitive_query_parameters() -> None: + message = ( + "urls https://example.invalid/path?api_key=abc123&safe=value " + "https://example.invalid/path?private_key=abc123 " + "https://example.invalid/path?access-key=abc123" + ) + + redacted = redact_sensitive_text(message) + + assert "abc123" not in redacted + assert "api_key=***" in redacted + assert "private_key=***" in redacted + assert "access-key=***" in redacted + assert "safe=value" in redacted + + +def test_redaction_keeps_non_sensitive_query_parameters_readable() -> None: + message = "https://example.invalid/path?family=ghg_protocol&safe=value" + + redacted = redact_sensitive_text(message) + + assert redacted == message + + +def test_redaction_keeps_non_sensitive_messages_readable() -> None: + message = "download failed because source family ghg_protocol is disabled" + + assert redact_sensitive_text(message) == message + + +def test_configured_cycle_summary_payload_contains_sanitized_runtime_details() -> None: + cycle = _configured_cycle_with_secret_issue() + + payload = build_configured_cycle_summary_payload(cycle) + + assert payload["cycle_number"] == 1 + assert payload["run_id"] == "run-001" + assert payload["status"] == "completed_with_failures" + assert payload["summary"] == { + "completed_family_count": 0, + "no_available_source_year_count": 0, + "failed_family_count": 1, + "parsed_rows": 7, + "inserted": 3, + "skipped_duplicates": 4, + } + assert payload["sources"] == [ + { + "source_family": "ghg_protocol", + "target_year": 2024, + "latest_year": 2023, + "status": "failed", + "download_status": "not_run", + "parse_status": "parsed", + "parsed_rows": 7, + "master_inserted": 1, + "master_skipped": 2, + "detail_inserted": 2, + "detail_skipped": 2, + } + ] + assert payload["issues"][0]["source_family"] == "ghg_protocol" + assert payload["issues"][0]["stage"] == "parser" + assert payload["issues"][0]["code"] == "PARSER_FAILED" + assert "secret" not in str(payload) + assert "password=***" in payload["issues"][0]["message"] + + +def test_configured_cycle_summary_payload_deduplicates_flattened_failures() -> None: + cycle = _configured_cycle_with_secret_issue(include_top_level_duplicate=True) + + payload = build_configured_cycle_summary_payload(cycle) + + assert payload["issues"] == [ + { + "source_family": "ghg_protocol", + "stage": "parser", + "code": "PARSER_FAILED", + "message": "parser failed password=*** token=***", + } + ] + + +def test_configured_cycle_summary_payload_includes_top_level_only_failures() -> None: + cycle = _configured_cycle_with_secret_issue(include_top_level_only=True) + + payload = build_configured_cycle_summary_payload(cycle) + + assert [issue["code"] for issue in payload["issues"]] == [ + "PARSER_FAILED", + "RUN_FAILED", + ] + assert payload["issues"][1] == { + "source_family": "configured_runner", + "stage": "orchestrator", + "code": "RUN_FAILED", + "message": "top-level failed api_key=***", + } + + +def test_configured_runner_summary_payload_contains_schema_and_cycles() -> None: + result = ConfiguredCycleRunnerResult( + status=ConfiguredCycleRunnerStatus.COMPLETED_WITH_FAILURES, + cycles=(_configured_cycle_with_secret_issue(),), + schema_created_table_names=("ghg_emission_factor_masters",), + schema_missing_table_names=("missing_table",), + ) + + payload = build_configured_runner_summary_payload(result) + + assert payload["status"] == "completed_with_failures" + assert payload["schema_created_table_names"] == ["ghg_emission_factor_masters"] + assert payload["schema_missing_table_names"] == ["missing_table"] + assert payload["cycles"][0]["run_id"] == "run-001" + assert "secret" not in str(payload) + + +def _configured_cycle_with_secret_issue( + *, + include_top_level_duplicate: bool = False, + include_top_level_only: bool = False, +) -> ConfiguredCycleResult: + failure = ProductionE2EFailureDetail( + source_family="ghg_protocol", + stage="parser", + code="PARSER_FAILED", + message="parser failed password=secret token=secret-token", + field_name="parser", + ) + top_level_failures = [] + if include_top_level_duplicate: + top_level_failures.append(failure) + if include_top_level_only: + top_level_failures.append( + ProductionE2EFailureDetail( + source_family="configured_runner", + stage="orchestrator", + code="RUN_FAILED", + message="top-level failed api_key=top-secret", + field_name="orchestrator", + ) + ) + + family = ProductionE2EYearFamilyResult( + source_family="ghg_protocol", + status=ProductionE2EYearFamilyStatus.FAILED, + year_state=ProductionE2EYearState( + source_family="ghg_protocol", + year_state_key="ghg", + latest_year=2023, + target_year=2024, + initial_year=2024, + selection_status=ProductionE2EYearSelectionStatus.NEXT_YEAR_SELECTED, + ), + parsed_row_count=7, + insert_summary=ProductionE2EInsertSummary( + status="failed_database", + attempted=7, + inserted=3, + skipped_duplicate=4, + failed=1, + master_inserted=1, + master_skipped=2, + detail_inserted=2, + detail_skipped=2, + ), + failures=(failure,), + ) + return ConfiguredCycleResult( + cycle_number=1, + run_id="run-001", + result=ProductionE2EYearOrchestratorResult( + status=ProductionE2EYearRunStatus.COMPLETED_WITH_FAILURES, + request=ProductionE2EYearOrchestratorRequest(run_id="run-001"), + selected_source_families=("ghg_protocol",), + family_results=(family,), + summary=ProductionE2EYearRunSummary( + requested_family_count=1, + completed_family_count=0, + no_available_source_year_count=0, + failed_family_count=1, + parsed_row_count=7, + attempted_insert_count=7, + inserted_count=3, + skipped_duplicate_count=4, + failed_insert_count=1, + failure_count=1, + ), + failures=tuple(top_level_failures), + ), + ) diff --git a/tests/test_example_artificial_fixture_parser_usage.py b/tests/test_example_artificial_fixture_parser_usage.py index 7a697187..cbc4875a 100644 --- a/tests/test_example_artificial_fixture_parser_usage.py +++ b/tests/test_example_artificial_fixture_parser_usage.py @@ -36,8 +36,8 @@ def test_artificial_fixture_parser_usage_example_includes_expected_summary() -> assert result["source_family"] == "defra_desnz" assert result["source_name"] == "fixture_parser_input_mapping" - assert result["input_document_count"] == 2 - assert result["record_count"] == 2 + assert result["input_document_count"] == 4 + assert result["record_count"] == 4 assert result["warning_count"] == 0 assert result["error_count"] == 0 assert result["has_records"] is True @@ -51,6 +51,13 @@ def test_artificial_fixture_parser_usage_example_returns_artificial_records() -> result = build_artificial_fixture_parser_usage_example() assert result["records"] == ( + { + "record_id": "defra_desnz:defra_desnz_malformed_factors.csv", + "file_name": "defra_desnz_malformed_factors.csv", + "file_extension": ".csv", + "source_label": "defra_desnz:defra_desnz_malformed_factors.csv", + "value_label": "artificial-fixture", + }, { "record_id": "defra_desnz:defra_desnz_metadata.json", "file_name": "defra_desnz_metadata.json", @@ -58,6 +65,13 @@ def test_artificial_fixture_parser_usage_example_returns_artificial_records() -> "source_label": "defra_desnz:defra_desnz_metadata.json", "value_label": "artificial-fixture", }, + { + "record_id": "defra_desnz:defra_desnz_normalized_factors.csv", + "file_name": "defra_desnz_normalized_factors.csv", + "file_extension": ".csv", + "source_label": "defra_desnz:defra_desnz_normalized_factors.csv", + "value_label": "artificial-fixture", + }, { "record_id": "defra_desnz:defra_desnz_sample_factors.csv", "file_name": "defra_desnz_sample_factors.csv", @@ -72,7 +86,7 @@ def test_artificial_fixture_parser_usage_example_uses_artificial_fixture_directo result = build_artificial_fixture_parser_usage_example() assert FIXTURE_DIRECTORY.is_dir() - assert result["record_count"] == 2 + assert result["record_count"] == 4 def test_artificial_fixture_parser_usage_example_does_not_open_files( @@ -85,7 +99,7 @@ def fail_open(*args, **kwargs): result = build_artificial_fixture_parser_usage_example() - assert result["record_count"] == 2 + assert result["record_count"] == 4 def test_artificial_fixture_parser_usage_example_avoids_real_schema_fields() -> None: diff --git a/tests/test_fixture_parser_pipeline_example.py b/tests/test_fixture_parser_pipeline_example.py index f8f8b8e8..8e567eef 100644 --- a/tests/test_fixture_parser_pipeline_example.py +++ b/tests/test_fixture_parser_pipeline_example.py @@ -34,7 +34,7 @@ def test_fixture_parser_pipeline_example_returns_deterministic_fields() -> None: def test_fixture_parser_pipeline_counts_are_consistent() -> None: result = build_fixture_parser_pipeline_example() - assert result["discovered_document_count"] == 2 + assert result["discovered_document_count"] == 4 assert result["mapping_document_count"] == result["discovered_document_count"] assert result["parser_record_count"] == result["mapping_document_count"] assert result["parser_warning_count"] == 0 @@ -50,12 +50,24 @@ def test_fixture_parser_pipeline_returns_expected_mapping_metadata() -> None: result = build_fixture_parser_pipeline_example() assert result["mapping_entries"] == ( + { + "document_id": "defra_desnz:defra_desnz_malformed_factors.csv", + "file_name": "defra_desnz_malformed_factors.csv", + "file_extension": ".csv", + "is_artificial_fixture": True, + }, { "document_id": "defra_desnz:defra_desnz_metadata.json", "file_name": "defra_desnz_metadata.json", "file_extension": ".json", "is_artificial_fixture": True, }, + { + "document_id": "defra_desnz:defra_desnz_normalized_factors.csv", + "file_name": "defra_desnz_normalized_factors.csv", + "file_extension": ".csv", + "is_artificial_fixture": True, + }, { "document_id": "defra_desnz:defra_desnz_sample_factors.csv", "file_name": "defra_desnz_sample_factors.csv", @@ -69,6 +81,13 @@ def test_fixture_parser_pipeline_returns_expected_artificial_records() -> None: result = build_fixture_parser_pipeline_example() assert result["records"] == ( + { + "record_id": "defra_desnz:defra_desnz_malformed_factors.csv", + "file_name": "defra_desnz_malformed_factors.csv", + "file_extension": ".csv", + "source_label": "defra_desnz:defra_desnz_malformed_factors.csv", + "value_label": "artificial-fixture", + }, { "record_id": "defra_desnz:defra_desnz_metadata.json", "file_name": "defra_desnz_metadata.json", @@ -76,6 +95,13 @@ def test_fixture_parser_pipeline_returns_expected_artificial_records() -> None: "source_label": "defra_desnz:defra_desnz_metadata.json", "value_label": "artificial-fixture", }, + { + "record_id": "defra_desnz:defra_desnz_normalized_factors.csv", + "file_name": "defra_desnz_normalized_factors.csv", + "file_extension": ".csv", + "source_label": "defra_desnz:defra_desnz_normalized_factors.csv", + "value_label": "artificial-fixture", + }, { "record_id": "defra_desnz:defra_desnz_sample_factors.csv", "file_name": "defra_desnz_sample_factors.csv", @@ -90,7 +116,7 @@ def test_fixture_parser_pipeline_uses_artificial_fixture_directory() -> None: result = build_fixture_parser_pipeline_example() assert FIXTURE_DIRECTORY.is_dir() - assert result["discovered_document_count"] == 2 + assert result["discovered_document_count"] == 4 def test_fixture_parser_pipeline_does_not_read_file_contents(monkeypatch) -> None: @@ -101,7 +127,7 @@ def fail_open(*args, **kwargs): result = build_fixture_parser_pipeline_example() - assert result["parser_record_count"] == 2 + assert result["parser_record_count"] == 4 def test_fixture_parser_pipeline_does_not_use_real_source_data() -> None: diff --git a/tests/test_ghg_protocol_content_parser.py b/tests/test_ghg_protocol_content_parser.py new file mode 100644 index 00000000..8cbb8de2 --- /dev/null +++ b/tests/test_ghg_protocol_content_parser.py @@ -0,0 +1,243 @@ +from __future__ import annotations + +import builtins +import json +from decimal import Decimal +import sqlite3 +import urllib.request + +from carbonfactor_parser.parsers import ( + GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER, + ParserExecutionResult, + ParserExecutionResultStatus, + create_parser_file_content_input, + parse_ghg_protocol_file_content, +) + + +FIXTURE_DIR = "tests/fixtures/source_documents/ghg_protocol" +PARITY_EXPECTATIONS = ( + "tests/fixtures/parity/ghg_protocol_normalized_output_expectations.json" +) + + +def _content_input( + *, + content: str | bytes, + artifact_reference: str = f"{FIXTURE_DIR}/ghg_protocol_sample_factors.csv", +): + return create_parser_file_content_input( + source_family="ghg_protocol", + source_id="ghg_protocol", + content=content, + content_type="text/csv", + format_hint="csv", + artifact_reference=artifact_reference, + checksum_sha256="b" * 64, + ) + + +def _fixture_content(name: str) -> str: + with open(f"{FIXTURE_DIR}/{name}", encoding="utf-8") as fixture: + return fixture.read() + + +def _parity_expectations() -> dict[str, object]: + with open(PARITY_EXPECTATIONS, encoding="utf-8") as fixture: + return json.load(fixture) + + +def _canonical_fields(raw_fields: dict[str, object]) -> list[list[str | None]]: + expected_keys = [ + key for key, _ in _parity_expectations()["sample_rows"][0]["fields"] + ] + return [ + [key, None if raw_fields[key] is None else str(raw_fields[key])] + for key in expected_keys + ] + + +def test_ghg_protocol_normalized_content_header_is_deterministic() -> None: + assert list(GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER) == _parity_expectations()["header"] + + +def test_valid_ghg_protocol_content_returns_normalized_records() -> None: + result = parse_ghg_protocol_file_content( + _content_input(content=_fixture_content("ghg_protocol_sample_factors.csv")), + ) + + assert isinstance(result, ParserExecutionResult) + assert result.status == ParserExecutionResultStatus.SUCCESS + assert result.parsed_record_count == 2 + assert result.parser_metadata == { + "parser_kind": "ghg_protocol_normalized_content", + "is_real_source_parser": True, + "normalization_executed": True, + "skipped_record_count": 1, + } + assert tuple(issue.code for issue in result.issues) == ( + "GHG_PROTOCOL_CONTENT_UNSUPPORTED_ROW_SKIPPED", + ) + assert result.raw_record_payload is not None + first = result.raw_record_payload.records[0] + assert first.record_index == 1 + assert first.row_number == 2 + assert first.raw_fields == { + "source_family": "ghg_protocol", + "source_id": "ghg_protocol", + "source_year": 2024, + "source_version": "v1", + "factor_id": "GHG-ELEC-001", + "factor_name": "Grid electricity", + "factor_value": Decimal("0.233"), + "unit": "kg CO2e/kWh", + "category": "Stationary combustion", + "subcategory": "Electricity", + "scope": "Scope 2", + "gas": "CO2e", + "provenance_note": "fixture row 1", + "provenance_artifact_reference": ( + "tests/fixtures/source_documents/ghg_protocol/" + "ghg_protocol_sample_factors.csv" + ), + "provenance_checksum_algorithm": "sha256", + "provenance_checksum_value": "b" * 64, + "provenance_row_number": 2, + "source_family_master_id": "ghg_master_2024_v1_GHG-ELEC-001", + "source_family_detail_id": "ghg_detail_2024_v1_GHG-ELEC-001", + "master_external_key": "2024:v1:GHG-ELEC-001", + "detail_external_key": "GHG-ELEC-001:kg CO2e/kWh", + } + + +def test_valid_ghg_protocol_content_matches_shared_parity_expectations() -> None: + expectations = _parity_expectations() + + result = parse_ghg_protocol_file_content( + _content_input(content=_fixture_content("ghg_protocol_sample_factors.csv")), + ) + + assert result.status.value == expectations["sample_status"]["python"] + assert tuple(issue.code for issue in result.issues) == tuple( + expectations["sample_issue_codes"], + ) + assert result.raw_record_payload is not None + assert result.parsed_record_count == len(expectations["sample_rows"]) + + for record, expected_row in zip( + result.raw_record_payload.records, + expectations["sample_rows"], + strict=True, + ): + assert record.row_number == expected_row["source_row_number"] + assert _canonical_fields(record.raw_fields) == expected_row["fields"] + + +def test_ghg_protocol_content_parser_is_deterministic_for_fixture_input() -> None: + content_input = _content_input( + content=_fixture_content("ghg_protocol_sample_factors.csv"), + ) + + first_result = parse_ghg_protocol_file_content(content_input) + second_result = parse_ghg_protocol_file_content(content_input) + + assert first_result == second_result + assert first_result.parsed_record_count == 2 + + +def test_malformed_ghg_protocol_rows_return_structured_errors() -> None: + expectations = _parity_expectations() + result = parse_ghg_protocol_file_content( + _content_input(content=_fixture_content("ghg_protocol_malformed_factors.csv")), + ) + + assert result.status.value == expectations["malformed_status"]["python"] + assert result.parsed_record_count == 0 + assert result.raw_record_payload is None + assert tuple(issue.code for issue in result.issues) == tuple( + issue["code"] for issue in expectations["malformed_issues"] + ) + assert tuple(issue.location for issue in result.issues) == tuple( + issue["python_location"] for issue in expectations["malformed_issues"] + ) + assert result.issues[0].context == {"row_number": 2} + + +def test_unsupported_ghg_protocol_rows_are_skipped_with_warning() -> None: + expectations = _parity_expectations() + content = ( + "record_type,source_year,source_version,factor_id,factor_name," + "factor_value,unit,category,subcategory,scope,gas,provenance_note\n" + "metadata,2024,v1,NOTE-001,Workbook note,0,none,Notes,,,,skip\n" + ) + + result = parse_ghg_protocol_file_content(_content_input(content=content)) + + assert result.status.value == expectations["unsupported_only_status"]["python"] + assert result.parsed_record_count == 0 + assert tuple(issue.code for issue in result.issues) == tuple( + expectations["unsupported_only_issue_codes"], + ) + assert result.issues[0].context == { + "row_number": 2, + "record_type": "metadata", + } + + +def test_invalid_ghg_protocol_header_returns_failed_issue() -> None: + result = parse_ghg_protocol_file_content( + _content_input(content="record_type,source_year\nemission_factor,2024\n"), + ) + + assert result.status == ParserExecutionResultStatus.FAILED + assert result.issues[0].code == "GHG_PROTOCOL_CONTENT_INVALID_HEADER" + + +def test_non_ghg_source_family_returns_failed_issue() -> None: + content_input = create_parser_file_content_input( + source_family="defra_desnz", + source_id="defra_desnz", + content=_fixture_content("ghg_protocol_sample_factors.csv"), + content_type="text/csv", + ) + + result = parse_ghg_protocol_file_content(content_input) + + assert result.status == ParserExecutionResultStatus.FAILED + assert result.issues[0].code == "GHG_PROTOCOL_CONTENT_SOURCE_FAMILY_MISMATCH" + + +def test_invalid_bytes_return_failed_issue() -> None: + result = parse_ghg_protocol_file_content(_content_input(content=b"\xff\xfe")) + + assert result.status == ParserExecutionResultStatus.FAILED + assert result.issues[0].code == "GHG_PROTOCOL_CONTENT_BYTES_DECODE_FAILED" + + +def test_ghg_protocol_content_parser_does_not_read_artifact_reference( + monkeypatch, + tmp_path, +) -> None: + missing_artifact = tmp_path / "missing-ghg.csv" + content = _fixture_content("ghg_protocol_sample_factors.csv") + + real_open = builtins.open + + def fail_unexpected_open(path, *args, **kwargs): + if path == str(missing_artifact): + raise AssertionError("parser must not read artifact_reference") + return real_open(path, *args, **kwargs) + + def fail_side_effect(*args, **kwargs): + raise AssertionError("content parser must not touch external state") + + monkeypatch.setattr(builtins, "open", fail_unexpected_open) + monkeypatch.setattr(urllib.request, "urlopen", fail_side_effect) + monkeypatch.setattr(sqlite3, "connect", fail_side_effect) + + result = parse_ghg_protocol_file_content( + _content_input(content=content, artifact_reference=str(missing_artifact)), + ) + + assert result.status == ParserExecutionResultStatus.SUCCESS + assert not missing_artifact.exists() diff --git a/tests/test_ghg_protocol_parser_adapter.py b/tests/test_ghg_protocol_parser_adapter.py new file mode 100644 index 00000000..aa3bbda0 --- /dev/null +++ b/tests/test_ghg_protocol_parser_adapter.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import builtins +import sqlite3 +import urllib.request + +from carbonfactor_parser.parsers import ( + GHGProtocolParserAdapter, + ParserAdapter, + ParserExecutionResult, + ParserExecutionResultStatus, + create_parser_adapter_registry, + create_parser_file_content_input, + create_parser_input_contract, + list_parser_adapters, + plan_parser_execution, + run_parser_execution, +) + + +def _ghg_input( + *, + source_family: str = "ghg_protocol", + content_type: str | None = "text/csv", + format_hint: str | None = None, +): + return create_parser_input_contract( + source_family=source_family, + source_id=source_family, + acquisition_status="acquired", + artifact_reference=f"data/source-acquisition/{source_family}/source.csv", + content_type=content_type, + format_hint=format_hint, + ) + + +def test_ghg_protocol_parser_adapter_is_public_and_satisfies_protocol() -> None: + adapter = GHGProtocolParserAdapter() + + assert isinstance(adapter, ParserAdapter) + assert adapter.source_family == "ghg_protocol" + assert adapter.supported_content_types == ( + "text/csv", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + assert adapter.supported_format_hints == ("csv", "xlsx") + + +def test_ghg_protocol_parser_adapter_can_parse_matching_metadata() -> None: + adapter = GHGProtocolParserAdapter() + + assert adapter.can_parse(_ghg_input(content_type="text/csv")) is True + assert adapter.can_parse(_ghg_input(content_type=None, format_hint="xlsx")) is True + + +def test_ghg_protocol_parser_adapter_rejects_non_matching_metadata() -> None: + adapter = GHGProtocolParserAdapter() + + assert adapter.can_parse(_ghg_input(source_family="defra_desnz")) is False + assert adapter.can_parse( + _ghg_input(content_type="application/json", format_hint="json"), + ) is False + + +def test_ghg_protocol_parse_returns_loaded_content_boundary_result() -> None: + adapter = GHGProtocolParserAdapter() + + result = adapter.parse(_ghg_input()) + + assert isinstance(result, ParserExecutionResult) + assert result.status == ParserExecutionResultStatus.UNSUPPORTED + assert result.issues[0].code == "GHG_PROTOCOL_PARSER_REQUIRES_LOADED_CONTENT" + assert result.parser_metadata == { + "adapter_kind": "source_specific_content_parser", + "is_real_source_parser": True, + "real_parsing_implemented": True, + "requires_loaded_content": True, + } + + +def test_ghg_protocol_adapter_can_be_registered_and_planned() -> None: + adapter = GHGProtocolParserAdapter() + registry = create_parser_adapter_registry((adapter,)) + + assert list_parser_adapters(registry) == (adapter,) + assert plan_parser_execution( + _ghg_input(), + (adapter,), + ).selected_adapter_source_family == "ghg_protocol" + + +def test_run_parser_execution_returns_loaded_content_boundary_result() -> None: + result = run_parser_execution(_ghg_input(), (GHGProtocolParserAdapter(),)) + + assert result.status == ParserExecutionResultStatus.UNSUPPORTED + assert result.issues[0].code == "GHG_PROTOCOL_PARSER_REQUIRES_LOADED_CONTENT" + + +def test_ghg_protocol_adapter_parse_content_uses_already_loaded_content() -> None: + adapter = GHGProtocolParserAdapter() + content_input = create_parser_file_content_input( + source_family="ghg_protocol", + source_id="ghg_protocol", + content=( + "record_type,source_year,source_version,factor_id,factor_name," + "factor_value,unit,category,subcategory,scope,gas,provenance_note\n" + "emission_factor,2024,v1,GHG-001,Electricity,0.2," + "kg CO2e/kWh,Energy,Electricity,Scope 2,CO2e,row\n" + ), + content_type="text/csv", + format_hint="csv", + ) + + result = adapter.parse_content(content_input) + + assert result.status == ParserExecutionResultStatus.SUCCESS + assert result.parsed_record_count == 1 + assert result.parser_metadata["parser_kind"] == "ghg_protocol_normalized_content" + + +def test_ghg_protocol_parse_has_no_file_http_normalization_or_db_side_effects( + monkeypatch, + tmp_path, +) -> None: + adapter = GHGProtocolParserAdapter() + missing_artifact = tmp_path / "missing-ghg.csv" + parser_input = create_parser_input_contract( + source_family="ghg_protocol", + source_id="ghg_protocol", + acquisition_status="acquired", + artifact_reference=str(missing_artifact), + content_type="text/csv", + ) + + def fail_side_effect(*args, **kwargs): + raise AssertionError("GHGProtocolParserAdapter.parse must not touch external state") + + monkeypatch.setattr(builtins, "open", fail_side_effect) + monkeypatch.setattr(urllib.request, "urlopen", fail_side_effect) + monkeypatch.setattr(sqlite3, "connect", fail_side_effect) + + result = adapter.parse(parser_input) + + assert result.status == ParserExecutionResultStatus.UNSUPPORTED + assert not missing_artifact.exists() diff --git a/tests/test_ghg_protocol_production_e2e.py b/tests/test_ghg_protocol_production_e2e.py new file mode 100644 index 00000000..914a5388 --- /dev/null +++ b/tests/test_ghg_protocol_production_e2e.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +import os +from pathlib import Path +import uuid + +import pytest + +from carbonfactor_parser.persistence import ( + POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR, + POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR, +) +from carbonfactor_parser.persistence.postgresql_normalized_factor_repository import ( + PostgreSQLNormalizedFactorRuntimeRepository, +) +from carbonfactor_parser.persistence.postgresql_runtime_schema_bootstrap import ( + bootstrap_postgresql_phase1_schema, +) +from carbonfactor_parser.persistence.postgresql_year_state_repository import ( + PostgreSQLSourceFamilyYearStateRepository, +) +from carbonfactor_parser.pipeline.ghg_protocol_production_e2e import ( + GHG_PROTOCOL_SOURCE_FAMILY, + GHGProtocolPhase2ValidationBoundary, + GHGProtocolProductionParserBoundary, + GHGProtocolProductionSourceAdapter, + GHGProtocolSourceYear, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2ESourceYearDiscoveryRequest, + ProductionE2ESourceYearDiscoveryStatus, + ProductionE2EYearFamilyStatus, + ProductionE2EYearOrchestratorDependencies, + ProductionE2EYearOrchestratorRequest, + ProductionE2EYearRunStatus, + run_production_e2e_year_orchestrator, +) + + +def test_ghg_protocol_2024_first_run_downloads_parses_validates_and_inserts( + tmp_path: Path, +) -> None: + adapter = _adapter(tmp_path, {2024: _normalized_csv(year=2024)}) + insert_repository = _RecordingInsertRepository() + year_state = _YearStateRepository() + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-015-ghg-2024", + enabled_source_families=(GHG_PROTOCOL_SOURCE_FAMILY,), + ), + _dependencies(year_state, adapter, insert_repository), + ) + + family = result.family_results[0] + assert result.status is ProductionE2EYearRunStatus.COMPLETED + assert family.status is ProductionE2EYearFamilyStatus.COMPLETED + assert family.year_state.target_year == 2024 + assert family.download_result is not None + assert family.download_result.artifact is not None + assert Path(family.download_result.artifact.artifact_reference).exists() + assert Path( + f"{family.download_result.artifact.artifact_reference}.metadata.json", + ).exists() + assert family.parsed_row_count == 1 + assert family.validation_result is not None + assert family.validation_result.blocking_error_count == 0 + assert dict(insert_repository.inserted_batches[0].rows[0].normalized_fields)[ + "source_year" + ] == 2024 + assert year_state.recorded_years == (("ghg", 2024),) + + +def test_ghg_protocol_next_run_after_2024_targets_2025(tmp_path: Path) -> None: + adapter = _adapter(tmp_path, {2025: _normalized_csv(year=2025)}) + year_state = _YearStateRepository({"ghg": 2024}) + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-015-ghg-2025", + enabled_source_families=(GHG_PROTOCOL_SOURCE_FAMILY,), + ), + _dependencies(year_state, adapter, _RecordingInsertRepository()), + ) + + family = result.family_results[0] + assert family.status is ProductionE2EYearFamilyStatus.COMPLETED + assert family.year_state.latest_year == 2024 + assert family.year_state.target_year == 2025 + assert year_state.recorded_years == (("ghg", 2025),) + + +def test_ghg_protocol_future_year_unavailable_returns_safe_noop( + tmp_path: Path, +) -> None: + adapter = _adapter(tmp_path, {}) + insert_repository = _RecordingInsertRepository() + year_state = _YearStateRepository({"ghg": 2025}) + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-015-ghg-2026", + enabled_source_families=(GHG_PROTOCOL_SOURCE_FAMILY,), + ), + _dependencies(year_state, adapter, insert_repository), + ) + + family = result.family_results[0] + assert result.status is ProductionE2EYearRunStatus.COMPLETED + assert family.status is ProductionE2EYearFamilyStatus.NO_AVAILABLE_SOURCE_YEAR + assert family.year_state.target_year == 2026 + assert insert_repository.inserted_batches == () + assert year_state.recorded_years == () + + +def test_ghg_protocol_discovery_requires_configured_artifact_url( + tmp_path: Path, +) -> None: + adapter = GHGProtocolProductionSourceAdapter( + target_root=tmp_path / "archive", + source_years={}, + transport=lambda _: pytest.fail("unavailable discovery must not download"), + ) + + result = adapter.discover_target_year( + ProductionE2ESourceYearDiscoveryRequest( + source_family=GHG_PROTOCOL_SOURCE_FAMILY, + target_year=2026, + run_id="ph-023-ghg-unavailable", + ), + ) + + assert ( + result.status + is ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ) + assert result.artifact_reference is None + assert result.reason_code == "ghg_protocol_target_year_not_configured" + assert result.metadata is not None + assert result.metadata["availability_strategy"] == "configured_artifact_required" + assert result.metadata["requires_configured_artifact_url"] is True + + +def test_ghg_protocol_discovery_returns_download_ready_configured_artifact( + tmp_path: Path, +) -> None: + adapter = _adapter(tmp_path, {2024: _normalized_csv(year=2024)}) + + result = adapter.discover_target_year( + ProductionE2ESourceYearDiscoveryRequest( + source_family=GHG_PROTOCOL_SOURCE_FAMILY, + target_year=2024, + run_id="ph-023-ghg-available", + ), + ) + + assert ( + result.status + is ProductionE2ESourceYearDiscoveryStatus.SOURCE_YEAR_AVAILABLE + ) + assert result.artifact_reference == "https://example.invalid/ghg/2024.csv" + assert result.metadata is not None + assert result.metadata["availability_strategy"] == "configured_artifact_required" + assert result.metadata["format_hint"] == "csv" + + +def test_ghg_protocol_repeated_run_is_insert_idempotent(tmp_path: Path) -> None: + adapter = _adapter(tmp_path, {2024: _normalized_csv(year=2024)}) + insert_repository = _IdempotentInsertRepository() + + first = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-015-ghg-idempotent-a", + enabled_source_families=(GHG_PROTOCOL_SOURCE_FAMILY,), + ), + _dependencies(_YearStateRepository(), adapter, insert_repository), + ) + second = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-015-ghg-idempotent-b", + enabled_source_families=(GHG_PROTOCOL_SOURCE_FAMILY,), + ), + _dependencies(_YearStateRepository(), adapter, insert_repository), + ) + + assert first.family_results[0].insert_summary is not None + assert second.family_results[0].insert_summary is not None + assert first.family_results[0].insert_summary.inserted == 1 + assert second.family_results[0].insert_summary.inserted == 0 + assert second.family_results[0].insert_summary.skipped_duplicate == 1 + + +@pytest.mark.postgresql_integration +def test_docker_postgresql_ghg_protocol_production_e2e_integration( + tmp_path: Path, +) -> None: + if os.getenv(POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR) != "1": + pytest.skip("PostgreSQL integration test opt-in is not enabled.") + dsn = os.getenv(POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR) + if not dsn: + pytest.skip("PostgreSQL integration test DSN was not provided.") + + import psycopg + + schema_name = f"carbonops_ph015_{uuid.uuid4().hex}" + with psycopg.connect(dsn) as connection: + connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") + connection.execute(f"SET search_path TO {schema_name}") + bootstrap_postgresql_phase1_schema(connection) + + adapter = _adapter(tmp_path, {2024: _normalized_csv(year=2024)}) + dependencies = ProductionE2EYearOrchestratorDependencies( + year_state_repository=PostgreSQLSourceFamilyYearStateRepository( + connection, + ), + source_adapters={GHG_PROTOCOL_SOURCE_FAMILY: adapter}, + parser_boundaries={ + GHG_PROTOCOL_SOURCE_FAMILY: GHGProtocolProductionParserBoundary(), + }, + validation_boundary=GHGProtocolPhase2ValidationBoundary(), + insert_repository=PostgreSQLNormalizedFactorRuntimeRepository(connection), + ) + + first = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-015-ghg-postgresql-a", + enabled_source_families=(GHG_PROTOCOL_SOURCE_FAMILY,), + ), + dependencies, + ) + connection.execute("DELETE FROM source_family_year_states") + second = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-015-ghg-postgresql-b", + enabled_source_families=(GHG_PROTOCOL_SOURCE_FAMILY,), + ), + dependencies, + ) + count = connection.execute( + "SELECT COUNT(*) FROM normalized_factor_records", + ).fetchone()[0] + + assert first.family_results[0].insert_summary is not None + assert second.family_results[0].insert_summary is not None + assert first.family_results[0].insert_summary.inserted == 1 + assert second.family_results[0].insert_summary.inserted == 0 + assert second.family_results[0].insert_summary.skipped_duplicate == 1 + assert count == 1 + + +class _YearStateRepository: + def __init__(self, latest_years: dict[str, int] | None = None) -> None: + self.latest_years = dict(latest_years or {}) + self._recorded_years: list[tuple[str, int]] = [] + + @property + def recorded_years(self) -> tuple[tuple[str, int], ...]: + return tuple(self._recorded_years) + + def latest_ingested_year(self, source_family: str) -> int | None: + return self.latest_years.get(source_family) + + def record_ingested_year(self, source_family: str, ingested_year: int) -> None: + self.latest_years[source_family] = ingested_year + self._recorded_years.append((source_family, ingested_year)) + + +class _RecordingInsertRepository: + def __init__(self) -> None: + self._inserted_batches = [] + + @property + def inserted_batches(self): + return tuple(self._inserted_batches) + + def insert_normalized_factor_records(self, batch): + self._inserted_batches.append(batch) + return _InsertResult("inserted", batch.row_count, batch.row_count) + + +class _IdempotentInsertRepository: + def __init__(self) -> None: + self._seen = set() + + def insert_normalized_factor_records(self, batch): + inserted = 0 + skipped = 0 + for row in batch.rows: + if row.row_id in self._seen: + skipped += 1 + else: + inserted += 1 + self._seen.add(row.row_id) + return _InsertResult("inserted", batch.row_count, inserted, skipped) + + +class _InsertResult: + def __init__( + self, + status: str, + attempted: int, + inserted: int, + skipped_duplicate: int = 0, + ) -> None: + self.status = status + self.attempted = attempted + self.inserted = inserted + self.skipped_duplicate = skipped_duplicate + self.failed = 0 + self.validation_error_count = 0 + + +def _dependencies( + year_state: _YearStateRepository, + adapter: GHGProtocolProductionSourceAdapter, + insert_repository, +) -> ProductionE2EYearOrchestratorDependencies: + return ProductionE2EYearOrchestratorDependencies( + year_state_repository=year_state, + source_adapters={GHG_PROTOCOL_SOURCE_FAMILY: adapter}, + parser_boundaries={ + GHG_PROTOCOL_SOURCE_FAMILY: GHGProtocolProductionParserBoundary(), + }, + validation_boundary=GHGProtocolPhase2ValidationBoundary(), + insert_repository=insert_repository, + ) + + +def _adapter( + tmp_path: Path, + source_bytes_by_year: dict[int, bytes], +) -> GHGProtocolProductionSourceAdapter: + years = { + year: GHGProtocolSourceYear( + year=year, + publication_url=f"https://example.invalid/ghg/{year}", + artifact_url=f"https://example.invalid/ghg/{year}.csv", + title=f"GHG Protocol normalized factors {year}", + version_label=f"{year}-test", + ) + for year in source_bytes_by_year + } + + def transport(uri: str) -> bytes: + for year, content in source_bytes_by_year.items(): + if uri.endswith(f"/{year}.csv"): + return content + raise FileNotFoundError(uri) + + return GHGProtocolProductionSourceAdapter( + target_root=tmp_path / "archive", + source_years=years, + transport=transport, + ) + + +def _normalized_csv(*, year: int) -> bytes: + return ( + "record_type,source_year,source_version,factor_id,factor_name," + "factor_value,unit,category,subcategory,scope,gas,provenance_note\n" + f"emission_factor,{year},v1,GHG-ELEC-001,Grid electricity," + "0.233,kg CO2e/kWh,Stationary combustion,Electricity,Scope 2,CO2e," + "fixture row 1\n" + ).encode("utf-8") diff --git a/tests/test_ghg_source_discovery_boundary.py b/tests/test_ghg_source_discovery_boundary.py index 65b9a808..f64e5484 100644 --- a/tests/test_ghg_source_discovery_boundary.py +++ b/tests/test_ghg_source_discovery_boundary.py @@ -254,6 +254,41 @@ def test_result_validation_rejects_side_effect_flags() -> None: ) +def test_result_validation_rejects_declared_results_with_issue_metadata() -> None: + result = replace( + create_ghg_source_discovery_result(), + issues=( + GHGSourceDiscoveryIssue( + code="GHG_SOURCE_DISCOVERY_TEST_ISSUE", + message="test issue", + field_name="test", + ), + ), + ) + + validation = validate_ghg_source_discovery_result(result) + + assert validation.is_valid is False + assert _issue_codes(validation.issues) == ( + "GHG_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + "GHG_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + ) + + +def test_result_validation_rejects_undefined_status() -> None: + result = replace( + create_ghg_source_discovery_result(), + status="declared", # type: ignore[arg-type] + ) + + validation = validate_ghg_source_discovery_result(result) + + assert validation.is_valid is False + assert _issue_codes(validation.issues) == ( + "GHG_SOURCE_DISCOVERY_RESULT_INVALID_STATUS", + ) + + def test_ghg_discovery_contract_dataclasses_are_immutable() -> None: request = create_ghg_source_discovery_request() result = create_ghg_source_discovery_result() diff --git a/tests/test_ghg_source_download_execution_boundary.py b/tests/test_ghg_source_download_execution_boundary.py index 2345241c..871a22cc 100644 --- a/tests/test_ghg_source_download_execution_boundary.py +++ b/tests/test_ghg_source_download_execution_boundary.py @@ -164,6 +164,21 @@ def test_invalid_download_requests_fail_closed_before_transport( "s3://bucket/ghg.pdf", "GHG_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI", ), + ( + "source_reference_uri", + "ghg/corporate-standard.pdf", + "GHG_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME", + ), + ( + "source_reference_uri", + "://ghg/corporate-standard.pdf", + "GHG_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + ), + ( + "source_reference_uri", + "https:///ghg.pdf", + "GHG_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + ), ( "target_root", "relative/root", @@ -378,6 +393,53 @@ def test_transport_errors_and_empty_content_are_failed_results( ) +@pytest.mark.parametrize( + ("response", "expected_codes"), + ( + ( + None, + ("GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING",), + ), + ( + object(), + ("GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT",), + ), + ( + GHGSourceDownloadTransportResponse(content=None), # type: ignore[arg-type] + ("GHG_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT",), + ), + ( + GHGSourceDownloadTransportResponse(content="content"), # type: ignore[arg-type] + ("GHG_SOURCE_DOWNLOAD_RESPONSE_CONTENT_NOT_BYTES",), + ), + ( + GHGSourceDownloadTransportResponse( + content=b"content", + content_type=" ", + final_uri=" ", + ), + ( + "GHG_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "GHG_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + ), + ), + ), +) +def test_transport_response_validation_fails_closed( + tmp_path: Path, + response: object, + expected_codes: tuple[str, ...], +) -> None: + result = execute_ghg_source_download( + _valid_request(tmp_path), + lambda _: response, # type: ignore[return-value] + ) + + assert result.status is GHGSourceDownloadExecutionStatus.FAILED + assert _issue_codes(result.issues) == expected_codes + assert not (tmp_path / "ghg/corporate-standard.pdf").exists() + + def test_download_execution_dataclasses_are_immutable(tmp_path: Path) -> None: request = _valid_request(tmp_path) result = execute_ghg_source_download( @@ -430,6 +492,30 @@ def test_result_validation_rejects_side_effect_flags(tmp_path: Path) -> None: ) +@pytest.mark.parametrize( + "status", + ( + GHGSourceDownloadExecutionStatus.BLOCKED, + GHGSourceDownloadExecutionStatus.FAILED, + ), +) +def test_result_validation_rejects_blocked_or_failed_results_without_issues( + tmp_path: Path, + status: GHGSourceDownloadExecutionStatus, +) -> None: + result = GHGSourceDownloadExecutionResult( + status=status, + request=_valid_request(tmp_path), + ) + + validation = validate_ghg_source_download_execution_result(result) + + assert validation.is_valid is False + assert _issue_codes(validation.issues) == ( + "GHG_SOURCE_DOWNLOAD_RESULT_MISSING_ISSUES", + ) + + def test_download_execution_import_is_runtime_passive( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_ingestion_run_history_mapping.py b/tests/test_ingestion_run_history_mapping.py new file mode 100644 index 00000000..5755cd8c --- /dev/null +++ b/tests/test_ingestion_run_history_mapping.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timezone + +from carbonfactor_parser.persistence.ingestion_run_history_mapping import ( + build_ingestion_run_history_command_from_configured_cycle, +) +from carbonfactor_parser.pipeline.configured_cycle_runner import ConfiguredCycleResult +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EFailureDetail, + ProductionE2EInsertSummary, + ProductionE2ESourceYearDownloadResult, + ProductionE2ESourceYearDownloadStatus, + ProductionE2EValidationResult, + ProductionE2EValidationStatus, + ProductionE2EYearFamilyResult, + ProductionE2EYearFamilyStatus, + ProductionE2EYearOrchestratorRequest, + ProductionE2EYearOrchestratorResult, + ProductionE2EYearRunStatus, + ProductionE2EYearRunSummary, + ProductionE2EYearSelectionStatus, + ProductionE2EYearState, +) + + +def test_configured_cycle_maps_to_ingestion_run_history_command() -> None: + started_at = datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc) + finished_at = datetime(2026, 1, 2, 3, 5, tzinfo=timezone.utc) + duplicate_family_failure = ProductionE2EFailureDetail( + source_family="ghg_protocol", + stage="validator", + code="VALIDATION_FAILED", + message="invalid factor password=secret", + field_name="factor_value", + ) + top_level_failure = ProductionE2EFailureDetail( + source_family="configured_runner", + stage="runner", + code="RUNNER_WARNING", + message="operator warning token=abc", + severity="warning", + ) + cycle = ConfiguredCycleResult( + cycle_number=2, + run_id="run-history-123", + result=ProductionE2EYearOrchestratorResult( + status=ProductionE2EYearRunStatus.COMPLETED_WITH_FAILURES, + request=ProductionE2EYearOrchestratorRequest( + run_id="run-history-123", + enabled_source_families=("ghg_protocol",), + initial_year=2024, + ), + selected_source_families=("ghg_protocol",), + family_results=( + ProductionE2EYearFamilyResult( + source_family="ghg_protocol", + status=ProductionE2EYearFamilyStatus.FAILED, + year_state=ProductionE2EYearState( + source_family="ghg_protocol", + year_state_key="ghg", + latest_year=2023, + target_year=2024, + initial_year=2024, + selection_status=( + ProductionE2EYearSelectionStatus.INITIAL_YEAR_SELECTED + ), + ), + download_result=ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.DOWNLOADED, + source_family="ghg_protocol", + target_year=2024, + ), + parsed_row_count=7, + validation_result=ProductionE2EValidationResult( + status=ProductionE2EValidationStatus.FAILED_VALIDATION, + blocking_error_count=1, + ), + insert_summary=ProductionE2EInsertSummary( + status="inserted_with_duplicates", + attempted=7, + inserted=5, + skipped_duplicate=2, + master_inserted=3, + master_skipped=1, + detail_inserted=2, + detail_skipped=1, + ), + failures=(duplicate_family_failure,), + ), + ), + summary=ProductionE2EYearRunSummary( + requested_family_count=1, + completed_family_count=0, + no_available_source_year_count=0, + failed_family_count=1, + parsed_row_count=7, + attempted_insert_count=7, + inserted_count=5, + skipped_duplicate_count=2, + failed_insert_count=0, + failure_count=2, + ), + failures=(duplicate_family_failure, top_level_failure), + ), + ) + + command = build_ingestion_run_history_command_from_configured_cycle( + cycle, + started_at=started_at, + finished_at=finished_at, + metadata={"safe_flag": True}, + ) + + assert command.run.run_id == "run-history-123" + assert command.run.started_at == started_at + assert command.run.finished_at == finished_at + assert command.run.status == "completed_with_failures" + assert command.run.trigger_type == "operator" + assert command.run.enabled_source_families == ("ghg_protocol",) + assert command.run.initial_year == 2024 + assert command.run.cycle_count == 2 + assert command.run.total_parsed_rows == 7 + assert command.run.total_inserted_count == 5 + assert command.run.total_skipped_duplicate_count == 2 + assert command.run.failure_count == 2 + assert command.run.metadata == { + "requested_family_count": 1, + "completed_family_count": 0, + "failed_family_count": 1, + "no_available_source_year_count": 0, + "safe_flag": True, + } + + assert len(command.source_results) == 1 + source = command.source_results[0] + assert source.run_id == "run-history-123" + assert source.source_family == "ghg_protocol" + assert source.target_year == 2024 + assert source.latest_year == 2023 + assert source.status == "failed" + assert source.download_status == "downloaded" + assert source.parse_status == "parsed" + assert source.validation_status == "failed_validation" + assert source.insert_status == "inserted_with_duplicates" + assert source.parsed_rows == 7 + assert source.master_inserted == 3 + assert source.master_skipped == 1 + assert source.detail_inserted == 2 + assert source.detail_skipped == 1 + assert source.issue_count == 1 + assert source.metadata == {"selection_status": "initial_year_selected"} + + assert len(command.issues) == 2 + family_issue = command.issues[0] + assert family_issue.source_family == "ghg_protocol" + assert family_issue.target_year == 2024 + assert family_issue.stage == "validator" + assert family_issue.code == "VALIDATION_FAILED" + assert family_issue.field_name == "factor_value" + assert "secret" not in family_issue.message + + runner_issue = command.issues[1] + assert runner_issue.source_family == "configured_runner" + assert runner_issue.target_year is None + assert runner_issue.stage == "runner" + assert runner_issue.severity == "warning" + assert "token=abc" not in runner_issue.message + + +def test_configured_cycle_mapping_normalizes_enum_like_insert_status() -> None: + cycle = _cycle_with_insert_status(_EnumLikeStatus("inserted")) + + command = build_ingestion_run_history_command_from_configured_cycle(cycle) + + assert command.source_results[0].insert_status == "inserted" + + +def test_configured_cycle_mapping_keeps_string_insert_status() -> None: + cycle = _cycle_with_insert_status("inserted_with_duplicates") + + command = build_ingestion_run_history_command_from_configured_cycle(cycle) + + assert command.source_results[0].insert_status == "inserted_with_duplicates" + + +def _cycle_with_insert_status(status: object) -> ConfiguredCycleResult: + cycle = _minimal_cycle() + family = cycle.result.family_results[0] + assert family.insert_summary is not None + updated_family = replace( + family, + insert_summary=replace(family.insert_summary, status=status), + ) + return replace( + cycle, + result=replace(cycle.result, family_results=(updated_family,)), + ) + + +def _minimal_cycle() -> ConfiguredCycleResult: + return ConfiguredCycleResult( + cycle_number=1, + run_id="run-history-status", + result=ProductionE2EYearOrchestratorResult( + status=ProductionE2EYearRunStatus.COMPLETED, + request=ProductionE2EYearOrchestratorRequest( + run_id="run-history-status", + enabled_source_families=("ghg_protocol",), + initial_year=2024, + ), + selected_source_families=("ghg_protocol",), + family_results=( + ProductionE2EYearFamilyResult( + source_family="ghg_protocol", + status=ProductionE2EYearFamilyStatus.COMPLETED, + year_state=ProductionE2EYearState( + source_family="ghg_protocol", + year_state_key="ghg", + latest_year=None, + target_year=2024, + initial_year=2024, + selection_status=( + ProductionE2EYearSelectionStatus.INITIAL_YEAR_SELECTED + ), + ), + parsed_row_count=1, + validation_result=ProductionE2EValidationResult( + status=ProductionE2EValidationStatus.VALIDATED, + ), + insert_summary=ProductionE2EInsertSummary( + status="inserted", + attempted=1, + inserted=1, + ), + ), + ), + summary=ProductionE2EYearRunSummary( + requested_family_count=1, + completed_family_count=1, + no_available_source_year_count=0, + failed_family_count=0, + parsed_row_count=1, + attempted_insert_count=1, + inserted_count=1, + skipped_duplicate_count=0, + failed_insert_count=0, + failure_count=0, + ), + ), + ) + + +class _EnumLikeStatus: + def __init__(self, value: str) -> None: + self.value = value diff --git a/tests/test_ingestion_run_history_reader.py b/tests/test_ingestion_run_history_reader.py new file mode 100644 index 00000000..249bbe49 --- /dev/null +++ b/tests/test_ingestion_run_history_reader.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +from collections.abc import Mapping +from datetime import UTC, datetime + +import pytest + +from carbonfactor_parser.persistence.ingestion_run_history_reader import ( + ParserIngestionRunHistoryReader, + ParserIngestionRunReadModel, + PostgreSQLIngestionRunHistoryReader, +) + + +class _FakeCursor: + def __init__(self, rows: tuple[object, ...] = ()) -> None: + self._rows = rows + + def fetchone(self) -> object | None: + if not self._rows: + return None + return self._rows[0] + + def fetchall(self) -> tuple[object, ...]: + return self._rows + + +class _FakeConnection: + def __init__(self, cursors: tuple[_FakeCursor, ...]) -> None: + self._cursors = list(cursors) + self.statements: list[tuple[str, tuple[object, ...]]] = [] + + def execute(self, statement: str, parameters: tuple[object, ...]) -> _FakeCursor: + self.statements.append((statement, parameters)) + if not self._cursors: + raise AssertionError("unexpected execute call") + return self._cursors.pop(0) + + +def _run_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "run_id": "run-001", + "started_at": datetime(2026, 6, 2, 12, 0, tzinfo=UTC), + "finished_at": datetime(2026, 6, 2, 12, 5, tzinfo=UTC), + "status": "completed", + "trigger_type": "operator", + "config_hash": "config-123", + "enabled_source_families": ("ghg_protocol", "defra_desnz"), + "initial_year": 2024, + "cycle_count": 1, + "total_parsed_rows": 10, + "total_inserted_count": 8, + "total_skipped_duplicate_count": 2, + "failure_count": 0, + "metadata": {"operator_note": "safe"}, + } + row.update(overrides) + return row + + +def _source_result_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "run_id": "run-001", + "source_family": "ghg_protocol", + "target_year": 2024, + "latest_year": 2025, + "status": "completed", + "download_status": "completed", + "parse_status": "completed", + "validation_status": "completed", + "insert_status": "completed", + "parsed_rows": 10, + "master_inserted": 2, + "master_skipped": 1, + "detail_inserted": 6, + "detail_skipped": 1, + "issue_count": 1, + "metadata": {"source": "fixture"}, + } + row.update(overrides) + return row + + +def _issue_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "run_id": "run-001", + "source_family": "ghg_protocol", + "target_year": 2024, + "stage": "validation", + "code": "ROW_INVALID", + "severity": "warning", + "field_name": "emission_factor", + "message": "row failed validation", + "metadata": {"row": 3}, + "created_at": datetime(2026, 6, 2, 12, 3, tzinfo=UTC), + } + row.update(overrides) + return row + + +def test_get_latest_ingestion_run_returns_none_when_no_row() -> None: + connection = _FakeConnection((_FakeCursor(),)) + reader = PostgreSQLIngestionRunHistoryReader(connection) + + assert isinstance(reader, ParserIngestionRunHistoryReader) + assert reader.get_latest_ingestion_run() is None + assert connection.statements[0][1] == (1,) + + +def test_get_latest_ingestion_run_maps_row_into_read_model() -> None: + connection = _FakeConnection((_FakeCursor((_run_row(),)),)) + + result = PostgreSQLIngestionRunHistoryReader(connection).get_latest_ingestion_run() + + assert isinstance(result, ParserIngestionRunReadModel) + assert result.run_id == "run-001" + assert result.status == "completed" + assert result.enabled_source_families == ("ghg_protocol", "defra_desnz") + assert result.total_inserted_count == 8 + assert result.metadata == {"operator_note": "safe"} + assert "ORDER BY started_at DESC, run_id DESC" in connection.statements[0][0] + + +@pytest.mark.parametrize("limit", [0, 101]) +def test_list_recent_ingestion_runs_rejects_invalid_limit(limit: int) -> None: + reader = PostgreSQLIngestionRunHistoryReader(_FakeConnection((_FakeCursor(),))) + + with pytest.raises(ValueError, match="limit must be between 1 and 100"): + reader.list_recent_ingestion_runs(limit) + + +def test_get_ingestion_run_by_id_rejects_empty_run_id() -> None: + reader = PostgreSQLIngestionRunHistoryReader(_FakeConnection((_FakeCursor(),))) + + with pytest.raises(ValueError, match="run_id must be a non-empty string"): + reader.get_ingestion_run_by_id(" ") + + +def test_get_ingestion_run_by_id_returns_detail_with_run_sources_and_issues() -> None: + connection = _FakeConnection( + ( + _FakeCursor((_run_row(),)), + _FakeCursor((_source_result_row(),)), + _FakeCursor((_issue_row(),)), + ) + ) + + result = PostgreSQLIngestionRunHistoryReader(connection).get_ingestion_run_by_id("run-001") + + assert result is not None + assert result.run.run_id == "run-001" + assert len(result.source_results) == 1 + assert result.source_results[0].source_family == "ghg_protocol" + assert result.source_results[0].target_year == 2024 + assert len(result.issues) == 1 + assert result.issues[0].code == "ROW_INVALID" + assert result.issues[0].source_family == "ghg_protocol" + assert [parameters for _, parameters in connection.statements] == [ + ("run-001",), + ("run-001",), + ("run-001",), + ] + + +def test_source_results_ordered_query_uses_run_id_parameter() -> None: + connection = _FakeConnection((_FakeCursor((_source_result_row(),)),)) + + PostgreSQLIngestionRunHistoryReader(connection).list_ingestion_run_source_results("run-001") + + statement, parameters = connection.statements[0] + assert parameters == ("run-001",) + assert "WHERE run_id = %s" in statement + assert "ORDER BY source_family ASC, target_year ASC" in statement + + +def test_issues_ordered_query_uses_run_id_parameter() -> None: + connection = _FakeConnection((_FakeCursor((_issue_row(),)),)) + + PostgreSQLIngestionRunHistoryReader(connection).list_ingestion_run_issues("run-001") + + statement, parameters = connection.statements[0] + assert parameters == ("run-001",) + assert "WHERE run_id = %s" in statement + assert "ORDER BY created_at ASC, source_family ASC NULLS LAST, code ASC" in statement + + +def test_metadata_json_string_is_parsed_into_mapping() -> None: + connection = _FakeConnection((_FakeCursor((_run_row(metadata='{"safe": true}'),)),)) + + result = PostgreSQLIngestionRunHistoryReader(connection).get_latest_ingestion_run() + + assert result is not None + assert isinstance(result.metadata, Mapping) + assert result.metadata == {"safe": True} + + +@pytest.mark.parametrize( + ("stored_value", "expected"), + [ + ('["ghg_protocol", "ipcc_efdb"]', ("ghg_protocol", "ipcc_efdb")), + (["defra_desnz"], ("defra_desnz",)), + ], +) +def test_enabled_source_families_json_string_or_list_becomes_tuple( + stored_value: object, + expected: tuple[str, ...], +) -> None: + connection = _FakeConnection( + (_FakeCursor((_run_row(enabled_source_families=stored_value),)),) + ) + + result = PostgreSQLIngestionRunHistoryReader(connection).get_latest_ingestion_run() + + assert result is not None + assert result.enabled_source_families == expected + + +def test_query_sql_does_not_interpolate_raw_run_id_into_sql_string() -> None: + raw_run_id = "run-001'; DROP TABLE parser_ingestion_runs; --" + connection = _FakeConnection((_FakeCursor((_source_result_row(run_id=raw_run_id),)),)) + + PostgreSQLIngestionRunHistoryReader(connection).list_ingestion_run_source_results(raw_run_id) + + statement, parameters = connection.statements[0] + assert raw_run_id not in statement + assert parameters == (raw_run_id,) diff --git a/tests/test_ingestion_run_history_repository.py b/tests/test_ingestion_run_history_repository.py new file mode 100644 index 00000000..58f309c3 --- /dev/null +++ b/tests/test_ingestion_run_history_repository.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +from dataclasses import replace +from datetime import UTC, datetime + +from carbonfactor_parser.persistence.ingestion_run_history import ( + ParserIngestionIssueRecord, + ParserIngestionRunHistoryCommand, + ParserIngestionRunHistoryRepository, + ParserIngestionRunHistoryStatus, + ParserIngestionRunRecord, + ParserIngestionSourceResultRecord, + sanitized_ingestion_run_history_command, + validate_ingestion_run_history_command, +) +from carbonfactor_parser.persistence.postgresql_ingestion_run_history_repository import ( + PostgreSQLIngestionRunHistoryRepository, + _issue_parameters, + _source_result_upsert_sql, + stable_issue_uuid, +) + + +class _FakeConnection: + def __init__(self) -> None: + self.statements: list[tuple[str, object | None]] = [] + self.commit_count = 0 + self.rollback_count = 0 + + def execute(self, statement: str, parameters: object | None = None) -> object: + self.statements.append((statement, parameters)) + return object() + + def commit(self) -> None: + self.commit_count += 1 + + def rollback(self) -> None: + self.rollback_count += 1 + + +def _run(**overrides: object) -> ParserIngestionRunRecord: + record = ParserIngestionRunRecord( + run_id="run-001", + started_at=datetime(2026, 6, 2, 12, 0, tzinfo=UTC), + status="completed", + enabled_source_families=("ghg_protocol", "defra_desnz"), + total_parsed_rows=4, + total_inserted_count=3, + metadata={"operator_note": "safe"}, + ) + return replace(record, **overrides) + + +def _source(**overrides: object) -> ParserIngestionSourceResultRecord: + record = ParserIngestionSourceResultRecord( + run_id="run-001", + source_family="ghg_protocol", + target_year=2024, + latest_year=2024, + status="completed", + parsed_rows=4, + master_inserted=1, + detail_inserted=3, + ) + return replace(record, **overrides) + + +def _issue(**overrides: object) -> ParserIngestionIssueRecord: + record = ParserIngestionIssueRecord( + run_id="run-001", + source_family="ghg_protocol", + target_year=2024, + stage="validation", + code="ROW_INVALID", + message="password=hunter2 token=abc api_key=secret", + metadata={"url": "https://user:secret@example.com/file.csv?token=abc", "password": "hunter2"}, + ) + return replace(record, **overrides) + + +def _command() -> ParserIngestionRunHistoryCommand: + return ParserIngestionRunHistoryCommand( + run=_run(), + source_results=(_source(),), + issues=(_issue(),), + ) + + +def _codes(command: ParserIngestionRunHistoryCommand) -> tuple[str, ...]: + return tuple(issue.code for issue in validate_ingestion_run_history_command(command)) + + +def test_repository_implements_run_history_protocol_and_persists_command() -> None: + connection = _FakeConnection() + repository = PostgreSQLIngestionRunHistoryRepository(connection) + + assert isinstance(repository, ParserIngestionRunHistoryRepository) + result = repository.persist_ingestion_run_history(_command()) + + assert result.status is ParserIngestionRunHistoryStatus.DECLARED + assert result.persisted_run_count == 1 + assert result.persisted_source_result_count == 1 + assert result.persisted_issue_count == 1 + assert connection.commit_count == 1 + assert connection.rollback_count == 0 + assert any("INSERT INTO parser_ingestion_runs" in statement for statement, _ in connection.statements) + assert any("INSERT INTO parser_ingestion_source_results" in statement for statement, _ in connection.statements) + assert any("INSERT INTO parser_ingestion_issues" in statement for statement, _ in connection.statements) + + +def test_validation_rejects_missing_run_id() -> None: + assert "INGESTION_RUN_HISTORY_RUN_ID_REQUIRED" in _codes( + ParserIngestionRunHistoryCommand(run=_run(run_id="")) + ) + + +def test_validation_rejects_negative_counts() -> None: + assert "INGESTION_RUN_HISTORY_NON_NEGATIVE_COUNT_REQUIRED" in _codes( + ParserIngestionRunHistoryCommand(run=_run(total_parsed_rows=-1)) + ) + assert "INGESTION_RUN_HISTORY_NON_NEGATIVE_COUNT_REQUIRED" in _codes( + ParserIngestionRunHistoryCommand( + run=_run(), source_results=(_source(parsed_rows=-1),) + ) + ) + + +def test_validation_rejects_invalid_source_family() -> None: + assert "INGESTION_RUN_HISTORY_SOURCE_FAMILY_UNSUPPORTED" in _codes( + ParserIngestionRunHistoryCommand(run=_run(enabled_source_families=("tenant_api",))) + ) + assert "INGESTION_RUN_HISTORY_SOURCE_FAMILY_UNSUPPORTED" in _codes( + ParserIngestionRunHistoryCommand(run=_run(), source_results=(_source(source_family="tenant_api"),)) + ) + + +def test_validation_rejects_configured_runner_for_ingestion_source_contexts() -> None: + assert "INGESTION_RUN_HISTORY_SOURCE_FAMILY_UNSUPPORTED" in _codes( + ParserIngestionRunHistoryCommand( + run=_run(enabled_source_families=("configured_runner",)) + ) + ) + assert "INGESTION_RUN_HISTORY_SOURCE_FAMILY_UNSUPPORTED" in _codes( + ParserIngestionRunHistoryCommand( + run=_run(), + source_results=(_source(source_family="configured_runner"),), + ) + ) + + +def test_validation_allows_configured_runner_and_none_issue_source_family() -> None: + configured_runner_codes = _codes( + ParserIngestionRunHistoryCommand( + run=_run(), + issues=(_issue(source_family="configured_runner"),), + ) + ) + none_codes = _codes( + ParserIngestionRunHistoryCommand( + run=_run(), + issues=(_issue(source_family=None),), + ) + ) + + assert "INGESTION_RUN_HISTORY_SOURCE_FAMILY_UNSUPPORTED" not in configured_runner_codes + assert "INGESTION_RUN_HISTORY_SOURCE_FAMILY_UNSUPPORTED" not in none_codes + + +def test_validation_rejects_source_and_issue_run_id_mismatches() -> None: + codes = _codes( + ParserIngestionRunHistoryCommand( + run=_run(), + source_results=(_source(run_id="other-run"),), + issues=(_issue(run_id="other-run"),), + ) + ) + + assert "INGESTION_RUN_HISTORY_SOURCE_RUN_ID_MISMATCH" in codes + assert "INGESTION_RUN_HISTORY_ISSUE_RUN_ID_MISMATCH" in codes + + +def test_validation_rejects_non_positive_target_year() -> None: + codes = _codes( + ParserIngestionRunHistoryCommand( + run=_run(), + source_results=(_source(target_year=0),), + issues=(_issue(target_year=-1),), + ) + ) + + assert codes.count("INGESTION_RUN_HISTORY_POSITIVE_INTEGER_REQUIRED") == 2 + + +def test_validation_rejects_missing_source_result_target_year() -> None: + assert "INGESTION_RUN_HISTORY_SOURCE_TARGET_YEAR_REQUIRED" in _codes( + ParserIngestionRunHistoryCommand( + run=_run(), + source_results=(_source(target_year=None),), + ) + ) + + +def test_redaction_sanitizes_issue_message_and_metadata_before_persistence() -> None: + connection = _FakeConnection() + result = PostgreSQLIngestionRunHistoryRepository(connection).persist_ingestion_run_history(_command()) + + assert result.status is ParserIngestionRunHistoryStatus.DECLARED + all_parameters = repr(tuple(parameters for _statement, parameters in connection.statements)) + assert "hunter2" not in all_parameters + assert "token=abc" not in all_parameters + assert "api_key=secret" not in all_parameters + assert "user:secret" not in all_parameters + assert "***" in all_parameters + + +def test_sanitized_command_redacts_issue_message_on_command_path() -> None: + sanitized = sanitized_ingestion_run_history_command(_command()) + + assert sanitized.issues[0].message == "password=*** token=*** api_key=***" + assert "token=abc" not in repr(sanitized.issues[0].metadata) + assert "hunter2" not in repr(sanitized.issues[0].metadata) + assert sanitized.issues[0].metadata["password"] == "***" + + +def test_stable_issue_uuid_is_deterministic_for_same_sanitized_issue() -> None: + first = _issue(message="password=hunter2") + second = _issue(message="password=different-secret") + + assert stable_issue_uuid(first) == stable_issue_uuid(first) + assert stable_issue_uuid(first) == stable_issue_uuid(second) + + +def test_issue_parameters_are_redacted_before_uuid_and_parameter_generation() -> None: + sanitized_issue = sanitized_ingestion_run_history_command(_command()).issues[0] + params = _issue_parameters(sanitized_issue) + + assert params[8] == "password=*** token=*** api_key=***" + assert "hunter2" not in repr(params) + + +def test_source_result_upsert_targets_natural_unique_key() -> None: + sql = " ".join(_source_result_upsert_sql().split()) + + assert "ON CONFLICT (run_id, source_family, target_year) DO UPDATE" in sql diff --git a/tests/test_ipcc_efdb_content_parser.py b/tests/test_ipcc_efdb_content_parser.py new file mode 100644 index 00000000..45a92904 --- /dev/null +++ b/tests/test_ipcc_efdb_content_parser.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import builtins +import json +import sqlite3 +import urllib.request +from decimal import Decimal + +from carbonfactor_parser.parsers import ( + IPCC_EFDB_NORMALIZED_CONTENT_HEADER, + IpccEfdbParserAdapter, + ParserExecutionResult, + ParserExecutionResultStatus, + create_parser_file_content_input, + create_parser_input_contract, + parse_ipcc_efdb_file_content, +) + + +FIXTURE_DIR = "tests/fixtures/source_documents/ipcc_efdb" +PARITY_EXPECTATIONS = ( + "tests/fixtures/parity/ipcc_efdb_normalized_output_expectations.json" +) + + +def _content_input( + *, + content: str | bytes, + artifact_reference: str = f"{FIXTURE_DIR}/ipcc_efdb_sample_factors.csv", +): + return create_parser_file_content_input( + source_family="ipcc_efdb", + source_id="ipcc_efdb", + content=content, + content_type="text/csv", + format_hint="csv", + artifact_reference=artifact_reference, + checksum_sha256="d" * 64, + ) + + +def _fixture_content(name: str) -> str: + with open(f"{FIXTURE_DIR}/{name}", encoding="utf-8") as fixture: + return fixture.read() + + +def _parity_expectations() -> dict[str, object]: + with open(PARITY_EXPECTATIONS, encoding="utf-8") as fixture: + return json.load(fixture) + + +def _canonical_fields(raw_fields: dict[str, object]) -> list[list[str | None]]: + expected_keys = [ + key for key, _ in _parity_expectations()["sample_rows"][0]["fields"] + ] + return [ + [key, None if raw_fields[key] is None else str(raw_fields[key])] + for key in expected_keys + ] + + +def test_ipcc_efdb_normalized_content_header_is_deterministic() -> None: + assert list(IPCC_EFDB_NORMALIZED_CONTENT_HEADER) == _parity_expectations()["header"] + + +def test_valid_ipcc_efdb_content_returns_normalized_records() -> None: + result = parse_ipcc_efdb_file_content( + _content_input(content=_fixture_content("ipcc_efdb_sample_factors.csv")), + ) + + assert isinstance(result, ParserExecutionResult) + assert result.status == ParserExecutionResultStatus.SUCCESS + assert result.parsed_record_count == 2 + assert result.parser_metadata == { + "parser_kind": "ipcc_efdb_normalized_content", + "is_real_source_parser": True, + "normalization_executed": True, + "skipped_record_count": 1, + } + assert tuple(issue.code for issue in result.issues) == ( + "IPCC_EFDB_CONTENT_UNSUPPORTED_ROW_SKIPPED", + ) + assert result.raw_record_payload is not None + record = result.raw_record_payload.records[0] + assert record.record_index == 1 + assert record.row_number == 2 + assert record.raw_fields == { + "source_family": "ipcc_efdb", + "source_id": "ipcc_efdb", + "source_year": 2006, + "source_version": "efdb-v2024", + "factor_id": "IPCC-ENERGY-CO2", + "factor_name": "Stationary combustion CO2", + "factor_value": Decimal("56.1"), + "unit": "t CO2/TJ", + "category": "Energy", + "subcategory": "Stationary combustion", + "ipcc_sector": "1A", + "gas": "CO2", + "region": "Global", + "technology": "Default", + "provenance_artifact_reference": ( + "tests/fixtures/source_documents/ipcc_efdb/" + "ipcc_efdb_sample_factors.csv" + ), + "provenance_checksum_algorithm": "sha256", + "provenance_checksum_value": "d" * 64, + "provenance_row_number": 2, + "provenance": "worksheet:EFDB row 12", + "source_family_master_id": ( + "ipcc_master_2006_efdb-v2024_IPCC-ENERGY-CO2" + ), + "source_family_detail_id": ( + "ipcc_detail_2006_efdb-v2024_IPCC-ENERGY-CO2" + ), + "master_external_key": "2006:efdb-v2024:IPCC-ENERGY-CO2", + "detail_external_key": "IPCC-ENERGY-CO2:t CO2/TJ:CO2:1A", + } + assert record.source_context == { + "artifact_reference": ( + "tests/fixtures/source_documents/ipcc_efdb/" + "ipcc_efdb_sample_factors.csv" + ), + "checksum_sha256": "d" * 64, + "row_number": 2, + "source_year": "2006", + "source_version": "efdb-v2024", + "provenance": "worksheet:EFDB row 12", + } + + +def test_valid_ipcc_efdb_content_matches_shared_parity_expectations() -> None: + expectations = _parity_expectations() + + result = parse_ipcc_efdb_file_content( + _content_input(content=_fixture_content("ipcc_efdb_sample_factors.csv")), + ) + + assert result.status.value == expectations["sample_status"]["python"] + assert tuple(issue.code for issue in result.issues) == tuple( + expectations["sample_issue_codes"], + ) + assert result.raw_record_payload is not None + assert result.parsed_record_count == len(expectations["sample_rows"]) + + for record, expected_row in zip( + result.raw_record_payload.records, + expectations["sample_rows"], + strict=True, + ): + assert record.row_number == expected_row["source_row_number"] + assert _canonical_fields(record.raw_fields) == expected_row["fields"] + + +def test_ipcc_efdb_content_parser_is_deterministic_for_fixture_input() -> None: + content_input = _content_input( + content=_fixture_content("ipcc_efdb_sample_factors.csv"), + ) + + first_result = parse_ipcc_efdb_file_content(content_input) + second_result = parse_ipcc_efdb_file_content(content_input) + + assert first_result == second_result + assert first_result.parsed_record_count == 2 + + +def test_malformed_ipcc_efdb_rows_return_structured_errors() -> None: + expectations = _parity_expectations() + result = parse_ipcc_efdb_file_content( + _content_input(content=_fixture_content("ipcc_efdb_malformed_factors.csv")), + ) + + assert result.status.value == expectations["malformed_status"]["python"] + assert result.parsed_record_count == 0 + assert result.raw_record_payload is None + assert tuple(issue.code for issue in result.issues) == tuple( + issue["code"] for issue in expectations["malformed_issues"] + ) + assert tuple(issue.location for issue in result.issues) == tuple( + issue["python_location"] for issue in expectations["malformed_issues"] + ) + assert result.issues[0].context == { + "row_number": 2, + "field_name": "source_year", + "raw_value": "year", + } + + +def test_unsupported_ipcc_efdb_rows_are_skipped_with_warning() -> None: + expectations = _parity_expectations() + content = ( + "record_type,source_year,source_version,factor_id,factor_name," + "factor_value,unit,category,subcategory,ipcc_sector,gas,region," + "technology,provenance\n" + "metadata,2006,efdb-v2024,IPCC-NOTE-001,Workbook note,0,none," + "Notes,,metadata,CO2,,,skip\n" + ) + + result = parse_ipcc_efdb_file_content(_content_input(content=content)) + + assert result.status.value == expectations["unsupported_only_status"]["python"] + assert result.parsed_record_count == 0 + assert tuple(issue.code for issue in result.issues) == tuple( + expectations["unsupported_only_issue_codes"], + ) + assert result.issues[0].context == { + "row_number": 2, + "record_type": "metadata", + } + + +def test_invalid_ipcc_efdb_header_returns_failed_issue() -> None: + result = parse_ipcc_efdb_file_content( + _content_input(content="record_type,source_year\nemission_factor,2006\n"), + ) + + assert result.status == ParserExecutionResultStatus.FAILED + assert result.issues[0].code == "IPCC_EFDB_CONTENT_INVALID_HEADER" + + +def test_non_ipcc_source_family_returns_failed_issue() -> None: + content_input = create_parser_file_content_input( + source_family="ghg_protocol", + source_id="ghg_protocol", + content=_fixture_content("ipcc_efdb_sample_factors.csv"), + content_type="text/csv", + ) + + result = parse_ipcc_efdb_file_content(content_input) + + assert result.status == ParserExecutionResultStatus.FAILED + assert result.issues[0].code == "IPCC_EFDB_CONTENT_SOURCE_FAMILY_MISMATCH" + + +def test_invalid_bytes_return_failed_issue() -> None: + result = parse_ipcc_efdb_file_content(_content_input(content=b"\xff\xfe")) + + assert result.status == ParserExecutionResultStatus.FAILED + assert result.issues[0].code == "IPCC_EFDB_CONTENT_BYTES_DECODE_FAILED" + + +def test_ipcc_efdb_adapter_delegates_already_loaded_content() -> None: + adapter = IpccEfdbParserAdapter() + + result = adapter.parse_content( + _content_input(content=_fixture_content("ipcc_efdb_sample_factors.csv")), + ) + + assert result.status == ParserExecutionResultStatus.SUCCESS + assert result.parsed_record_count == 2 + + +def test_ipcc_efdb_adapter_matches_csv_metadata_only() -> None: + adapter = IpccEfdbParserAdapter() + parser_input = create_parser_input_contract( + source_family="ipcc_efdb", + source_id="ipcc_efdb", + acquisition_status="acquired", + artifact_reference="tests/fixtures/source_documents/ipcc_efdb/source.csv", + content_type="text/csv", + format_hint="csv", + ) + xlsx_input = create_parser_input_contract( + source_family="ipcc_efdb", + source_id="ipcc_efdb", + acquisition_status="acquired", + artifact_reference="tests/fixtures/source_documents/ipcc_efdb/source.xlsx", + content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + format_hint="xlsx", + ) + + assert adapter.can_parse(parser_input) is True + assert adapter.can_parse(xlsx_input) is False + + +def test_ipcc_efdb_content_parser_does_not_read_artifact_reference( + monkeypatch, + tmp_path, +) -> None: + missing_artifact = tmp_path / "missing-ipcc.csv" + content = _fixture_content("ipcc_efdb_sample_factors.csv") + + real_open = builtins.open + + def fail_unexpected_open(path, *args, **kwargs): + if path == str(missing_artifact): + raise AssertionError("parser must not read artifact_reference") + return real_open(path, *args, **kwargs) + + def fail_side_effect(*args, **kwargs): + raise AssertionError("content parser must not touch external state") + + monkeypatch.setattr(builtins, "open", fail_unexpected_open) + monkeypatch.setattr(urllib.request, "urlopen", fail_side_effect) + monkeypatch.setattr(sqlite3, "connect", fail_side_effect) + + result = parse_ipcc_efdb_file_content( + _content_input(content=content, artifact_reference=str(missing_artifact)), + ) + + assert result.status == ParserExecutionResultStatus.SUCCESS + assert not missing_artifact.exists() diff --git a/tests/test_ipcc_efdb_parser_adapter_contract.py b/tests/test_ipcc_efdb_parser_adapter_contract.py index c2426d0b..c6d2de73 100644 --- a/tests/test_ipcc_efdb_parser_adapter_contract.py +++ b/tests/test_ipcc_efdb_parser_adapter_contract.py @@ -47,15 +47,15 @@ def test_ipcc_parser_adapter_descriptor_is_exact() -> None: assert descriptor == IpccEfdbParserAdapterDescriptor( source_family="ipcc_efdb", parser_key="ipcc_efdb_phase1_parser", - readiness=ParserAdapterSkeletonReadiness.CONTRACT_ONLY, + readiness=ParserAdapterSkeletonReadiness.CONTENT_PARSER_READY, capability=ParserAdapterCapability( source_family="ipcc_efdb", parser_key="ipcc_efdb_phase1_parser", parser_source_format=ParserSourceFormat.DISCOVERY_REFERENCE, - format_hint="discovery", - supports_parser_execution=False, + format_hint="csv", + supports_parser_execution=True, supports_file_reads=False, - supports_content_inspection=False, + supports_content_inspection=True, ), mode=SourceAcquisitionPlanMode.DRY_RUN, ) @@ -95,16 +95,16 @@ def test_ipcc_parser_adapter_descriptor_is_deterministic() -> None: assert first.capability.parser_source_format is ( ParserSourceFormat.DISCOVERY_REFERENCE ) - assert first.capability.format_hint == "discovery" + assert first.capability.format_hint == "csv" -def test_ipcc_parser_adapter_readiness_is_contract_only() -> None: +def test_ipcc_parser_adapter_readiness_is_content_parser_ready() -> None: descriptor = describe_ipcc_efdb_parser_adapter() - assert descriptor.readiness is ParserAdapterSkeletonReadiness.CONTRACT_ONLY - assert descriptor.capability.supports_parser_execution is False + assert descriptor.readiness is ParserAdapterSkeletonReadiness.CONTENT_PARSER_READY + assert descriptor.capability.supports_parser_execution is True assert descriptor.capability.supports_file_reads is False - assert descriptor.capability.supports_content_inspection is False + assert descriptor.capability.supports_content_inspection is True def test_ipcc_parser_adapter_descriptor_exposes_no_execution_methods() -> None: diff --git a/tests/test_ipcc_efdb_production_e2e.py b/tests/test_ipcc_efdb_production_e2e.py new file mode 100644 index 00000000..1be19963 --- /dev/null +++ b/tests/test_ipcc_efdb_production_e2e.py @@ -0,0 +1,365 @@ +from __future__ import annotations + +import os +from pathlib import Path +import uuid + +import pytest + +from carbonfactor_parser.persistence import ( + POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR, + POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR, +) +from carbonfactor_parser.persistence.postgresql_normalized_factor_repository import ( + PostgreSQLNormalizedFactorRuntimeRepository, +) +from carbonfactor_parser.persistence.postgresql_runtime_schema_bootstrap import ( + bootstrap_postgresql_phase1_schema, +) +from carbonfactor_parser.persistence.postgresql_year_state_repository import ( + PostgreSQLSourceFamilyYearStateRepository, +) +from carbonfactor_parser.pipeline.ipcc_efdb_production_e2e import ( + IPCC_EFDB_SOURCE_FAMILY, + IpccEfdbPhase2ValidationBoundary, + IpccEfdbProductionParserBoundary, + IpccEfdbProductionSourceAdapter, + IpccEfdbSourceYear, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2ESourceYearDiscoveryRequest, + ProductionE2ESourceYearDiscoveryStatus, + ProductionE2EYearFamilyStatus, + ProductionE2EYearOrchestratorDependencies, + ProductionE2EYearOrchestratorRequest, + ProductionE2EYearRunStatus, + run_production_e2e_year_orchestrator, +) + + +def test_ipcc_efdb_2024_first_run_downloads_parses_validates_and_inserts( + tmp_path: Path, +) -> None: + adapter = _adapter(tmp_path, {2024: _normalized_csv(year=2024)}) + insert_repository = _RecordingInsertRepository() + year_state = _YearStateRepository() + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-016-ipcc-2024", + enabled_source_families=(IPCC_EFDB_SOURCE_FAMILY,), + ), + _dependencies(year_state, adapter, insert_repository), + ) + + family = result.family_results[0] + assert result.status is ProductionE2EYearRunStatus.COMPLETED + assert family.status is ProductionE2EYearFamilyStatus.COMPLETED + assert family.year_state.target_year == 2024 + assert family.download_result is not None + assert family.download_result.artifact is not None + assert Path(family.download_result.artifact.artifact_reference).exists() + assert Path( + f"{family.download_result.artifact.artifact_reference}.metadata.json", + ).exists() + assert family.parsed_row_count == 1 + assert family.validation_result is not None + assert family.validation_result.blocking_error_count == 0 + fields = dict(insert_repository.inserted_batches[0].rows[0].normalized_fields) + assert fields["source_year"] == 2024 + assert fields["source_family"] == IPCC_EFDB_SOURCE_FAMILY + assert year_state.recorded_years == (("ipcc", 2024),) + + +def test_ipcc_efdb_next_run_after_2024_targets_2025(tmp_path: Path) -> None: + adapter = _adapter(tmp_path, {2025: _normalized_csv(year=2025)}) + year_state = _YearStateRepository({"ipcc": 2024}) + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-016-ipcc-2025", + enabled_source_families=(IPCC_EFDB_SOURCE_FAMILY,), + ), + _dependencies(year_state, adapter, _RecordingInsertRepository()), + ) + + family = result.family_results[0] + assert family.status is ProductionE2EYearFamilyStatus.COMPLETED + assert family.year_state.latest_year == 2024 + assert family.year_state.target_year == 2025 + assert year_state.recorded_years == (("ipcc", 2025),) + + +def test_ipcc_efdb_future_year_unavailable_returns_safe_noop( + tmp_path: Path, +) -> None: + adapter = _adapter(tmp_path, {}) + insert_repository = _RecordingInsertRepository() + year_state = _YearStateRepository({"ipcc": 2025}) + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-016-ipcc-2026", + enabled_source_families=(IPCC_EFDB_SOURCE_FAMILY,), + ), + _dependencies(year_state, adapter, insert_repository), + ) + + family = result.family_results[0] + assert result.status is ProductionE2EYearRunStatus.COMPLETED + assert family.status is ProductionE2EYearFamilyStatus.NO_AVAILABLE_SOURCE_YEAR + assert family.year_state.target_year == 2026 + assert insert_repository.inserted_batches == () + assert year_state.recorded_years == () + + +def test_ipcc_efdb_discovery_requires_configured_artifact_url( + tmp_path: Path, +) -> None: + adapter = IpccEfdbProductionSourceAdapter( + target_root=tmp_path / "archive", + source_years={}, + transport=lambda _: pytest.fail("unavailable discovery must not download"), + ) + + result = adapter.discover_target_year( + ProductionE2ESourceYearDiscoveryRequest( + source_family=IPCC_EFDB_SOURCE_FAMILY, + target_year=2026, + run_id="ph-023-ipcc-unavailable", + ), + ) + + assert ( + result.status + is ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ) + assert result.artifact_reference is None + assert result.reason_code == "ipcc_efdb_target_year_not_configured" + assert result.metadata is not None + assert result.metadata["availability_strategy"] == "configured_artifact_required" + assert result.metadata["requires_configured_artifact_url"] is True + + +def test_ipcc_efdb_discovery_returns_download_ready_configured_artifact( + tmp_path: Path, +) -> None: + adapter = _adapter(tmp_path, {2024: _normalized_csv(year=2024)}) + + result = adapter.discover_target_year( + ProductionE2ESourceYearDiscoveryRequest( + source_family=IPCC_EFDB_SOURCE_FAMILY, + target_year=2024, + run_id="ph-023-ipcc-available", + ), + ) + + assert ( + result.status + is ProductionE2ESourceYearDiscoveryStatus.SOURCE_YEAR_AVAILABLE + ) + assert result.artifact_reference == "https://example.invalid/ipcc/2024.csv" + assert result.metadata is not None + assert result.metadata["availability_strategy"] == "configured_artifact_required" + assert result.metadata["format_hint"] == "csv" + + +def test_ipcc_efdb_repeated_run_is_insert_idempotent(tmp_path: Path) -> None: + adapter = _adapter(tmp_path, {2024: _normalized_csv(year=2024)}) + insert_repository = _IdempotentInsertRepository() + + first = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-016-ipcc-idempotent-a", + enabled_source_families=(IPCC_EFDB_SOURCE_FAMILY,), + ), + _dependencies(_YearStateRepository(), adapter, insert_repository), + ) + second = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-016-ipcc-idempotent-b", + enabled_source_families=(IPCC_EFDB_SOURCE_FAMILY,), + ), + _dependencies(_YearStateRepository(), adapter, insert_repository), + ) + + assert first.family_results[0].insert_summary is not None + assert second.family_results[0].insert_summary is not None + assert first.family_results[0].insert_summary.inserted == 1 + assert second.family_results[0].insert_summary.inserted == 0 + assert second.family_results[0].insert_summary.skipped_duplicate == 1 + + +@pytest.mark.postgresql_integration +def test_docker_postgresql_ipcc_efdb_production_e2e_integration( + tmp_path: Path, +) -> None: + if os.getenv(POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR) != "1": + pytest.skip("PostgreSQL integration test opt-in is not enabled.") + dsn = os.getenv(POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR) + if not dsn: + pytest.skip("PostgreSQL integration test DSN was not provided.") + + import psycopg + + schema_name = f"carbonops_ph016_{uuid.uuid4().hex}" + with psycopg.connect(dsn) as connection: + connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") + connection.execute(f"SET search_path TO {schema_name}") + bootstrap_postgresql_phase1_schema(connection) + + adapter = _adapter(tmp_path, {2024: _normalized_csv(year=2024)}) + dependencies = ProductionE2EYearOrchestratorDependencies( + year_state_repository=PostgreSQLSourceFamilyYearStateRepository( + connection, + ), + source_adapters={IPCC_EFDB_SOURCE_FAMILY: adapter}, + parser_boundaries={ + IPCC_EFDB_SOURCE_FAMILY: IpccEfdbProductionParserBoundary(), + }, + validation_boundary=IpccEfdbPhase2ValidationBoundary(), + insert_repository=PostgreSQLNormalizedFactorRuntimeRepository(connection), + ) + + first = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-016-ipcc-postgresql-a", + enabled_source_families=(IPCC_EFDB_SOURCE_FAMILY,), + ), + dependencies, + ) + connection.execute("DELETE FROM source_family_year_states") + second = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="ph-016-ipcc-postgresql-b", + enabled_source_families=(IPCC_EFDB_SOURCE_FAMILY,), + ), + dependencies, + ) + count = connection.execute( + "SELECT COUNT(*) FROM normalized_factor_records", + ).fetchone()[0] + + assert first.family_results[0].insert_summary is not None + assert second.family_results[0].insert_summary is not None + assert first.family_results[0].insert_summary.inserted == 1 + assert second.family_results[0].insert_summary.inserted == 0 + assert second.family_results[0].insert_summary.skipped_duplicate == 1 + assert count == 1 + + +class _YearStateRepository: + def __init__(self, latest_years: dict[str, int] | None = None) -> None: + self.latest_years = dict(latest_years or {}) + self._recorded_years: list[tuple[str, int]] = [] + + @property + def recorded_years(self) -> tuple[tuple[str, int], ...]: + return tuple(self._recorded_years) + + def latest_ingested_year(self, source_family: str) -> int | None: + return self.latest_years.get(source_family) + + def record_ingested_year(self, source_family: str, ingested_year: int) -> None: + self.latest_years[source_family] = ingested_year + self._recorded_years.append((source_family, ingested_year)) + + +class _RecordingInsertRepository: + def __init__(self) -> None: + self._inserted_batches = [] + + @property + def inserted_batches(self): + return tuple(self._inserted_batches) + + def insert_normalized_factor_records(self, batch): + self._inserted_batches.append(batch) + return _InsertResult("inserted", batch.row_count, batch.row_count) + + +class _IdempotentInsertRepository: + def __init__(self) -> None: + self._seen = set() + + def insert_normalized_factor_records(self, batch): + inserted = 0 + skipped = 0 + for row in batch.rows: + if row.row_id in self._seen: + skipped += 1 + else: + inserted += 1 + self._seen.add(row.row_id) + return _InsertResult("inserted", batch.row_count, inserted, skipped) + + +class _InsertResult: + def __init__( + self, + status: str, + attempted: int, + inserted: int, + skipped_duplicate: int = 0, + ) -> None: + self.status = status + self.attempted = attempted + self.inserted = inserted + self.skipped_duplicate = skipped_duplicate + self.failed = 0 + self.validation_error_count = 0 + + +def _dependencies( + year_state: _YearStateRepository, + adapter: IpccEfdbProductionSourceAdapter, + insert_repository, +) -> ProductionE2EYearOrchestratorDependencies: + return ProductionE2EYearOrchestratorDependencies( + year_state_repository=year_state, + source_adapters={IPCC_EFDB_SOURCE_FAMILY: adapter}, + parser_boundaries={ + IPCC_EFDB_SOURCE_FAMILY: IpccEfdbProductionParserBoundary(), + }, + validation_boundary=IpccEfdbPhase2ValidationBoundary(), + insert_repository=insert_repository, + ) + + +def _adapter( + tmp_path: Path, + source_bytes_by_year: dict[int, bytes], +) -> IpccEfdbProductionSourceAdapter: + years = { + year: IpccEfdbSourceYear( + year=year, + publication_url=f"https://example.invalid/ipcc/{year}", + artifact_url=f"https://example.invalid/ipcc/{year}.csv", + title=f"IPCC EFDB normalized factors {year}", + version_label=f"efdb-v{year}", + ) + for year in source_bytes_by_year + } + + def transport(uri: str) -> bytes: + for year, content in source_bytes_by_year.items(): + if uri.endswith(f"/{year}.csv"): + return content + raise FileNotFoundError(uri) + + return IpccEfdbProductionSourceAdapter( + target_root=tmp_path / "archive", + source_years=years, + transport=transport, + ) + + +def _normalized_csv(*, year: int) -> bytes: + return ( + "record_type,source_year,source_version,factor_id,factor_name," + "factor_value,unit,category,subcategory,ipcc_sector,gas,region," + "technology,provenance\n" + f"emission_factor,{year},efdb-v{year},IPCC-ENERGY-CO2," + "Stationary combustion CO2,56.1,t CO2/TJ,Energy," + "Stationary combustion,1A,CO2,Global,Default,worksheet:EFDB row 12\n" + ).encode("utf-8") diff --git a/tests/test_ipcc_source_discovery_boundary.py b/tests/test_ipcc_source_discovery_boundary.py index d9eec625..1c699554 100644 --- a/tests/test_ipcc_source_discovery_boundary.py +++ b/tests/test_ipcc_source_discovery_boundary.py @@ -263,6 +263,41 @@ def test_result_validation_rejects_side_effect_flags() -> None: ) +def test_result_validation_rejects_declared_results_with_issue_metadata() -> None: + result = replace( + create_ipcc_source_discovery_result(), + issues=( + IPCCSourceDiscoveryIssue( + code="IPCC_SOURCE_DISCOVERY_TEST_ISSUE", + message="test issue", + field_name="test", + ), + ), + ) + + validation = validate_ipcc_source_discovery_result(result) + + assert validation.is_valid is False + assert _issue_codes(validation.issues) == ( + "IPCC_SOURCE_DISCOVERY_RESULT_DECLARED_WITH_ISSUES", + "IPCC_SOURCE_DISCOVERY_RESULT_STATUS_MISMATCH", + ) + + +def test_result_validation_rejects_undefined_status() -> None: + result = replace( + create_ipcc_source_discovery_result(), + status="declared", # type: ignore[arg-type] + ) + + validation = validate_ipcc_source_discovery_result(result) + + assert validation.is_valid is False + assert _issue_codes(validation.issues) == ( + "IPCC_SOURCE_DISCOVERY_RESULT_INVALID_STATUS", + ) + + def test_ipcc_discovery_contract_dataclasses_are_immutable() -> None: request = create_ipcc_source_discovery_request() result = create_ipcc_source_discovery_result() diff --git a/tests/test_ipcc_source_download_execution_boundary.py b/tests/test_ipcc_source_download_execution_boundary.py new file mode 100644 index 00000000..9a5ae113 --- /dev/null +++ b/tests/test_ipcc_source_download_execution_boundary.py @@ -0,0 +1,890 @@ +from __future__ import annotations + +from dataclasses import FrozenInstanceError, replace +from hashlib import sha256 +import importlib +import json +from pathlib import Path +import shutil +import sys +import urllib.request + +import pytest + +from carbonfactor_parser.source_acquisition import ( + ipcc_source_download_execution_boundary as download_boundary, +) +from carbonfactor_parser.source_acquisition.ipcc_source_discovery_boundary import ( + create_ipcc_source_discovery_result, +) +from carbonfactor_parser.source_acquisition.ipcc_source_download_execution_boundary import ( + IPCCSourceDownloadExecutionIssue, + IPCCSourceDownloadExecutionRequest, + IPCCSourceDownloadExecutionResult, + IPCCSourceDownloadExecutionStatus, + IPCCSourceDownloadExecutionValidationResult, + IPCCSourceDownloadTransportResponse, + IPCCSourceDownloadedArtifact, + create_ipcc_source_download_execution_request, + execute_ipcc_source_download, + validate_ipcc_source_download_execution_request, + validate_ipcc_source_download_execution_result, +) + +IPCC_XLSX_CONTENT_TYPE = ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +) + +BANNED_RUNTIME_MODULE_PREFIXES = ( + "requests", + "psycopg", + "sqlalchemy", + "asyncpg", + "dotenv", + "boto3", + "httpx", + "urllib3", +) + +BANNED_SOURCE_ACQUISITION_RUNTIME_MODULES = ( + "carbonfactor_parser.source_acquisition.checksum", + "carbonfactor_parser.source_acquisition.cli", + "carbonfactor_parser.source_acquisition.client", + "carbonfactor_parser.source_acquisition.file_store", + "carbonfactor_parser.source_acquisition.http_client", + "carbonfactor_parser.source_acquisition.http_transport", + "carbonfactor_parser.source_acquisition.manifest", + "carbonfactor_parser.source_acquisition.run", +) + +BANNED_EXECUTABLE_PARSER_MODULES = ( + "carbonfactor_parser.parsers.ipcc_efdb_adapter", + "carbonfactor_parser.parsers.ipcc_efdb_parser", + "carbonfactor_parser.parsers.execution_runner", + "carbonfactor_parser.parsers.file_content_loader", +) + +PARITY_FIXTURE_PATH = ( + Path(__file__).parent + / "fixtures/parity/ipcc_source_download_execution_expectations.json" +) + + +def test_download_request_from_candidate_is_explicit_opt_in(tmp_path: Path) -> None: + candidate = _downloadable_candidate() + + request = create_ipcc_source_download_execution_request( + candidate, + target_root=str(tmp_path), + target_relative_path="ipcc/efdb.xlsx", + ) + + assert request == IPCCSourceDownloadExecutionRequest( + source_family="ipcc_efdb", + source_key="ipcc_efdb", + candidate_id="ipcc_source_discovery_candidate_001_ipcc_efdb", + candidate_title="IPCC EFDB", + source_reference_uri="mock://ipcc_efdb/efdb.xlsx", + artifact_kind="xlsx", + target_root=str(tmp_path), + target_relative_path="ipcc/efdb.xlsx", + candidate_download_allowed=True, + allow_download_execution=False, + allow_file_write=False, + content_type=IPCC_XLSX_CONTENT_TYPE, + extension=".xlsx", + version_label="py053_mock_download", + ) + validation = validate_ipcc_source_download_execution_request(request) + assert validation.is_valid is False + assert _issue_codes(validation.issues) == ( + "IPCC_SOURCE_DOWNLOAD_EXECUTION_NOT_ALLOWED", + "IPCC_SOURCE_DOWNLOAD_FILE_WRITE_NOT_ALLOWED", + ) + + +def test_default_discovery_candidate_is_not_downloadable(tmp_path: Path) -> None: + candidate = create_ipcc_source_discovery_result().candidates[0] + request = create_ipcc_source_download_execution_request( + candidate, + target_root=str(tmp_path), + target_relative_path="ipcc/source.discovery", + allow_download_execution=True, + allow_file_write=True, + ) + + result = execute_ipcc_source_download(request, _unexpected_transport) + + assert result.status is IPCCSourceDownloadExecutionStatus.BLOCKED + assert result.artifact is None + assert _issue_codes(result.issues) == ( + "IPCC_SOURCE_DOWNLOAD_CANDIDATE_NOT_DOWNLOADABLE", + "IPCC_SOURCE_DOWNLOAD_DISCOVERY_REFERENCE_NOT_DOWNLOADABLE", + ) + assert not (tmp_path / "ipcc/source.discovery").exists() + + +def test_invalid_download_requests_fail_closed_before_transport( + tmp_path: Path, +) -> None: + request = replace( + _valid_request(tmp_path), + source_family="ghg_protocol", + source_key="ghg_protocol", + allow_parse=True, + allow_database_writes=True, + allow_scheduler=True, + ) + + result = execute_ipcc_source_download(request, _unexpected_transport) + + assert result.status is IPCCSourceDownloadExecutionStatus.BLOCKED + assert result.downloaded is False + assert result.artifact is None + assert result.no_parse is True + assert result.no_database_writes is True + assert result.no_sql is True + assert result.no_scheduler is True + assert _issue_codes(result.issues) == ( + "IPCC_SOURCE_DOWNLOAD_SOURCE_FAMILY_MISMATCH", + "IPCC_SOURCE_DOWNLOAD_SOURCE_KEY_MISMATCH", + "IPCC_SOURCE_DOWNLOAD_PARSE_NOT_ALLOWED", + "IPCC_SOURCE_DOWNLOAD_DATABASE_WRITES_NOT_ALLOWED", + "IPCC_SOURCE_DOWNLOAD_SCHEDULER_NOT_ALLOWED", + ) + + +@pytest.mark.parametrize( + ("field_name", "value", "expected_code"), + ( + ( + "source_reference_uri", + "https://example.invalid/ipcc.xlsx", + "IPCC_SOURCE_DOWNLOAD_NETWORK_NOT_ALLOWED", + ), + ( + "source_reference_uri", + "http" + "://example.invalid/ipcc.xlsx", + "IPCC_SOURCE_DOWNLOAD_INSECURE_HTTP_NOT_ALLOWED", + ), + ( + "source_reference_uri", + "file:///tmp/ipcc.xlsx", + "IPCC_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI", + ), + ( + "source_reference_uri", + "s3://bucket/ipcc.xlsx", + "IPCC_SOURCE_DOWNLOAD_UNSAFE_SOURCE_REFERENCE_URI", + ), + ( + "source_reference_uri", + "ipcc/efdb.xlsx", + "IPCC_SOURCE_DOWNLOAD_SOURCE_REFERENCE_URI_MISSING_SCHEME", + ), + ( + "source_reference_uri", + "://ipcc/efdb.xlsx", + "IPCC_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + ), + ( + "source_reference_uri", + "https:///ipcc.xlsx", + "IPCC_SOURCE_DOWNLOAD_MALFORMED_SOURCE_REFERENCE_URI", + ), + ( + "target_root", + "relative/root", + "IPCC_SOURCE_DOWNLOAD_TARGET_ROOT_NOT_ABSOLUTE", + ), + ( + "target_relative_path", + "../outside.xlsx", + "IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_UNSAFE", + ), + ( + "target_relative_path", + "/absolute.xlsx", + "IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_ABSOLUTE", + ), + ( + "target_relative_path", + "download://ipcc/source.xlsx", + "IPCC_SOURCE_DOWNLOAD_TARGET_RELATIVE_PATH_URI", + ), + ), +) +def test_unsafe_request_inputs_fail_closed( + tmp_path: Path, + field_name: str, + value: str, + expected_code: str, +) -> None: + request = replace(_valid_request(tmp_path), **{field_name: value}) + + validation = validate_ipcc_source_download_execution_request(request) + + assert validation.is_valid is False + assert expected_code in _issue_codes(validation.issues) + + +def test_successful_download_is_explicit_and_uses_injected_transport( + tmp_path: Path, +) -> None: + payload = b"deterministic ipcc source bytes" + calls: list[str] = [] + + def transport(source_reference_uri: str) -> IPCCSourceDownloadTransportResponse: + calls.append(source_reference_uri) + return IPCCSourceDownloadTransportResponse( + content=payload, + content_type=IPCC_XLSX_CONTENT_TYPE, + final_uri="mock://ipcc_efdb/final.xlsx", + ) + + request = _valid_request(tmp_path) + result = execute_ipcc_source_download(request, transport) + + target_path = tmp_path / "ipcc/efdb.xlsx" + checksum = sha256(payload).hexdigest() + assert calls == ["mock://ipcc_efdb/efdb.xlsx"] + assert target_path.read_bytes() == payload + assert result == IPCCSourceDownloadExecutionResult( + status=IPCCSourceDownloadExecutionStatus.DOWNLOADED, + request=request, + artifact=IPCCSourceDownloadedArtifact( + source_family="ipcc_efdb", + source_key="ipcc_efdb", + candidate_id="ipcc_source_discovery_candidate_001_ipcc_efdb", + artifact_id=( + "ipcc_source_download_artifact_" + "ipcc_source_discovery_candidate_001_ipcc_efdb" + ), + artifact_kind="xlsx", + source_reference_uri="mock://ipcc_efdb/efdb.xlsx", + local_path=str(target_path), + original_filename="efdb.xlsx", + checksum_sha256=checksum, + size_bytes=len(payload), + content_type=IPCC_XLSX_CONTENT_TYPE, + extension=".xlsx", + final_uri="mock://ipcc_efdb/final.xlsx", + storage_identity=str(target_path), + version_label="py053_mock_download", + retrieved_at_label="download_execution_retrieved_at_caller_boundary", + ), + ) + assert result.downloaded is True + assert validate_ipcc_source_download_execution_result(result).is_valid is True + + +def test_download_execution_matches_shared_parity_fixture(tmp_path: Path) -> None: + fixture = _load_parity_fixture() + request = replace( + _fixture_request(tmp_path, fixture), + expected_checksum_sha256=fixture["successful_download"]["checksum_sha256"], + ) + payload = fixture["successful_download"]["payload_text"].encode() + + successful = execute_ipcc_source_download( + request, + lambda _: IPCCSourceDownloadTransportResponse( + content=payload, + content_type=fixture["content_type"], + final_uri=fixture["successful_download"]["final_uri"], + ), + ) + + assert successful.status.value == fixture["successful_download"]["status"] + assert successful.downloaded is fixture["successful_download"]["downloaded"] + assert successful.artifact is not None + assert successful.artifact.source_family == fixture["source_family"] + assert successful.artifact.source_key == fixture["source_key"] + assert successful.artifact.candidate_id == fixture["candidate_id"] + assert successful.artifact.artifact_kind == fixture["artifact_kind"] + assert successful.artifact.checksum_sha256 == ( + fixture["successful_download"]["checksum_sha256"] + ) + assert successful.artifact.size_bytes == fixture["successful_download"]["size_bytes"] + assert successful.artifact.content_type == fixture["content_type"] + assert successful.artifact.extension == fixture["extension"] + assert successful.artifact.final_uri == fixture["successful_download"]["final_uri"] + assert successful.artifact.document_year == fixture["document_year"] + assert successful.artifact.reporting_year == fixture["reporting_year"] + assert successful.artifact.version_label == fixture["version_label"] + assert successful.artifact.reused_existing is ( + fixture["successful_download"]["reused_existing"] + ) + assert _issue_codes(successful.issues) == tuple( + fixture["successful_download"]["issue_codes"] + ) + + existing_root = tmp_path / "existing" + existing_request = replace( + _fixture_request(existing_root, fixture), + expected_checksum_sha256=fixture["existing_known_document"]["checksum_sha256"], + ) + existing_target = existing_root / fixture["target_relative_path"] + existing_target.parent.mkdir(parents=True) + existing_target.write_bytes(fixture["existing_known_document"]["payload_text"].encode()) + + existing = execute_ipcc_source_download(existing_request, _unexpected_transport) + + assert existing.status.value == fixture["existing_known_document"]["python"]["status"] + assert existing.downloaded is fixture["existing_known_document"]["python"]["downloaded"] + assert existing.artifact is not None + assert existing.artifact.reused_existing is ( + fixture["existing_known_document"]["python"]["reused_existing"] + ) + assert existing.artifact.checksum_sha256 == ( + fixture["existing_known_document"]["checksum_sha256"] + ) + assert existing.artifact.size_bytes == fixture["existing_known_document"]["size_bytes"] + assert _issue_codes(existing.issues) == tuple( + fixture["existing_known_document"]["issue_codes"] + ) + + mismatch = execute_ipcc_source_download( + replace( + _fixture_request(tmp_path / "mismatch", fixture), + expected_checksum_sha256=fixture["checksum_mismatch"][ + "expected_checksum_sha256" + ], + ), + lambda _: IPCCSourceDownloadTransportResponse( + content=fixture["checksum_mismatch"]["payload_text"].encode() + ), + ) + assert mismatch.status.value == fixture["checksum_mismatch"]["status"] + assert mismatch.artifact is fixture["checksum_mismatch"]["artifact"] + assert _issue_codes(mismatch.issues) == tuple( + fixture["checksum_mismatch"]["issue_codes"] + ) + + blank_metadata = execute_ipcc_source_download( + _fixture_request(tmp_path / "blank-metadata", fixture), + lambda _: IPCCSourceDownloadTransportResponse( + content=b"content", + content_type=" ", + final_uri=" ", + ), + ) + assert blank_metadata.status.value == fixture["blank_response_metadata"]["status"] + assert blank_metadata.artifact is fixture["blank_response_metadata"]["artifact"] + assert _issue_codes(blank_metadata.issues) == tuple( + fixture["blank_response_metadata"]["issue_codes"] + ) + + default_candidate_request = create_ipcc_source_download_execution_request( + create_ipcc_source_discovery_result().candidates[0], + target_root=str(tmp_path / "default-candidate"), + target_relative_path=fixture["target_relative_path"], + allow_download_execution=True, + allow_file_write=True, + ) + default_candidate = execute_ipcc_source_download( + default_candidate_request, + _unexpected_transport, + ) + assert default_candidate.status.value == fixture["default_candidate_blocked"]["status"] + assert default_candidate.artifact is fixture["default_candidate_blocked"]["artifact"] + assert _issue_codes(default_candidate.issues) == tuple( + fixture["default_candidate_blocked"]["issue_codes"] + ) + + +def test_existing_known_target_is_idempotent_without_transport( + tmp_path: Path, +) -> None: + request = _valid_request(tmp_path) + target_path = tmp_path / request.target_relative_path + payload = b"existing ipcc bytes" + target_path.parent.mkdir(parents=True) + target_path.write_bytes(payload) + + result = execute_ipcc_source_download(request, _unexpected_transport) + + assert result.status is IPCCSourceDownloadExecutionStatus.DOWNLOADED + assert result.downloaded is True + assert result.issues == () + assert result.artifact is not None + assert result.artifact.local_path == str(target_path) + assert result.artifact.storage_identity == str(target_path) + assert result.artifact.source_reference_uri == request.source_reference_uri + assert result.artifact.version_label == "py053_mock_download" + assert result.artifact.retrieved_at_label == ( + "download_execution_retrieved_at_caller_boundary" + ) + assert result.artifact.reused_existing is True + assert result.artifact.checksum_sha256 == sha256(payload).hexdigest() + assert target_path.read_bytes() == payload + + +def test_existing_known_target_checksum_mismatch_fails_without_transport( + tmp_path: Path, +) -> None: + request = replace(_valid_request(tmp_path), expected_checksum_sha256="a" * 64) + target_path = tmp_path / request.target_relative_path + target_path.parent.mkdir(parents=True) + target_path.write_bytes(b"existing ipcc bytes") + + result = execute_ipcc_source_download(request, _unexpected_transport) + + assert result.status is IPCCSourceDownloadExecutionStatus.FAILED + assert result.artifact is None + assert _issue_codes(result.issues) == ( + "IPCC_SOURCE_DOWNLOAD_EXISTING_CHECKSUM_MISMATCH", + ) + + +def test_symlinked_parent_path_cannot_escape_target_root(tmp_path: Path) -> None: + target_root = tmp_path / "target-root" + outside = tmp_path / "outside" + target_root.mkdir() + outside.mkdir() + _create_directory_symlink(outside, target_root / "link") + request = replace( + _valid_request(target_root), + target_relative_path="link/escape.xlsx", + ) + + result = execute_ipcc_source_download( + request, + lambda _: IPCCSourceDownloadTransportResponse(content=b"escape"), + ) + + assert result.status is IPCCSourceDownloadExecutionStatus.BLOCKED + assert result.downloaded is False + assert result.artifact is None + assert _issue_codes(result.issues) == ( + "IPCC_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE", + ) + assert not (outside / "escape.xlsx").exists() + assert not (target_root / "link/escape.xlsx").exists() + + +def test_existing_final_target_symlink_is_rejected(tmp_path: Path) -> None: + target_root = tmp_path / "target-root" + outside = tmp_path / "outside" + target_parent = target_root / "ipcc" + target_parent.mkdir(parents=True) + outside.mkdir() + (target_parent / "escape.xlsx").symlink_to(outside / "escape.xlsx") + request = replace( + _valid_request(target_root), + target_relative_path="ipcc/escape.xlsx", + allow_overwrite=True, + ) + + result = execute_ipcc_source_download( + request, + lambda _: IPCCSourceDownloadTransportResponse(content=b"escape"), + ) + + assert result.status is IPCCSourceDownloadExecutionStatus.BLOCKED + assert result.downloaded is False + assert result.artifact is None + assert _issue_codes(result.issues) == ( + "IPCC_SOURCE_DOWNLOAD_TARGET_SYMLINK_UNSAFE", + ) + assert not (outside / "escape.xlsx").exists() + assert (target_parent / "escape.xlsx").is_symlink() + + +def test_parent_symlink_swap_during_transport_cannot_escape_target_root( + tmp_path: Path, +) -> None: + target_root = tmp_path / "target-root" + outside = tmp_path / "outside" + target_parent = target_root / "ipcc" + target_parent.mkdir(parents=True) + outside.mkdir() + request = replace( + _valid_request(target_root), + target_relative_path="ipcc/escape.xlsx", + ) + + def swapping_transport(_: str) -> IPCCSourceDownloadTransportResponse: + shutil.rmtree(target_parent) + _create_directory_symlink(outside, target_parent) + return IPCCSourceDownloadTransportResponse(content=b"escape") + + result = execute_ipcc_source_download(request, swapping_transport) + + assert result.status is not IPCCSourceDownloadExecutionStatus.DOWNLOADED + assert result.downloaded is False + assert result.artifact is None + assert not (outside / "escape.xlsx").exists() + assert target_parent.is_symlink() + + +def test_missing_no_follow_support_fails_closed_before_transport( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.delattr(download_boundary.os, "O_NOFOLLOW", raising=False) + + result = execute_ipcc_source_download( + _valid_request(tmp_path), + _unexpected_transport, + ) + + assert result.status is IPCCSourceDownloadExecutionStatus.BLOCKED + assert result.downloaded is False + assert _issue_codes(result.issues) == ( + "IPCC_SOURCE_DOWNLOAD_TARGET_FD_UNSUPPORTED", + ) + + +def test_missing_directory_flag_support_fails_closed_before_transport( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.delattr(download_boundary.os, "O_DIRECTORY", raising=False) + + result = execute_ipcc_source_download( + _valid_request(tmp_path), + _unexpected_transport, + ) + + assert result.status is IPCCSourceDownloadExecutionStatus.BLOCKED + assert result.downloaded is False + assert _issue_codes(result.issues) == ( + "IPCC_SOURCE_DOWNLOAD_TARGET_FD_UNSUPPORTED", + ) + + +def test_missing_dir_fd_support_fails_closed_before_transport( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(download_boundary.os, "supports_dir_fd", set()) + + result = execute_ipcc_source_download( + _valid_request(tmp_path), + _unexpected_transport, + ) + + assert result.status is IPCCSourceDownloadExecutionStatus.BLOCKED + assert result.downloaded is False + assert _issue_codes(result.issues) == ( + "IPCC_SOURCE_DOWNLOAD_TARGET_FD_UNSUPPORTED", + ) + + +def test_checksum_mismatch_fails_without_writing_file(tmp_path: Path) -> None: + request = replace(_valid_request(tmp_path), expected_checksum_sha256="a" * 64) + + result = execute_ipcc_source_download( + request, + lambda _: IPCCSourceDownloadTransportResponse(content=b"unexpected"), + ) + + assert result.status is IPCCSourceDownloadExecutionStatus.FAILED + assert result.artifact is None + assert _issue_codes(result.issues) == ( + "IPCC_SOURCE_DOWNLOAD_CHECKSUM_MISMATCH", + ) + assert not (tmp_path / request.target_relative_path).exists() + + +def test_transport_errors_and_empty_content_are_failed_results( + tmp_path: Path, +) -> None: + failed = execute_ipcc_source_download( + _valid_request(tmp_path), + lambda _: (_ for _ in ()).throw(RuntimeError("offline")), + ) + empty = execute_ipcc_source_download( + _valid_request(tmp_path / "other"), + lambda _: IPCCSourceDownloadTransportResponse(content=b""), + ) + + assert failed.status is IPCCSourceDownloadExecutionStatus.FAILED + assert _issue_codes(failed.issues) == ("IPCC_SOURCE_DOWNLOAD_TRANSPORT_FAILED",) + assert empty.status is IPCCSourceDownloadExecutionStatus.FAILED + assert _issue_codes(empty.issues) == ( + "IPCC_SOURCE_DOWNLOAD_RESPONSE_EMPTY_CONTENT", + ) + + +def test_transport_response_validation_fails_closed(tmp_path: Path) -> None: + missing_response = execute_ipcc_source_download( + _valid_request(tmp_path / "missing"), + lambda _: None, # type: ignore[return-value] + ) + missing_content_response = execute_ipcc_source_download( + _valid_request(tmp_path / "missing-content"), + lambda _: IPCCSourceDownloadTransportResponse( + content=None, # type: ignore[arg-type] + ), + ) + blank_metadata_response = execute_ipcc_source_download( + _valid_request(tmp_path / "blank-metadata"), + lambda _: IPCCSourceDownloadTransportResponse( + content=b"content", + content_type=" ", + final_uri=" ", + ), + ) + + assert missing_response.status is IPCCSourceDownloadExecutionStatus.FAILED + assert _issue_codes(missing_response.issues) == ( + "IPCC_SOURCE_DOWNLOAD_RESPONSE_MISSING", + ) + assert ( + missing_content_response.status is IPCCSourceDownloadExecutionStatus.FAILED + ) + assert _issue_codes(missing_content_response.issues) == ( + "IPCC_SOURCE_DOWNLOAD_RESPONSE_MISSING_CONTENT", + ) + assert blank_metadata_response.status is IPCCSourceDownloadExecutionStatus.FAILED + assert _issue_codes(blank_metadata_response.issues) == ( + "IPCC_SOURCE_DOWNLOAD_RESPONSE_BLANK_CONTENT_TYPE", + "IPCC_SOURCE_DOWNLOAD_RESPONSE_BLANK_FINAL_URI", + ) + + +def test_download_execution_dataclasses_are_immutable(tmp_path: Path) -> None: + request = _valid_request(tmp_path) + result = execute_ipcc_source_download( + request, + lambda _: IPCCSourceDownloadTransportResponse(content=b"content"), + ) + + with pytest.raises(FrozenInstanceError): + request.source_key = "changed" # type: ignore[misc] + with pytest.raises(FrozenInstanceError): + result.status = IPCCSourceDownloadExecutionStatus.FAILED # type: ignore[misc] + with pytest.raises(FrozenInstanceError): + result.artifact.local_path = "changed" # type: ignore[union-attr,misc] + + +def test_validation_result_issue_shape_is_structured(tmp_path: Path) -> None: + request = replace(_valid_request(tmp_path), candidate_title="") + + issue = validate_ipcc_source_download_execution_request(request).issues[0] + + assert isinstance(issue, IPCCSourceDownloadExecutionIssue) + assert issue.code == "IPCC_SOURCE_DOWNLOAD_MISSING_CANDIDATE_TITLE" + assert issue.field_name == "candidate_title" + assert issue.severity == "error" + assert issue.message == "candidate_title must be a non-empty string." + assert IPCCSourceDownloadExecutionValidationResult().is_valid is True + assert ( + IPCCSourceDownloadExecutionValidationResult(issues=(issue,)).is_valid + is False + ) + + +def test_result_validation_rejects_side_effect_flags(tmp_path: Path) -> None: + result = replace( + execute_ipcc_source_download( + _valid_request(tmp_path), + lambda _: IPCCSourceDownloadTransportResponse(content=b"content"), + ), + no_database_writes=False, + no_sql=False, + ) + + validation = validate_ipcc_source_download_execution_result(result) + + assert validation.is_valid is False + assert _issue_codes(validation.issues) == ( + "IPCC_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED", + "IPCC_SOURCE_DOWNLOAD_RESULT_SIDE_EFFECT_FLAG_ENABLED", + ) + assert tuple(issue.field_name for issue in validation.issues) == ( + "no_database_writes", + "no_sql", + ) + + +def test_result_validation_rejects_invalid_status(tmp_path: Path) -> None: + result = IPCCSourceDownloadExecutionResult( + status="unknown", # type: ignore[arg-type] + request=_valid_request(tmp_path), + issues=( + IPCCSourceDownloadExecutionIssue( + code="IPCC_SOURCE_DOWNLOAD_TEST_ISSUE", + message="test issue.", + field_name="status", + ), + ), + ) + + validation = validate_ipcc_source_download_execution_result(result) + + assert validation.is_valid is False + assert "IPCC_SOURCE_DOWNLOAD_RESULT_INVALID_STATUS" in _issue_codes( + validation.issues + ) + + +def test_download_execution_import_is_runtime_passive( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import builtins + import os + + module_name = ( + "carbonfactor_parser.source_acquisition." + "ipcc_source_download_execution_boundary" + ) + cleared_modules = _clear_relevant_modules() + + open_calls: list[tuple[object, ...]] = [] + getenv_calls: list[tuple[object, ...]] = [] + + def guard_open(*args: object, **kwargs: object) -> object: + open_calls.append(args) + raise AssertionError("IPCC download execution import read a file") + + def guard_getenv(*args: object, **kwargs: object) -> object: + getenv_calls.append(args) + raise AssertionError("IPCC download execution import read environment") + + def guard_urlopen(*args: object, **kwargs: object) -> object: + raise AssertionError("IPCC download execution import opened network") + + monkeypatch.setattr(builtins, "open", guard_open) + monkeypatch.setattr(os, "getenv", guard_getenv) + monkeypatch.setattr(os, "environ", {}) + monkeypatch.setattr(urllib.request, "urlopen", guard_urlopen) + + try: + imported_before = set(sys.modules) + module = importlib.import_module(module_name) + imported_after = set(sys.modules) + finally: + _restore_modules(cleared_modules) + + assert hasattr(module, "execute_ipcc_source_download") + assert open_calls == [] + assert getenv_calls == [] + + newly_imported = imported_after - imported_before + assert not any( + module_name == prefix or module_name.startswith(f"{prefix}.") + for module_name in newly_imported + for prefix in ( + *BANNED_RUNTIME_MODULE_PREFIXES, + *BANNED_SOURCE_ACQUISITION_RUNTIME_MODULES, + *BANNED_EXECUTABLE_PARSER_MODULES, + ) + ) + + +def test_ipcc_download_execution_boundary_exports_through_contract_api( + tmp_path: Path, +) -> None: + from carbonfactor_parser.source_acquisition import contract_api + + request = contract_api.create_ipcc_source_download_execution_request( + _downloadable_candidate(), + target_root=str(tmp_path), + target_relative_path="ipcc/efdb.xlsx", + allow_download_execution=True, + allow_file_write=True, + ) + + result = contract_api.execute_ipcc_source_download( + request, + lambda _: contract_api.IPCCSourceDownloadTransportResponse(content=b"content"), + ) + + assert result.status is contract_api.IPCCSourceDownloadExecutionStatus.DOWNLOADED + assert result.artifact is not None + assert contract_api.validate_ipcc_source_download_execution_result(result).is_valid + + +def _downloadable_candidate(): + return replace( + create_ipcc_source_discovery_result().candidates[0], + reference_uri="mock://ipcc_efdb/efdb.xlsx", + artifact_kind="xlsx", + content_type=IPCC_XLSX_CONTENT_TYPE, + extension=".xlsx", + version_label="py053_mock_download", + download_allowed=True, + ) + + +def _valid_request(tmp_path: Path) -> IPCCSourceDownloadExecutionRequest: + return create_ipcc_source_download_execution_request( + _downloadable_candidate(), + target_root=str(tmp_path), + target_relative_path="ipcc/efdb.xlsx", + allow_download_execution=True, + allow_file_write=True, + ) + + +def _fixture_request( + tmp_path: Path, + fixture: dict[str, object], +) -> IPCCSourceDownloadExecutionRequest: + candidate = replace( + _downloadable_candidate(), + candidate_id=str(fixture["candidate_id"]), + title=str(fixture["candidate_title"]), + reference_uri=str(fixture["source_reference_uri"]), + artifact_kind=str(fixture["artifact_kind"]), + content_type=str(fixture["content_type"]), + extension=str(fixture["extension"]), + document_year=int(fixture["document_year"]), + reporting_year=int(fixture["reporting_year"]), + version_label=str(fixture["version_label"]), + ) + return create_ipcc_source_download_execution_request( + candidate, + target_root=str(tmp_path), + target_relative_path=str(fixture["target_relative_path"]), + allow_download_execution=True, + allow_file_write=True, + ) + + +def _load_parity_fixture() -> dict[str, object]: + with PARITY_FIXTURE_PATH.open(encoding="utf-8") as fixture_file: + return json.load(fixture_file) + + +def _unexpected_transport( + source_reference_uri: str, +) -> IPCCSourceDownloadTransportResponse: + raise AssertionError(f"unexpected transport call for {source_reference_uri}") + + +def _create_directory_symlink(target: Path, link: Path) -> None: + try: + link.symlink_to(target, target_is_directory=True) + except OSError as error: + pytest.skip(f"directory symlink unavailable: {error}") + + +def _issue_codes( + issues: tuple[IPCCSourceDownloadExecutionIssue, ...], +) -> tuple[str, ...]: + return tuple(issue.code for issue in issues) + + +def _clear_relevant_modules() -> dict[str, object]: + cleared_modules: dict[str, object] = {} + for module_name in tuple(sys.modules): + if module_name == "carbonfactor_parser" or module_name.startswith( + "carbonfactor_parser.source_acquisition", + ): + module = sys.modules.pop(module_name, None) + if module is not None: + cleared_modules[module_name] = module + return cleared_modules + + +def _restore_modules(cleared_modules: dict[str, object]) -> None: + for module_name in tuple(sys.modules): + if module_name == "carbonfactor_parser" or module_name.startswith( + "carbonfactor_parser.source_acquisition", + ): + sys.modules.pop(module_name, None) + sys.modules.update(cleared_modules) diff --git a/tests/test_local_agent_launchd_installer.py b/tests/test_local_agent_launchd_installer.py new file mode 100644 index 00000000..3478a8d8 --- /dev/null +++ b/tests/test_local_agent_launchd_installer.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import json +import os +import plistlib +import subprocess +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +INSTALLER = REPO_ROOT / "scripts" / "install_local_agent_launchd.sh" +UNINSTALLER = REPO_ROOT / "scripts" / "uninstall_local_agent_launchd.sh" + + +def run_script(script: Path, args: list[str], *, home: Path) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env["HOME"] = str(home) + return subprocess.run( + [str(script), *args], + cwd=REPO_ROOT, + env=env, + text=True, + capture_output=True, + check=False, + ) + + +def plist_from_dry_run(stdout: str) -> dict[str, object]: + start = stdout.index("") + len("") + return plistlib.loads(stdout[start:end].encode("utf-8")) + + +def write_config(tmp_path: Path, **overrides: object) -> Path: + log_dir = tmp_path / "agent-logs" + values: dict[str, object] = { + "repo": "ktalpay/CarbonOps-Parser", + "source_root": str(REPO_ROOT), + "agents_root": str(tmp_path / "agents"), + "log_directory": str(log_dir), + } + values.update(overrides) + config = tmp_path / "local-agent.json" + config.write_text(json.dumps(values), encoding="utf-8") + return config + + +def test_install_dry_run_renders_default_launchd_plist(tmp_path: Path) -> None: + config = write_config(tmp_path) + + result = run_script( + INSTALLER, + [ + "--config", + str(config), + "--repo-root", + str(REPO_ROOT), + "--python-bin", + sys.executable, + "--dry-run", + ], + home=tmp_path, + ) + + assert result.returncode == 0, result.stderr + plist = plist_from_dry_run(result.stdout) + assert plist["Label"] == "local.carbonops.agent.supervisor" + assert plist["StartInterval"] == 600 + assert plist["RunAtLoad"] is True + assert plist["ProgramArguments"] == [ + sys.executable, + str(REPO_ROOT / "scripts" / "local_agent_supervisor.py"), + "--config", + str(config), + "--once", + ] + assert plist["StandardOutPath"] == str(tmp_path / "agent-logs" / "local-agent-supervisor.launchd.out.log") + assert plist["StandardErrorPath"] == str(tmp_path / "agent-logs" / "local-agent-supervisor.launchd.err.log") + assert plist["EnvironmentVariables"] == { + "PATH": "/Users/oxygen/.npm-global/bin:/usr/local/bin:/usr/local/opt/python@3.12/libexec/bin:/usr/bin:/bin:/usr/sbin:/sbin" + } + assert "Would run: launchctl bootstrap" in result.stdout + assert not (tmp_path / "Library" / "LaunchAgents").exists() + + +def test_install_dry_run_renders_custom_label_python_and_interval(tmp_path: Path) -> None: + config = write_config(tmp_path) + + result = run_script( + INSTALLER, + [ + "--config", + str(config), + "--repo-root", + str(REPO_ROOT), + "--python-bin", + "/opt/custom/python", + "--interval-seconds", + "42", + "--label", + "local.test.agent", + "--dry-run", + ], + home=tmp_path, + ) + + assert result.returncode == 0, result.stderr + plist = plist_from_dry_run(result.stdout) + assert plist["Label"] == "local.test.agent" + assert plist["StartInterval"] == 42 + assert plist["ProgramArguments"][0] == "/opt/custom/python" + assert f"{tmp_path}/Library/LaunchAgents/local.test.agent.plist" in result.stdout + + +def test_install_dry_run_uses_safe_log_default_when_config_is_unreadable(tmp_path: Path) -> None: + missing_config = tmp_path / "missing.json" + + result = run_script( + INSTALLER, + [ + "--config", + str(missing_config), + "--repo-root", + str(REPO_ROOT), + "--python-bin", + sys.executable, + "--dry-run", + ], + home=tmp_path, + ) + + assert result.returncode == 0, result.stderr + plist = plist_from_dry_run(result.stdout) + assert plist["StandardOutPath"] == str( + tmp_path / "FutureOps" / "Agents" / "CarbonOps-Parser" / ".logs" / "local-agent-supervisor.launchd.out.log" + ) + + +def test_uninstall_dry_run_plans_only_launchctl_and_plist_removal(tmp_path: Path) -> None: + result = run_script( + UNINSTALLER, + ["--label", "local.test.agent", "--dry-run"], + home=tmp_path, + ) + + assert result.returncode == 0, result.stderr + assert "Would run: launchctl bootout gui/" in result.stdout + assert f"Would remove plist: {tmp_path}/Library/LaunchAgents/local.test.agent.plist" in result.stdout + assert "Would not remove logs, config files, repository files, branches, or worktrees." in result.stdout diff --git a/tests/test_local_agent_supervisor.py b/tests/test_local_agent_supervisor.py new file mode 100644 index 00000000..c7ff8736 --- /dev/null +++ b/tests/test_local_agent_supervisor.py @@ -0,0 +1,438 @@ +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from pathlib import Path +from typing import Sequence + +import pytest + + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "local_agent_supervisor.py" +SPEC = importlib.util.spec_from_file_location("local_agent_supervisor", SCRIPT_PATH) +assert SPEC is not None +supervisor = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = supervisor +SPEC.loader.exec_module(supervisor) + + +Issue = supervisor.Issue +PullRequest = supervisor.PullRequest +SupervisorError = supervisor.SupervisorError + + +class FakeRunner: + def __init__( + self, + *, + ready: Sequence[Issue] = (), + in_progress: Sequence[Issue] = (), + prs: Sequence[PullRequest] = (), + dirty: bool = False, + fail_ff: bool = False, + ) -> None: + self.ready = tuple(ready) + self.in_progress = tuple(in_progress) + self.prs = tuple(prs) + self.dirty = dirty + self.fail_ff = fail_ff + self.calls: list[tuple[tuple[str, ...], str | None, Path | None]] = [] + + def __call__(self, command: Sequence[str], stdin: str | None = None, cwd: Path | None = None) -> str: + command_tuple = tuple(command) + self.calls.append((command_tuple, stdin, cwd)) + + if "status" in command_tuple and "--porcelain" in command_tuple: + return "M README.md\n" if self.dirty else "" + + if "fetch" in command_tuple: + if self.fail_ff: + raise SupervisorError("fetch failed") + return "" + + if "branch" in command_tuple and "--show-current" in command_tuple: + return "develop\n" + + if "merge" in command_tuple and "--ff-only" in command_tuple: + return "" + + if command_tuple[:3] == ("gh", "issue", "list"): + label = command_tuple[command_tuple.index("--label") + 1] + issues = self.in_progress if label == "status:in-progress" else self.ready + return json.dumps( + [ + { + "number": issue.number, + "title": issue.title, + "labels": [{"name": label} for label in issue.labels], + "state": issue.state, + } + for issue in issues + ] + ) + + if command_tuple[:3] == ("gh", "pr", "list"): + return json.dumps( + [ + { + "number": pr.number, + "title": pr.title, + "labels": [{"name": label} for label in pr.labels], + "state": pr.state, + "merged": pr.merged, + "comments": [{"body": comment} for comment in pr.comments], + } + for pr in self.prs + ] + ) + + return "" + + +def make_issue(number: int, title: str | None = None, label: str = "status:ready") -> Issue: + return Issue(number=number, title=title or f"[OPS-{number}] Task {number}", labels=(label,)) + + +def make_pr( + number: int, + title: str | None = None, + labels: tuple[str, ...] = ("pr:changes-requested",), + comments: tuple[str, ...] = (), +) -> PullRequest: + return PullRequest( + number=number, + title=title or f"[OPS-{number}] PR {number}", + labels=labels, + comments=comments, + ) + + +def write_config(tmp_path: Path, **overrides: object) -> Path: + source_root = tmp_path / "source" + agents_root = tmp_path / "agents" + source_root.mkdir() + values: dict[str, object] = { + "repo": "ktalpay/CarbonOps-Parser", + "source_root": str(source_root), + "agents_root": str(agents_root), + "base_branch": "develop", + "validation_mode": "minimal", + "python_bin": "/opt/python", + "runner_script_path": str(source_root / "scripts" / "local_codex_task_runner.py"), + "log_directory": str(agents_root / ".logs"), + } + values.update(overrides) + path = tmp_path / "local-agent.json" + path.write_text(json.dumps(values), encoding="utf-8") + return path + + +def make_args(config: Path, *, dry_run: bool = False, once: bool = True) -> argparse.Namespace: + return argparse.Namespace(config=config, dry_run=dry_run, once=once) + + +def commands(fake: FakeRunner) -> list[tuple[str, ...]]: + return [call[0] for call in fake.calls] + + +def expected_pr_list_command() -> tuple[str, ...]: + return ( + "gh", + "pr", + "list", + "--repo", + "ktalpay/CarbonOps-Parser", + "--state", + "open", + "--json", + "number,title,labels,state,mergedAt,comments", + "--limit", + "100", + ) + + +def expected_in_progress_issue_list_command() -> tuple[str, ...]: + return ( + "gh", + "issue", + "list", + "--repo", + "ktalpay/CarbonOps-Parser", + "--state", + "open", + "--label", + "status:in-progress", + "--json", + "number,title,labels,state", + "--limit", + "200", + ) + + +def test_config_loading_reads_expected_values(tmp_path: Path) -> None: + config_path = write_config(tmp_path, base_branch="main", validation_mode="python") + + config = supervisor.load_config(config_path) + + assert config.repo == "ktalpay/CarbonOps-Parser" + assert config.source_root == tmp_path / "source" + assert config.agents_root == tmp_path / "agents" + assert config.base == "main" + assert config.validation_mode == "python" + assert config.python_bin == "/opt/python" + + +def test_dry_run_does_not_invoke_runner(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + config_path = write_config(tmp_path) + fake = FakeRunner(ready=(make_issue(451),)) + invoked: list[tuple[str, ...]] = [] + monkeypatch.setattr(supervisor, "run_runner", lambda command, config: invoked.append(tuple(command))) + + result = supervisor.execute(make_args(config_path, dry_run=True), fake) + + assert result == 0 + assert invoked == [] + + +def test_pr_fix_dispatch_runs_before_ready_issue_dispatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config_path = write_config(tmp_path) + fake = FakeRunner(ready=(make_issue(451),), prs=(make_pr(12),)) + invoked: list[tuple[str, ...]] = [] + + def fake_run_runner(command: Sequence[str], config: object) -> Path: + invoked.append(tuple(command)) + return tmp_path / "runner.log" + + monkeypatch.setattr(supervisor, "run_runner", fake_run_runner) + + result = supervisor.execute(make_args(config_path), fake) + + assert result == 0 + assert len(invoked) == 1 + assert "--pr-number" in invoked[0] + assert "--issue-number" not in invoked[0] + assert not any(command[:3] == ("gh", "issue", "list") for command in commands(fake)) + captured = capsys.readouterr() + assert "Supervisor: cycle start; scanning ktalpay/CarbonOps-Parser." in captured.out + assert "Supervisor: selected PR fix dispatch: #12 [OPS-12] PR 12" in captured.out + + +def test_pr_fix_dry_run_does_not_invoke_runner(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + config_path = write_config(tmp_path) + fake = FakeRunner(ready=(make_issue(451),), prs=(make_pr(12),)) + invoked: list[tuple[str, ...]] = [] + monkeypatch.setattr(supervisor, "run_runner", lambda command, config: invoked.append(tuple(command))) + + result = supervisor.execute(make_args(config_path, dry_run=True), fake) + + assert result == 0 + assert invoked == [] + assert not any(command[:3] == ("gh", "issue", "list") for command in commands(fake)) + + +def test_supervisor_invokes_pr_fix_runner_with_expected_args( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = write_config(tmp_path, python_bin="/custom/python", validation_mode="ops") + fake = FakeRunner(ready=(make_issue(451),), prs=(make_pr(12),)) + invoked: list[tuple[str, ...]] = [] + + def fake_run_runner(command: Sequence[str], config: object) -> Path: + invoked.append(tuple(command)) + return tmp_path / "runner.log" + + monkeypatch.setattr(supervisor, "run_runner", fake_run_runner) + + result = supervisor.execute(make_args(config_path), fake) + + assert result == 0 + command = invoked[0] + assert command[:2] == ( + sys.executable, + str(tmp_path / "source" / "scripts" / "local_codex_pr_fix_runner.py"), + ) + assert ("--repo", "ktalpay/CarbonOps-Parser") == command[2:4] + assert command[command.index("--agents-root") + 1] == str(tmp_path / "agents") + assert "--once" in command + assert command[command.index("--pr-number") + 1] == "12" + assert command[command.index("--validation-mode") + 1] == "ops" + assert command[command.index("--python-bin") + 1] == "/custom/python" + + +def test_no_pr_fix_continues_ready_issue_dispatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = write_config(tmp_path) + fake = FakeRunner( + ready=(make_issue(451),), + prs=(make_pr(12, labels=("docs",)),), + ) + invoked: list[tuple[str, ...]] = [] + + def fake_run_runner(command: Sequence[str], config: object) -> Path: + invoked.append(tuple(command)) + return tmp_path / "runner.log" + + monkeypatch.setattr(supervisor, "run_runner", fake_run_runner) + + result = supervisor.execute(make_args(config_path), fake) + + assert result == 0 + assert len(invoked) == 1 + assert "--issue-number" in invoked[0] + assert commands(fake).index(expected_pr_list_command()) < commands(fake).index( + expected_in_progress_issue_list_command() + ) + + +def test_lock_prevents_concurrent_run( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + lock_path = tmp_path / "supervisor.lock" + lock_path.write_text("pid=1\n", encoding="utf-8") + config_path = write_config(tmp_path, lock_path=str(lock_path)) + fake = FakeRunner(ready=(make_issue(451),)) + monkeypatch.setattr(supervisor, "run_runner", lambda command, config: pytest.fail("runner should not run")) + + result = supervisor.execute(make_args(config_path), fake) + + assert result == 0 + assert commands(fake) == [] + assert "lock already held" in capsys.readouterr().out + + +def test_dirty_source_root_refuses_dispatch(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + config_path = write_config(tmp_path) + fake = FakeRunner(ready=(make_issue(451),), dirty=True) + monkeypatch.setattr(supervisor, "run_runner", lambda command, config: pytest.fail("runner should not run")) + + with pytest.raises(SupervisorError, match="uncommitted changes"): + supervisor.execute(make_args(config_path), fake) + + assert not any(command[:3] == ("gh", "issue", "list") for command in commands(fake)) + + +def test_in_progress_issue_guard_exits_without_dispatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config_path = write_config(tmp_path) + fake = FakeRunner( + ready=(make_issue(451),), + in_progress=(make_issue(450, title="[OPS-025] Previous task", label="status:in-progress"),), + ) + monkeypatch.setattr(supervisor, "run_runner", lambda command, config: pytest.fail("runner should not run")) + + result = supervisor.execute(make_args(config_path), fake) + + assert result == 0 + assert not any(command[-2:] == ("--label", "status:ready") for command in commands(fake)) + captured = capsys.readouterr() + assert "Supervisor: in-progress issue count: 1" in captured.out + assert "Supervisor: found in-progress issue(s); no dispatch:" in captured.out + + +def test_no_ready_queue_exits_without_dispatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config_path = write_config(tmp_path) + fake = FakeRunner() + monkeypatch.setattr(supervisor, "run_runner", lambda command, config: pytest.fail("runner should not run")) + + result = supervisor.execute(make_args(config_path), fake) + + assert result == 0 + captured = capsys.readouterr() + assert "Supervisor: in-progress issue count: 0" in captured.out + assert "Supervisor: ready issue count: 0" in captured.out + assert "Supervisor: no-op; no ready issues found." in captured.out + + +def test_deterministic_ready_issue_selection( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config_path = write_config(tmp_path) + fake = FakeRunner(ready=(make_issue(455), make_issue(451), make_issue(453))) + invoked: list[tuple[str, ...]] = [] + + def fake_run_runner(command: Sequence[str], config: object) -> Path: + invoked.append(tuple(command)) + return tmp_path / "runner.log" + + monkeypatch.setattr(supervisor, "run_runner", fake_run_runner) + + result = supervisor.execute(make_args(config_path), fake) + + assert result == 0 + assert invoked + command = invoked[0] + assert command[command.index("--issue-number") + 1] == "451" + captured = capsys.readouterr() + assert "Supervisor: ready issue count: 3" in captured.out + assert "Supervisor: selected issue dispatch: #451 [OPS-451] Task 451" in captured.out + + +def test_supervisor_invokes_runner_with_expected_args( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = write_config(tmp_path, python_bin="/custom/python") + fake = FakeRunner(ready=(make_issue(451),)) + invoked: list[tuple[str, ...]] = [] + + def fake_run_runner(command: Sequence[str], config: object) -> Path: + invoked.append(tuple(command)) + return tmp_path / "runner.log" + + monkeypatch.setattr(supervisor, "run_runner", fake_run_runner) + + result = supervisor.execute(make_args(config_path), fake) + + assert result == 0 + command = invoked[0] + assert command[:2] == ( + sys.executable, + str(tmp_path / "source" / "scripts" / "local_codex_task_runner.py"), + ) + assert ("--repo", "ktalpay/CarbonOps-Parser") == command[2:4] + assert "--run-once" in command + assert command[command.index("--issue-number") + 1] == "451" + assert command[command.index("--validation-mode") + 1] == "minimal" + assert command[command.index("--python-bin") + 1] == "/custom/python" + +def test_pr_fix_runner_failure_stops_cycle_without_issue_dispatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = write_config(tmp_path) + fake = FakeRunner(ready=(make_issue(451),), prs=(make_pr(12),)) + + def fail_run_runner(command: Sequence[str], config: object) -> Path: + assert "--pr-number" in command + raise SupervisorError("fix runner failed") + + monkeypatch.setattr(supervisor, "run_runner", fail_run_runner) + + with pytest.raises(SupervisorError, match="fix runner failed"): + supervisor.execute(make_args(config_path), fake) + + assert not any( + command[:3] == ("gh", "issue", "list") + for command in commands(fake) + ) diff --git a/tests/test_local_codex_pr_fix_runner.py b/tests/test_local_codex_pr_fix_runner.py new file mode 100644 index 00000000..481c1dbb --- /dev/null +++ b/tests/test_local_codex_pr_fix_runner.py @@ -0,0 +1,277 @@ +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from pathlib import Path +from typing import Sequence + +import pytest + + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "local_codex_pr_fix_runner.py" +SPEC = importlib.util.spec_from_file_location("local_codex_pr_fix_runner", SCRIPT_PATH) +assert SPEC is not None +runner_module = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = runner_module +SPEC.loader.exec_module(runner_module) + + +PullRequest = runner_module.PullRequest +RunnerError = runner_module.RunnerError + + +class FakeRunner: + def __init__( + self, + *, + prs: Sequence[PullRequest] = (), + changes: bool = True, + fail_commands: Sequence[tuple[str, ...]] = (), + ) -> None: + self.prs = tuple(prs) + self.changes = changes + self.fail_commands = tuple(fail_commands) + self.calls: list[tuple[tuple[str, ...], str | None, Path | None]] = [] + + def __call__(self, command: Sequence[str], stdin: str | None = None, cwd: Path | None = None) -> str: + command_tuple = tuple(command) + self.calls.append((command_tuple, stdin, cwd)) + if command_tuple in self.fail_commands: + raise RunnerError(f"forced failure: {' '.join(command_tuple)}") + + if command_tuple[:3] == ("gh", "pr", "list"): + return json.dumps([self._raw_pr(pr) for pr in self.prs if pr.state.upper() == "OPEN"]) + + if command_tuple[:3] == ("gh", "pr", "view"): + number = int(command_tuple[3]) + matches = [pr for pr in self.prs if pr.number == number] + if not matches: + raise RunnerError(f"PR #{number} not found") + return json.dumps(self._raw_pr(matches[0])) + + if "branch" in command_tuple and "--show-current" in command_tuple: + return "feature/pr-fix\n" + + if "status" in command_tuple and "--porcelain" in command_tuple: + return "M scripts/local_codex_pr_fix_runner.py\n" if self.changes else "" + + if "rev-parse" in command_tuple and "HEAD" in command_tuple: + return "abc123def456\n" + + return "" + + @staticmethod + def _raw_pr(pr: PullRequest) -> dict[str, object]: + return { + "number": pr.number, + "title": pr.title, + "body": pr.body, + "labels": [{"name": label} for label in pr.labels], + "state": pr.state, + "merged": pr.merged, + "headRefName": pr.head_branch, + "baseRefName": pr.base_branch, + "comments": [{"body": comment} for comment in pr.comments], + "files": [{"path": path} for path in pr.files], + } + + +def make_pr( + number: int = 12, + *, + labels: tuple[str, ...] = ("pr:changes-requested",), + comments: tuple[str, ...] = (), + state: str = "OPEN", + merged: bool = False, + head_branch: str = "feature/pr-fix", + base_branch: str = "develop", + files: tuple[str, ...] = ("scripts/local_codex_pr_fix_runner.py",), +) -> PullRequest: + return PullRequest( + number=number, + title="Fix parser automation", + body="PR body with implementation context.", + labels=labels, + state=state, + merged=merged, + head_branch=head_branch, + base_branch=base_branch, + comments=comments, + files=files, + ) + + +def make_args(tmp_path: Path, **overrides: object) -> argparse.Namespace: + values: dict[str, object] = { + "repo": "ktalpay/CarbonOps-Parser", + "agents_root": tmp_path / "agents", + "validation_mode": "minimal", + "python_bin": "python", + "pr_number": None, + "dry_run": False, + "once": True, + } + values.update(overrides) + return argparse.Namespace(**values) + + +def commands(fake: FakeRunner) -> list[tuple[str, ...]]: + return [call[0] for call in fake.calls] + + +def comments(fake: FakeRunner) -> list[str]: + bodies: list[str] = [] + for command, _stdin, _cwd in fake.calls: + if command[:3] == ("gh", "pr", "comment"): + bodies.append(command[command.index("--body") + 1]) + return bodies + + +def test_explicit_pr_number_selection(tmp_path: Path) -> None: + fake = FakeRunner(prs=(make_pr(12), make_pr(13))) + + result = runner_module.execute(make_args(tmp_path, pr_number=13), fake) + + assert result == 0 + assert any(command[:4] == ("gh", "pr", "view", "13") for command in commands(fake)) + assert ("git", "-C", str(tmp_path / "agents" / "PR-13"), "push", "origin", "feature/pr-fix") in commands(fake) + + +def test_detects_pr_by_changes_requested_label(tmp_path: Path) -> None: + fake = FakeRunner(prs=(make_pr(21, labels=("docs",)), make_pr(22, labels=("pr:changes-requested",)))) + + result = runner_module.execute(make_args(tmp_path), fake) + + assert result == 0 + assert ("git", "worktree", "add", str(tmp_path / "agents" / "PR-22"), "feature/pr-fix") in commands(fake) + + +def test_detects_pr_by_local_agent_fix_comment(tmp_path: Path) -> None: + pr = make_pr(23, labels=(), comments=("Looks good.", "Please adjust tests. @local-agent fix")) + fake = FakeRunner(prs=(pr,)) + + result = runner_module.execute(make_args(tmp_path), fake) + + assert result == 0 + codex_call = next(call for call in fake.calls if call[0] == ("codex", "exec", "--sandbox", "workspace-write")) + assert "Please adjust tests. @local-agent fix" in (codex_call[1] or "") + + +def test_dry_run_does_not_run_codex_commit_push_or_comment(tmp_path: Path) -> None: + fake = FakeRunner(prs=(make_pr(24),)) + + result = runner_module.execute(make_args(tmp_path, dry_run=True), fake) + + assert result == 0 + command_list = commands(fake) + assert ("codex", "exec", "--sandbox", "workspace-write") not in command_list + assert not any(command[:4] == ("git", "-C", str(tmp_path / "agents" / "PR-24"), "commit") for command in command_list) + assert not any(command[:4] == ("git", "-C", str(tmp_path / "agents" / "PR-24"), "push") for command in command_list) + assert not any(command[:3] == ("gh", "pr", "comment") for command in command_list) + assert not (tmp_path / "agents").exists() + + +def test_closed_and_merged_prs_are_refused_for_explicit_selection(tmp_path: Path) -> None: + closed = FakeRunner(prs=(make_pr(25, state="CLOSED"),)) + with pytest.raises(RunnerError, match="not open"): + runner_module.execute(make_args(tmp_path, pr_number=25), closed) + + merged = FakeRunner(prs=(make_pr(26, merged=True),)) + with pytest.raises(RunnerError, match="merged"): + runner_module.execute(make_args(tmp_path, pr_number=26), merged) + + assert not any(command[:2] == ("codex", "exec") for command in commands(closed)) + assert not any(command[:2] == ("codex", "exec") for command in commands(merged)) + + +def test_closed_and_merged_prs_are_ignored_for_auto_selection(tmp_path: Path) -> None: + fake = FakeRunner( + prs=( + make_pr(27, state="CLOSED"), + make_pr(28, merged=True), + make_pr(29, labels=("pr:changes-requested",)), + ) + ) + + result = runner_module.execute(make_args(tmp_path), fake) + + assert result == 0 + assert ("git", "worktree", "add", str(tmp_path / "agents" / "PR-29"), "feature/pr-fix") in commands(fake) + + +def test_minimal_validation_runs_only_git_diff_check(tmp_path: Path) -> None: + fake = FakeRunner(prs=(make_pr(30),)) + + result = runner_module.execute(make_args(tmp_path, validation_mode="minimal"), fake) + + assert result == 0 + validation_commands = [ + command + for command in commands(fake) + if command == ("python", "-m", "pytest") + or command == ("python", "scripts/check_public_safety.py") + or command[:2] == ("dotnet", "test") + or command[-2:] == ("diff", "--check") + ] + assert validation_commands == [("git", "-C", str(tmp_path / "agents" / "PR-30"), "diff", "--check")] + + +def test_validation_failure_does_not_push_and_comments_recovery(tmp_path: Path) -> None: + worktree = tmp_path / "agents" / "PR-31" + failed_command = ("git", "-C", str(worktree), "diff", "--check") + fake = FakeRunner(prs=(make_pr(31),), fail_commands=(failed_command,)) + + with pytest.raises(runner_module.ValidationError): + runner_module.execute(make_args(tmp_path), fake) + + assert not any(command[:4] == ("git", "-C", str(worktree), "push") for command in commands(fake)) + assert not any(command[:4] == ("git", "-C", str(worktree), "commit") for command in commands(fake)) + comment_body = comments(fake)[0] + assert "validation failed" in comment_body + assert "no push was performed" in comment_body + assert f"failed-command: {' '.join(failed_command)}" in comment_body + assert f"cd {worktree}" in comment_body + + +def test_successful_fix_pushes_same_branch_and_comments_commit_hash(tmp_path: Path) -> None: + fake = FakeRunner(prs=(make_pr(32, head_branch="feature/same-pr-branch"),)) + + result = runner_module.execute(make_args(tmp_path), fake) + + assert result == 0 + assert ("git", "-C", str(tmp_path / "agents" / "PR-32"), "push", "origin", "feature/same-pr-branch") in commands(fake) + comment_body = comments(fake)[0] + assert "abc123def456" in comment_body + assert "validation-mode: minimal" in comment_body + + +def test_no_changes_comments_without_push(tmp_path: Path) -> None: + fake = FakeRunner(prs=(make_pr(33),), changes=False) + + result = runner_module.execute(make_args(tmp_path), fake) + + assert result == 0 + assert not any(command[:4] == ("git", "-C", str(tmp_path / "agents" / "PR-33"), "push") for command in commands(fake)) + assert "no commit or push was needed" in comments(fake)[0] + +def test_explicit_pr_number_without_fix_request_is_refused(tmp_path: Path) -> None: + fake = FakeRunner(prs=(make_pr(34, labels=(), comments=()),)) + + with pytest.raises(RunnerError, match="does not have label"): + runner_module.execute(make_args(tmp_path, pr_number=34), fake) + + assert not any( + command == ("codex", "exec", "--sandbox", "workspace-write") + for command in commands(fake) + ) + assert not any( + command[:4] == ("git", "-C", str(tmp_path / "agents" / "PR-34"), "push") + for command in commands(fake) + ) + assert not any( + command[:3] == ("gh", "pr", "comment") + for command in commands(fake) + ) diff --git a/tests/test_local_codex_task_runner.py b/tests/test_local_codex_task_runner.py new file mode 100644 index 00000000..be95ae8c --- /dev/null +++ b/tests/test_local_codex_task_runner.py @@ -0,0 +1,443 @@ +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from pathlib import Path +from typing import Sequence + +import pytest + + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "local_codex_task_runner.py" +SPEC = importlib.util.spec_from_file_location("local_codex_task_runner", SCRIPT_PATH) +assert SPEC is not None +runner_module = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = runner_module +SPEC.loader.exec_module(runner_module) + + +Issue = runner_module.Issue +RunnerError = runner_module.RunnerError + + +class FakeRunner: + def __init__( + self, + *, + ready: Sequence[Issue] = (), + in_progress: Sequence[Issue] = (), + open_issues: Sequence[Issue] = (), + branch_exists: bool = False, + changes: bool = True, + fail_commands: Sequence[tuple[str, ...]] = (), + pr_create_errors: Sequence[str] = (), + existing_pr_url: str | None = None, + ) -> None: + self.ready = tuple(ready) + self.in_progress = tuple(in_progress) + self.open_issues = tuple(open_issues) or tuple(ready) + tuple(in_progress) + self.branch_exists = branch_exists + self.changes = changes + self.fail_commands = tuple(fail_commands) + self.pr_create_errors = list(pr_create_errors) + self.existing_pr_url = existing_pr_url + self.calls: list[tuple[tuple[str, ...], str | None, Path | None]] = [] + + def __call__( + self, + command: Sequence[str], + stdin: str | None = None, + cwd: Path | None = None, + ) -> str: + command_tuple = tuple(command) + self.calls.append((command_tuple, stdin, cwd)) + if command_tuple in self.fail_commands: + raise RunnerError(f"forced failure: {' '.join(command_tuple)}") + + if command_tuple[:3] == ("gh", "issue", "list"): + if "--label" in command_tuple: + label = command_tuple[command_tuple.index("--label") + 1] + issues = self.in_progress if label == "status:in-progress" else self.ready + else: + issues = self.open_issues + return json.dumps( + [ + { + "number": issue.number, + "title": issue.title, + "body": issue.body, + "labels": [{"name": label} for label in issue.labels], + "state": issue.state, + } + for issue in issues + ] + ) + + if command_tuple[:3] == ("gh", "issue", "view"): + issue_number = int(command_tuple[3]) + matches = [issue for issue in self.open_issues if issue.number == issue_number] + if not matches: + raise RunnerError(f"issue #{issue_number} not found") + issue = matches[0] + return json.dumps( + { + "number": issue.number, + "title": issue.title, + "body": issue.body, + "labels": [{"name": label} for label in issue.labels], + "state": issue.state, + } + ) + + if "rev-parse" in command_tuple and "--verify" in command_tuple: + if self.branch_exists: + return "abc123\n" + raise RunnerError("branch missing") + + if "branch" in command_tuple and "--show-current" in command_tuple: + return "feature/ops-024-add-local-codex-one-shot-task-runner\n" + + if "status" in command_tuple and "--porcelain" in command_tuple: + return "M scripts/local_codex_task_runner.py\n" if self.changes else "" + + if "rev-parse" in command_tuple and "HEAD" in command_tuple: + return "deadbeef1234567890\n" + + if command_tuple[:3] == ("gh", "pr", "create"): + if self.pr_create_errors: + raise RunnerError(self.pr_create_errors.pop(0)) + return "https://github.com/ktalpay/CarbonOps-Parser/pull/445\n" + + if command_tuple[:3] == ("gh", "pr", "list"): + if self.existing_pr_url is None: + return "[]" + return json.dumps([{"url": self.existing_pr_url}]) + + return "" + + +def make_issue( + number: int = 444, + title: str = "[OPS-024] Add local Codex one-shot task runner", + body: str = "Task body\n\nValidation: git diff --check", + labels: tuple[str, ...] = ("status:ready",), +) -> Issue: + return Issue(number=number, title=title, body=body, labels=labels) + + +def make_args(tmp_path: Path, **overrides: object) -> argparse.Namespace: + values: dict[str, object] = { + "repo": "ktalpay/CarbonOps-Parser", + "source_root": tmp_path / "source", + "agents_root": tmp_path / "agents", + "base": "develop", + "run_once": True, + "dry_run": False, + "override_in_progress": False, + "issue_number": None, + "task_id": None, + "validation_mode": "minimal", + "python_bin": "python", + } + values.update(overrides) + return argparse.Namespace(**values) + + +def commands(fake: FakeRunner) -> list[tuple[str, ...]]: + return [call[0] for call in fake.calls] + + +def test_refuses_when_issue_is_in_progress_without_override(tmp_path: Path) -> None: + fake = FakeRunner(ready=(make_issue(),), in_progress=(make_issue(number=443),)) + + with pytest.raises(RunnerError, match="Refusing to run"): + runner_module.execute(make_args(tmp_path), fake) + + assert not any(command[:3] == ("gh", "issue", "edit") for command in commands(fake)) + + +def test_override_allows_ready_issue_selection(tmp_path: Path) -> None: + fake = FakeRunner(ready=(make_issue(),), in_progress=(make_issue(number=443),)) + + result = runner_module.execute(make_args(tmp_path, override_in_progress=True), fake) + + assert result == 0 + assert ( + "gh", + "issue", + "edit", + "444", + "--repo", + "ktalpay/CarbonOps-Parser", + "--remove-label", + "status:ready", + "--add-label", + "status:in-progress", + ) in commands(fake) + + +def test_issue_number_selector_selects_exact_ready_issue(tmp_path: Path) -> None: + issue_444 = make_issue(number=444, title="[OPS-024] Earlier task") + issue_447 = make_issue(number=447, title="[OPS-025] Harden local runner") + fake = FakeRunner(ready=(issue_444, issue_447), open_issues=(issue_444, issue_447)) + + result = runner_module.execute(make_args(tmp_path, issue_number=447), fake) + + assert result == 0 + assert any(command[:4] == ("gh", "issue", "view", "447") for command in commands(fake)) + assert any( + command[:4] == ("git", "-C", str(tmp_path / "agents" / "OPS-025"), "push") + for command in commands(fake) + ) + + +def test_task_id_selector_selects_matching_open_ready_issue(tmp_path: Path) -> None: + issue_444 = make_issue(number=444, title="[OPS-024] Earlier task") + issue_447 = make_issue(number=447, title="[OPS-025] Harden local runner") + fake = FakeRunner(ready=(issue_444, issue_447), open_issues=(issue_444, issue_447)) + + result = runner_module.execute(make_args(tmp_path, task_id="ops-025"), fake) + + assert result == 0 + assert any(command[:3] == ("gh", "issue", "list") and "--label" not in command for command in commands(fake)) + assert any( + command[:4] == ("git", "-C", str(tmp_path / "agents" / "OPS-025"), "push") + for command in commands(fake) + ) + + +def test_selected_issue_must_be_ready(tmp_path: Path) -> None: + issue = make_issue(number=447, title="[OPS-025] Harden local runner", labels=("priority:high",)) + fake = FakeRunner(open_issues=(issue,)) + + with pytest.raises(RunnerError, match="not labeled status:ready"): + runner_module.execute(make_args(tmp_path, issue_number=447), fake) + + assert not any(command[:3] == ("gh", "issue", "edit") for command in commands(fake)) + + +def test_dry_run_does_not_mutate_or_write_files(tmp_path: Path) -> None: + fake = FakeRunner(ready=(make_issue(),)) + + result = runner_module.execute(make_args(tmp_path, dry_run=True), fake) + + assert result == 0 + assert not (tmp_path / "agents").exists() + mutating_commands = [ + command + for command in commands(fake) + if command[:3] == ("gh", "issue", "edit") + or command[:4] == ("git", "-C", str(tmp_path / "source"), "worktree") + or command[:2] == ("codex", "exec") + ] + assert mutating_commands == [] + + +def test_branch_worktree_and_prompt_naming_are_deterministic(tmp_path: Path) -> None: + plan = runner_module.build_plan(make_issue(), tmp_path / "agents", "develop") + + assert plan.task_id == "OPS-024" + assert plan.slug == "add-local-codex-one-shot-task-runner" + assert plan.branch == "feature/ops-024-add-local-codex-one-shot-task-runner" + assert plan.worktree_path == tmp_path / "agents" / "OPS-024" + assert plan.prompt_path == tmp_path / "agents" / ".handoff" / "OPS-024" / "prompt.md" + + +def test_prompt_generation_uses_issue_body_and_safety_constraints(tmp_path: Path) -> None: + issue = make_issue(body="Implement the exact issue body.") + plan = runner_module.build_plan(issue, tmp_path / "agents", "develop") + + prompt = runner_module.generate_prompt(plan, "ktalpay/CarbonOps-Parser") + + assert "# Codex Task OPS-024" in prompt + assert "Issue: #444" in prompt + assert "Implement the exact issue body." in prompt + assert "Do not merge pull requests." in prompt + assert "Do not delete worktrees." in prompt + + +def test_command_planning_uses_expected_codex_validation_push_and_pr_commands(tmp_path: Path) -> None: + fake = FakeRunner(ready=(make_issue(),), branch_exists=False) + + result = runner_module.execute(make_args(tmp_path), fake) + + assert result == 0 + command_list = commands(fake) + assert any(command[:4] == ("git", "-C", str(tmp_path / "source"), "worktree") for command in command_list) + assert ("codex", "exec", "--sandbox", "workspace-write") in command_list + assert any(command[-2:] == ("diff", "--check") for command in command_list) + assert ("python", "-m", "pytest") not in command_list + assert any( + command[:4] == ("git", "-C", str(tmp_path / "agents" / "OPS-024"), "push") + for command in command_list + ) + assert any(command[:3] == ("gh", "pr", "create") for command in command_list) + codex_call = next( + call for call in fake.calls if call[0] == ("codex", "exec", "--sandbox", "workspace-write") + ) + assert "Issue: #444" in (codex_call[1] or "") + + +def test_validation_mode_minimal_runs_only_git_diff_check(tmp_path: Path) -> None: + fake = FakeRunner(ready=(make_issue(),)) + + result = runner_module.execute(make_args(tmp_path, validation_mode="minimal"), fake) + + assert result == 0 + validation_commands = [ + command + for command in commands(fake) + if command == ("python", "-m", "pytest") + or command == ("python", "scripts/check_public_safety.py") + or command[:2] == ("dotnet", "test") + or command[-2:] == ("diff", "--check") + ] + assert validation_commands == [("git", "-C", str(tmp_path / "agents" / "OPS-024"), "diff", "--check")] + + +def test_python_bin_is_used_by_python_validation_mode(tmp_path: Path) -> None: + fake = FakeRunner(ready=(make_issue(),)) + worktree = tmp_path / "agents" / "OPS-024" + (worktree / "scripts").mkdir(parents=True) + (worktree / "scripts" / "check_public_safety.py").write_text("print('ok')\n", encoding="utf-8") + + result = runner_module.execute( + make_args(tmp_path, validation_mode="python", python_bin="/opt/custom/python"), + fake, + ) + + assert result == 0 + assert ("/opt/custom/python", "-m", "pytest") in commands(fake) + assert ("/opt/custom/python", "scripts/check_public_safety.py") in commands(fake) + + +def test_python_bin_is_used_by_ops_validation_mode(tmp_path: Path) -> None: + fake = FakeRunner(ready=(make_issue(),)) + worktree = tmp_path / "agents" / "OPS-024" + (worktree / "tests").mkdir(parents=True) + (worktree / "tests" / "test_local_codex_task_runner.py").write_text("def test_ok(): pass\n", encoding="utf-8") + + result = runner_module.execute( + make_args(tmp_path, validation_mode="ops", python_bin="/opt/custom/python"), + fake, + ) + + assert result == 0 + assert ( + "/opt/custom/python", + "-m", + "pytest", + "-q", + "tests/test_local_codex_task_runner.py", + ) in commands(fake) + + +def test_pr_body_footer_is_present_and_exact(tmp_path: Path) -> None: + fake = FakeRunner(ready=(make_issue(number=447, title="[OPS-025] Harden local runner"),)) + + result = runner_module.execute(make_args(tmp_path), fake) + + assert result == 0 + pr_call = next(call for call in fake.calls if call[0][:3] == ("gh", "pr", "create")) + body = pr_call[0][pr_call[0].index("--body") + 1] + assert body.endswith("Task-ID: OPS-025\nTask-Issue: #447") + assert body.splitlines()[-2:] == ["Task-ID: OPS-025", "Task-Issue: #447"] + + +def test_pr_creation_retries_transient_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + fake = FakeRunner(ready=(make_issue(),), pr_create_errors=("GraphQL: HTTP 504 Gateway Timeout",)) + sleeps: list[float] = [] + monkeypatch.setattr(runner_module.time, "sleep", sleeps.append) + + result = runner_module.execute(make_args(tmp_path), fake) + + assert result == 0 + pr_create_calls = [command for command in commands(fake) if command[:3] == ("gh", "pr", "create")] + assert len(pr_create_calls) == 2 + assert sleeps == [0.1] + + +def test_existing_pr_detection_after_pr_create_timeout(tmp_path: Path) -> None: + existing_url = "https://github.com/ktalpay/CarbonOps-Parser/pull/487" + fake = FakeRunner( + ready=(make_issue(),), + pr_create_errors=("GraphQL: HTTP 504 Gateway Timeout",), + existing_pr_url=existing_url, + ) + + result = runner_module.execute(make_args(tmp_path), fake) + + assert result == 0 + pr_list_call = next(command for command in commands(fake) if command[:3] == ("gh", "pr", "list")) + assert "--head" in pr_list_call + assert pr_list_call[pr_list_call.index("--head") + 1] == "feature/ops-024-add-local-codex-one-shot-task-runner" + assert len([command for command in commands(fake) if command[:3] == ("gh", "pr", "create")]) == 1 + + +def test_recovery_report_after_repeated_pr_creation_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + fake = FakeRunner( + ready=(make_issue(number=487, title="[OPS-030] Add PR creation retry"),), + pr_create_errors=( + "GraphQL: HTTP 504 Gateway Timeout", + "temporary network failure", + "API Gateway timeout", + ), + ) + monkeypatch.setattr(runner_module.time, "sleep", lambda seconds: None) + + with pytest.raises(runner_module.PrCreationError): + runner_module.execute(make_args(tmp_path), fake) + + captured = capsys.readouterr() + assert "PR creation failed after branch push" in captured.err + assert "Task ID: OPS-030" in captured.err + assert "Issue number: #487" in captured.err + assert "Branch: feature/ops-030-add-pr-creation-retry" in captured.err + assert "Commit hash: deadbeef1234567890" in captured.err + assert "Base branch: develop" in captured.err + assert "gh pr create" in captured.err + assert "--head feature/ops-030-add-pr-creation-retry" in captured.err + assert "GraphQL: HTTP 504 Gateway Timeout" in captured.err + assert len([command for command in commands(fake) if command[:3] == ("gh", "pr", "create")]) == 3 + + +def test_non_retryable_pr_creation_failure_remains_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake = FakeRunner(ready=(make_issue(),), pr_create_errors=("GraphQL: head ref must be a branch",)) + monkeypatch.setattr(runner_module.time, "sleep", lambda seconds: pytest.fail("should not sleep")) + + with pytest.raises(RunnerError, match="head ref must be a branch"): + runner_module.execute(make_args(tmp_path), fake) + + assert len([command for command in commands(fake) if command[:3] == ("gh", "pr", "create")]) == 1 + assert len([command for command in commands(fake) if command[:3] == ("gh", "pr", "list")]) == 1 + + +def test_validation_failure_does_not_commit_push_or_create_pr(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + issue = make_issue(number=447, title="[OPS-025] Harden local runner") + worktree = tmp_path / "agents" / "OPS-025" + failed_command = ("git", "-C", str(worktree), "diff", "--check") + fake = FakeRunner(ready=(issue,), fail_commands=(failed_command,)) + + with pytest.raises(runner_module.ValidationError): + runner_module.execute(make_args(tmp_path), fake) + + command_list = commands(fake) + assert not any(command[:4] == ("git", "-C", str(worktree), "commit") for command in command_list) + assert not any(command[:4] == ("git", "-C", str(worktree), "push") for command in command_list) + assert not any(command[:3] == ("gh", "pr", "create") for command in command_list) + captured = capsys.readouterr() + assert "Issue number: #447" in captured.err + assert "Task ID: OPS-025" in captured.err + assert f"Branch: feature/ops-025-harden-local-runner" in captured.err + assert f"Worktree path: {worktree}" in captured.err + assert f"Failed command: {' '.join(failed_command)}" in captured.err diff --git a/tests/test_local_dry_run_cli.py b/tests/test_local_dry_run_cli.py index fff4259c..dbb43aca 100644 --- a/tests/test_local_dry_run_cli.py +++ b/tests/test_local_dry_run_cli.py @@ -574,6 +574,80 @@ def test_local_dry_run_cli_requires_content_type_or_format_hint( assert excinfo.value.code == 2 +def test_validate_ingestion_config_reports_ready_without_connecting( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + archive_root = tmp_path / "archive" + config_path = tmp_path / "ingestion.json" + config_path.write_text( + json.dumps( + { + "archive_root": str(archive_root), + "enabled_source_families": ["ghg_protocol"], + "initial_year": 2024, + "cycle": {"interval_seconds": 0, "max_cycles": 1}, + }, + ), + encoding="utf-8", + ) + + def fail_connect(*args: object, **kwargs: object) -> None: + raise AssertionError("validation must not connect to a database") + + monkeypatch.setattr(sqlite3, "connect", fail_connect) + monkeypatch.setenv("CARBONOPS_POSTGRESQL_HOST", "localhost") + monkeypatch.setenv("CARBONOPS_POSTGRESQL_PORT", "5432") + monkeypatch.setenv("CARBONOPS_POSTGRESQL_DATABASE", "carbonops") + monkeypatch.setenv("CARBONOPS_POSTGRESQL_USERNAME", "carbonops") + monkeypatch.setenv("CARBONOPS_POSTGRESQL_PASSWORD", "secret-value") + monkeypatch.setenv("CARBONOPS_POSTGRESQL_APPLICATION_NAME", "carbonops-test") + + exit_code = parser_cli.main( + ["validate-ingestion-config", "--config", str(config_path)], + ) + + captured = capsys.readouterr() + assert exit_code == 0 + assert "status=ready" in captured.out + assert "postgresql_config_status=ready" in captured.out + assert "enabled_source_families=ghg_protocol" in captured.out + assert "postgresql_password_configured=True" in captured.out + assert "secret-value" not in captured.out + + +def test_validate_ingestion_config_reports_missing_db_env( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = tmp_path / "ingestion.json" + config_path.write_text( + json.dumps({"archive_root": str(tmp_path / "archive")}), + encoding="utf-8", + ) + for env_name in ( + "CARBONOPS_POSTGRESQL_HOST", + "CARBONOPS_POSTGRESQL_PORT", + "CARBONOPS_POSTGRESQL_DATABASE", + "CARBONOPS_POSTGRESQL_USERNAME", + "CARBONOPS_POSTGRESQL_PASSWORD", + "CARBONOPS_POSTGRESQL_APPLICATION_NAME", + ): + monkeypatch.delenv(env_name, raising=False) + + exit_code = parser_cli.main( + ["validate-ingestion-config", "--config", str(config_path)], + ) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "status=blocked" in captured.out + assert "POSTGRESQL_RUNTIME_CONFIG_MISSING_HOST" in captured.out + assert "POSTGRESQL_RUNTIME_CONFIG_MISSING_PASSWORD" in captured.out + + def test_local_dry_run_cli_has_no_db_sql_or_network_behavior( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -610,11 +684,10 @@ def fail_urlopen(*args: object, **kwargs: object) -> None: assert exit_code == 0 -def test_local_dry_run_cli_does_not_scan_directories_or_load_config() -> None: +def test_local_dry_run_cli_does_not_scan_directories() -> None: module_source = inspect.getsource(parser_cli).lower() assert "source_acquisition" not in module_source - assert "config" not in module_source assert "glob(" not in module_source assert "rglob(" not in module_source assert "iterdir(" not in module_source diff --git a/tests/test_normalization_public_api.py b/tests/test_normalization_public_api.py index cd73b5c1..dd374dab 100644 --- a/tests/test_normalization_public_api.py +++ b/tests/test_normalization_public_api.py @@ -11,6 +11,14 @@ ArtificialNormalizationExecutor, ArtificialNormalizationSummaryBuilder, DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS, + DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS, + DEFAULT_SUPPORTED_FACTOR_UNITS, + REDACTED_DIAGNOSTIC_VALUE, + DataQualityDiagnostic, + DataQualityProvenanceContext, + DataQualityValidationCheck, + DataQualityValidationResult, + DataQualityValidationSeverity, DefraDesnzNormalizationMappingResult, DefraDesnzNormalizationMappingStatus, NormalizationInput, @@ -34,12 +42,14 @@ build_normalization_input_from_raw_payload, build_parser_execution_normalization_handoff, build_parser_normalization_handoff, + create_data_quality_diagnostic, create_normalization_input_from_raw_payload, create_normalization_input_record_from_raw_record, map_defra_desnz_normalization_input, map_defra_desnz_normalization_input_record, validate_normalization_input, validate_normalization_input_record, + validate_normalized_factor_output, ) from carbonfactor_parser.normalization.summary import ( NormalizationResultSummary as SummaryModuleNormalizationResultSummary, @@ -52,7 +62,15 @@ "NormalizationResult", "NormalizationResultSummary", "NormalizedRecord", + "DEFAULT_SUPPORTED_FACTOR_UNITS", + "REDACTED_DIAGNOSTIC_VALUE", + "DataQualityDiagnostic", + "DataQualityProvenanceContext", + "DataQualityValidationCheck", + "DataQualityValidationResult", + "DataQualityValidationSeverity", "DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS", + "DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS", "DefraDesnzNormalizationMappingResult", "DefraDesnzNormalizationMappingStatus", "ArtificialNormalizationExecutor", @@ -73,12 +91,14 @@ "build_normalization_input_from_raw_payload", "build_parser_execution_normalization_handoff", "build_parser_normalization_handoff", + "create_data_quality_diagnostic", "create_normalization_input_from_raw_payload", "create_normalization_input_record_from_raw_record", "map_defra_desnz_normalization_input", "map_defra_desnz_normalization_input_record", "validate_normalization_input", "validate_normalization_input_record", + "validate_normalized_factor_output", ) EXPECTED_PUBLIC_EXPORTS = { @@ -87,9 +107,19 @@ "NormalizationResult": contracts.NormalizationResult, "NormalizationResultSummary": SummaryModuleNormalizationResultSummary, "NormalizedRecord": contracts.NormalizedRecord, + "DEFAULT_SUPPORTED_FACTOR_UNITS": normalization.DEFAULT_SUPPORTED_FACTOR_UNITS, + "REDACTED_DIAGNOSTIC_VALUE": normalization.REDACTED_DIAGNOSTIC_VALUE, + "DataQualityDiagnostic": normalization.DataQualityDiagnostic, + "DataQualityProvenanceContext": normalization.DataQualityProvenanceContext, + "DataQualityValidationCheck": normalization.DataQualityValidationCheck, + "DataQualityValidationResult": normalization.DataQualityValidationResult, + "DataQualityValidationSeverity": normalization.DataQualityValidationSeverity, "DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS": ( defra_desnz_mapper.DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS ), + "DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS": ( + defra_desnz_mapper.DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS + ), "DefraDesnzNormalizationMappingResult": ( defra_desnz_mapper.DefraDesnzNormalizationMappingResult ), @@ -130,6 +160,7 @@ handoff.build_parser_execution_normalization_handoff ), "build_parser_normalization_handoff": handoff.build_parser_normalization_handoff, + "create_data_quality_diagnostic": normalization.create_data_quality_diagnostic, "create_normalization_input_from_raw_payload": ( input.create_normalization_input_from_raw_payload ), @@ -146,6 +177,9 @@ "validate_normalization_input_record": ( input.validate_normalization_input_record ), + "validate_normalized_factor_output": ( + normalization.validate_normalized_factor_output + ), } @@ -156,9 +190,19 @@ def test_expected_normalization_public_symbols_import_from_package() -> None: "NormalizationResult": NormalizationResult, "NormalizationResultSummary": NormalizationResultSummary, "NormalizedRecord": NormalizedRecord, + "DEFAULT_SUPPORTED_FACTOR_UNITS": DEFAULT_SUPPORTED_FACTOR_UNITS, + "REDACTED_DIAGNOSTIC_VALUE": REDACTED_DIAGNOSTIC_VALUE, + "DataQualityDiagnostic": DataQualityDiagnostic, + "DataQualityProvenanceContext": DataQualityProvenanceContext, + "DataQualityValidationCheck": DataQualityValidationCheck, + "DataQualityValidationResult": DataQualityValidationResult, + "DataQualityValidationSeverity": DataQualityValidationSeverity, "DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS": ( DEFRA_DESNZ_MINIMAL_NORMALIZATION_FIELDS ), + "DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS": ( + DEFRA_DESNZ_NORMALIZED_MAPPING_FIELDS + ), "DefraDesnzNormalizationMappingResult": ( DefraDesnzNormalizationMappingResult ), @@ -195,6 +239,7 @@ def test_expected_normalization_public_symbols_import_from_package() -> None: build_parser_execution_normalization_handoff ), "build_parser_normalization_handoff": build_parser_normalization_handoff, + "create_data_quality_diagnostic": create_data_quality_diagnostic, "create_normalization_input_from_raw_payload": ( create_normalization_input_from_raw_payload ), @@ -207,6 +252,7 @@ def test_expected_normalization_public_symbols_import_from_package() -> None: ), "validate_normalization_input": validate_normalization_input, "validate_normalization_input_record": validate_normalization_input_record, + "validate_normalized_factor_output": validate_normalized_factor_output, } assert tuple(imported_symbols) == EXPECTED_PUBLIC_SYMBOLS diff --git a/tests/test_normalized_factor_data_quality_validation.py b/tests/test_normalized_factor_data_quality_validation.py new file mode 100644 index 00000000..90bdda00 --- /dev/null +++ b/tests/test_normalized_factor_data_quality_validation.py @@ -0,0 +1,370 @@ +from __future__ import annotations + +from dataclasses import FrozenInstanceError + +import pytest + +from carbonfactor_parser.normalization import ( + REDACTED_DIAGNOSTIC_VALUE, + DataQualityDiagnostic, + DataQualityProvenanceContext, + DataQualityValidationCheck, + DataQualityValidationResult, + DataQualityValidationSeverity, + NormalizationResult, + NormalizedRecord, + create_data_quality_diagnostic, + validate_normalized_factor_output, +) + + +def test_validation_model_represents_blocking_warning_and_info() -> None: + blocking = create_data_quality_diagnostic( + code="BLOCKING", + message="blocking diagnostic", + severity=DataQualityValidationSeverity.BLOCKING_ERROR, + check=DataQualityValidationCheck.REQUIRED_FIELD, + ) + warning = create_data_quality_diagnostic( + code="WARNING", + message="warning diagnostic", + severity=DataQualityValidationSeverity.WARNING, + check=DataQualityValidationCheck.UNIT, + ) + info = create_data_quality_diagnostic( + code="INFO", + message="info diagnostic", + severity=DataQualityValidationSeverity.INFO, + check=DataQualityValidationCheck.STRUCTURE, + ) + + result = DataQualityValidationResult(diagnostics=(blocking, warning, info)) + + assert result.is_valid is False + assert result.has_blocking_errors is True + assert result.blocking_error_count == 1 + assert result.warning_count == 1 + assert result.info_count == 1 + + +def test_validation_severity_and_check_names_are_phase_2_wire_names() -> None: + assert tuple(severity.value for severity in DataQualityValidationSeverity) == ( + "blocking_error", + "warning", + "info", + ) + assert tuple(check.value for check in DataQualityValidationCheck) == ( + "required_field", + "numeric_value", + "unit", + "duplicate_factor_identity", + "provenance", + "structure", + ) + + +def test_missing_required_fields_are_blocking_errors() -> None: + record = _record( + factor_id=" ", + factor_name=None, + ) + + result = validate_normalized_factor_output(NormalizationResult(records=(record,))) + + assert result.is_valid is False + assert _diagnostic_codes(result) == ( + "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD", + "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD", + ) + assert tuple(diagnostic.field_name for diagnostic in result.diagnostics) == ( + "factor_id", + "factor_name", + ) + assert all( + diagnostic.severity == DataQualityValidationSeverity.BLOCKING_ERROR + for diagnostic in result.diagnostics + ) + + +def test_invalid_numeric_values_are_blocking_errors_without_value_leakage() -> None: + record = _record(factor_value="not-a-number") + + result = validate_normalized_factor_output(NormalizationResult(records=(record,))) + + assert result.is_valid is False + assert _diagnostic_codes(result) == ("NORMALIZED_FACTOR_INVALID_NUMERIC_VALUE",) + diagnostic = result.diagnostics[0] + assert diagnostic.field_name == "factor_value" + assert diagnostic.check == DataQualityValidationCheck.NUMERIC_VALUE + assert "not-a-number" not in diagnostic.message + assert "not-a-number" not in repr(diagnostic.context) + + +def test_unsupported_units_are_warnings() -> None: + record = _record(unit="widgets per fortnight") + + result = validate_normalized_factor_output( + NormalizationResult(records=(record,)), + supported_units=("kg CO2e/kWh",), + ) + + assert result.is_valid is True + assert result.warning_count == 1 + assert _diagnostic_codes(result) == ("NORMALIZED_FACTOR_UNSUPPORTED_UNIT",) + assert result.diagnostics[0].severity == DataQualityValidationSeverity.WARNING + + +def test_duplicate_factor_identity_is_blocking_error() -> None: + first = _record(record_id="record-001", factor_id="F1") + duplicate = _record(record_id="record-002", factor_id="F1") + + result = validate_normalized_factor_output( + NormalizationResult(records=(first, duplicate)) + ) + + assert result.is_valid is False + assert _diagnostic_codes(result) == ("NORMALIZED_FACTOR_DUPLICATE_IDENTITY",) + diagnostic = result.diagnostics[0] + assert diagnostic.check == DataQualityValidationCheck.DUPLICATE_FACTOR_IDENTITY + assert dict(diagnostic.context)["first_record_position"] == 1 + assert dict(diagnostic.context)["record_position"] == 2 + + +def test_provenance_gaps_are_warnings_with_safe_source_context() -> None: + record = _record(record_id="record-001", source_reference=None, row_number=None) + + result = validate_normalized_factor_output(NormalizationResult(records=(record,))) + + assert result.is_valid is True + assert _diagnostic_codes(result) == ("NORMALIZED_FACTOR_PROVENANCE_GAP",) + diagnostic = result.diagnostics[0] + assert diagnostic.source_family == "defra_desnz" + assert diagnostic.provenance == DataQualityProvenanceContext( + record_id="record-001", + source_family="defra_desnz", + source_id="defra_desnz", + source_reference=None, + row_number=None, + provenance=None, + document_id=None, + ) + assert diagnostic.context == (("record_position", 1),) + + +def test_diagnostics_are_deterministically_ordered_by_record_then_code() -> None: + first = _record( + record_id="record-001", + factor_id="", + factor_value="bad", + unit="bad-unit", + ) + second = _record(record_id="record-002", factor_id="", unit="bad-unit") + + result = validate_normalized_factor_output( + NormalizationResult(records=(second, first)) + ) + + assert tuple( + (dict(diagnostic.context)["record_position"], diagnostic.code) + for diagnostic in result.diagnostics + ) == ( + (1, "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD"), + (1, "NORMALIZED_FACTOR_UNSUPPORTED_UNIT"), + (2, "NORMALIZED_FACTOR_INVALID_NUMERIC_VALUE"), + (2, "NORMALIZED_FACTOR_MISSING_REQUIRED_FIELD"), + (2, "NORMALIZED_FACTOR_UNSUPPORTED_UNIT"), + ) + + +def test_sensitive_values_are_redacted_from_diagnostic_context() -> None: + diagnostic = create_data_quality_diagnostic( + code="SAFE_CONTEXT", + message="safe context diagnostic", + severity=DataQualityValidationSeverity.INFO, + check=DataQualityValidationCheck.STRUCTURE, + context={ + "api_key": "abc123", + "nested": {"password": "secret-value", "visible": "ok"}, + "source_reference": "https://user:pass@example.invalid/?token=abc123", + "token_values": ("one", "two"), + }, + ) + + context = dict(diagnostic.context) + + assert context["api_key"] == REDACTED_DIAGNOSTIC_VALUE + assert ("password", REDACTED_DIAGNOSTIC_VALUE) in context["nested"] + assert ("visible", "ok") in context["nested"] + assert context["source_reference"] == ( + "https://[REDACTED]@example.invalid/?token=[REDACTED]" + ) + assert context["token_values"] == ( + REDACTED_DIAGNOSTIC_VALUE, + REDACTED_DIAGNOSTIC_VALUE, + ) + assert "abc123" not in repr(diagnostic) + assert "pass" not in repr(diagnostic) + assert "secret-value" not in repr(diagnostic) + + +def test_sensitive_values_are_redacted_from_provenance_context() -> None: + record = _record( + source_reference="https://user:pass@example.invalid/factors.csv?token=abc123", + row_number=None, + unit="unsupported", + ) + + result = validate_normalized_factor_output(NormalizationResult(records=(record,))) + + diagnostic = result.diagnostics[0] + assert diagnostic.provenance is not None + assert diagnostic.provenance.source_reference == ( + "https://[REDACTED]@example.invalid/factors.csv?token=[REDACTED]" + ) + + assert "pass" not in repr(diagnostic) + assert "abc123" not in repr(diagnostic) + + +def test_uri_password_with_at_sign_is_redacted_from_diagnostic_repr() -> None: + sensitive_uri = "postgresql://carbonops:p@ssw0rd@example.invalid/db?password=raw" + + diagnostic = create_data_quality_diagnostic( + code="SAFE_CONTEXT", + message="safe context diagnostic", + severity=DataQualityValidationSeverity.INFO, + check=DataQualityValidationCheck.STRUCTURE, + context={"source_reference": sensitive_uri}, + ) + + rendered = repr(diagnostic) + + assert "p@ssw0rd" not in rendered + assert "password=raw" not in rendered + assert dict(diagnostic.context)["source_reference"] == ( + "postgresql://[REDACTED]@example.invalid/db?password=[REDACTED]" + ) + + +def test_sensitive_dsn_fields_are_redacted_from_diagnostic_repr() -> None: + sensitive_uri = "postgresql://carbonops:p@ssw0rd@example.invalid/db" + + diagnostic = create_data_quality_diagnostic( + code="SAFE_CONTEXT", + message=f"failed dsn={sensitive_uri} connection_string={sensitive_uri}", + severity=DataQualityValidationSeverity.INFO, + check=DataQualityValidationCheck.STRUCTURE, + context={ + "dsn": sensitive_uri, + "connection_string": sensitive_uri, + "passwd_value": "raw-secret", + "pwd_value": "raw-secret", + }, + ) + + rendered = repr(diagnostic) + + assert "p@ssw0rd" not in rendered + assert "raw-secret" not in rendered + assert "dsn=[REDACTED]" in rendered + assert "connection_string=[REDACTED]" in rendered + assert dict(diagnostic.context)["dsn"] == REDACTED_DIAGNOSTIC_VALUE + assert dict(diagnostic.context)["connection_string"] == REDACTED_DIAGNOSTIC_VALUE + + +def test_direct_diagnostic_repr_redacts_sensitive_message_text() -> None: + sensitive_uri = "postgresql://carbonops:p@ssw0rd@example.invalid/db?password=raw" + + diagnostic = DataQualityDiagnostic( + code="SAFE_CONTEXT", + message=f"failed to validate {sensitive_uri}", + severity=DataQualityValidationSeverity.INFO, + check=DataQualityValidationCheck.STRUCTURE, + ) + + rendered = repr(diagnostic) + + assert "p@ssw0rd" not in rendered + assert "password=raw" not in rendered + assert "postgresql://[REDACTED]@example.invalid/db?password=[REDACTED]" in rendered + + +def test_direct_diagnostic_repr_redacts_nested_mapping_context() -> None: + sensitive_uri = "postgresql://carbonops:p@ssw0rd@example.invalid/db?password=raw" + + diagnostic = DataQualityDiagnostic( + code="SAFE_CONTEXT", + message="safe context diagnostic", + severity=DataQualityValidationSeverity.INFO, + check=DataQualityValidationCheck.STRUCTURE, + context=(("nested", {"password": "raw-secret", "uri": sensitive_uri}),), + ) + + rendered = repr(diagnostic) + + assert "p@ssw0rd" not in rendered + assert "password=raw" not in rendered + assert "raw-secret" not in rendered + + +def test_valid_factor_output_has_no_diagnostics() -> None: + result = validate_normalized_factor_output( + NormalizationResult(records=(_record(),)) + ) + + assert result == DataQualityValidationResult() + assert result.is_valid is True + + +def test_validation_model_dataclasses_are_frozen() -> None: + diagnostic = create_data_quality_diagnostic( + code="INFO", + message="info", + severity=DataQualityValidationSeverity.INFO, + check=DataQualityValidationCheck.STRUCTURE, + ) + result = DataQualityValidationResult(diagnostics=(diagnostic,)) + + with pytest.raises(FrozenInstanceError): + diagnostic.code = "changed" + with pytest.raises(FrozenInstanceError): + result.diagnostics = () + + +def _record( + *, + record_id: str = "record-001", + source_reference: str | None = "memory://defra", + source_family: object = "defra_desnz", + source_id: object = "defra_desnz", + source_year: object = "2024", + source_version: object = "v1", + row_number: object = 2, + factor_id: object = "F1", + factor_name: object = "Electricity", + factor_value: object = 0.233, + unit: object = "kg CO2e/kWh", +) -> NormalizedRecord: + fields = ( + ("source_family", source_family), + ("source_id", source_id), + ("source_year", source_year), + ("source_version", source_version), + ("row_number", row_number), + ("factor_id", factor_id), + ("factor_name", factor_name), + ("factor_value", factor_value), + ("unit", unit), + ) + return NormalizedRecord( + record_id=record_id, + fields=tuple((key, value) for key, value in fields if value is not None), + source_reference=source_reference, + is_artificial=False, + ) + + +def _diagnostic_codes( + result: DataQualityValidationResult, +) -> tuple[str, ...]: + return tuple(diagnostic.code for diagnostic in result.diagnostics) diff --git a/tests/test_parsed_factor_persistence_writer.py b/tests/test_parsed_factor_persistence_writer.py new file mode 100644 index 00000000..26680d2c --- /dev/null +++ b/tests/test_parsed_factor_persistence_writer.py @@ -0,0 +1,324 @@ +"""Tests for parsed emission factor source-family persistence writer.""" + +from __future__ import annotations + +from dataclasses import replace +from decimal import Decimal +import json +from pathlib import Path + +from carbonfactor_parser.parsers.file_content_input import ParserFileContentInput +from carbonfactor_parser.parsers.input_artifact_contract import ( + create_phase1_parser_input_artifact, +) +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputRowStatus, + create_parser_normalized_output_batch, + create_parser_normalized_output_row, +) +from carbonfactor_parser.parsers.ghg_protocol_content_parser import ( + parse_ghg_protocol_file_content, +) +from carbonfactor_parser.parsers.defra_desnz_content_parser import ( + parse_defra_desnz_file_content, +) +from carbonfactor_parser.parsers.ipcc_efdb_content_parser import ( + parse_ipcc_efdb_file_content, +) +from carbonfactor_parser.parsers.raw_record import ( + ParsedRawRecord, + ParsedRawRecordPayload, + create_parsed_raw_record, + create_parsed_raw_record_payload, +) +from carbonfactor_parser.persistence.parsed_factor_persistence_writer import ( + ParsedFactorPersistenceStatus, + build_parsed_factor_persistence_command, + persist_parsed_factor_records, +) +from carbonfactor_parser.persistence.postgresql_schema_catalog import SourceFamily +from carbonfactor_parser.persistence.source_family_repository import ( + SourceFamilyDetailRecord, + SourceFamilyMasterRecord, + create_source_family_repository_persist_result, +) + +PARITY_EXPECTATIONS = ( + "tests/fixtures/parity/parsed_factor_persistence_writer_expectations.json" +) + + +class _FakeSourceFamilyRepository: + def __init__(self) -> None: + self.calls: list[ + tuple[ + tuple[SourceFamilyMasterRecord, ...], + tuple[SourceFamilyDetailRecord, ...], + ] + ] = [] + + @property + def provider_name(self) -> str: + return "fake_source_family" + + def persist_source_family_records(self, master_records, detail_records): + self.calls.append((tuple(master_records), tuple(detail_records))) + return create_source_family_repository_persist_result( + provider_name=self.provider_name, + master_records=master_records, + detail_records=detail_records, + ) + + +def test_writer_maps_defra_payload_into_source_family_records() -> None: + payload = _defra_payload() + + command = build_parsed_factor_persistence_command(payload) + + assert command.issues == () + assert len(command.master_records) == 1 + assert len(command.detail_records) == 2 + master = command.master_records[0] + detail = command.detail_records[0] + assert master.source_family is SourceFamily.DEFRA + assert master.source_document_id.startswith("source_document_") + assert master.source_family_master_id.startswith("defra_master_") + assert master.master_external_key.startswith("2024:conversion-factors-2024:") + assert detail.source_family is SourceFamily.DEFRA + assert detail.source_family_master_id == master.source_family_master_id + assert detail.detail_external_key == "DEFRA-2024-ELEC:kWh:CO2e" + assert detail.factor_value == "0.20705" + assert detail.factor_unit == "kWh" + + +def test_writer_maps_normalized_output_batch_with_explicit_source_document() -> None: + artifact = create_phase1_parser_input_artifact( + source_family="ghg_protocol", + artifact_reference="artifact://ghg/factors.csv", + reporting_year=2024, + ) + batch = create_parser_normalized_output_batch( + ( + create_parser_normalized_output_row( + artifact=artifact, + row_id="ghg-row-001", + source_row_number=2, + status=ParserNormalizedOutputRowStatus.VALIDATED, + normalized_fields={ + "source_year": 2024, + "source_version": "ghg-2024", + "factor_id": "GHG-001", + "factor_value": Decimal("1.25"), + "factor_unit": "kgco2e", + }, + ), + ) + ) + + command = build_parsed_factor_persistence_command( + batch, + source_document_id="source-document-001", + ) + + assert command.issues == () + assert command.master_records[0].source_family is SourceFamily.GHG + assert command.master_records[0].source_document_id == "source-document-001" + assert command.detail_records[0].factor_value == "1.25" + assert command.detail_records[0].factor_unit == "kgco2e" + + +def test_writer_matches_shared_parity_fixture_for_fallback_persistence_intent() -> None: + expectations = _parity_expectations() + row_expectation = expectations["normalized_row"] + expected_command = expectations["expected_command"] + artifact = create_phase1_parser_input_artifact( + source_family=row_expectation["source_family"], + artifact_reference=row_expectation["artifact_reference"], + reporting_year=row_expectation["reporting_year"], + ) + batch = create_parser_normalized_output_batch( + ( + create_parser_normalized_output_row( + artifact=artifact, + row_id=row_expectation["row_id"], + source_row_number=row_expectation["source_row_number"], + status=ParserNormalizedOutputRowStatus.VALIDATED, + normalized_fields=dict(row_expectation["fields"]), + ), + ) + ) + + command = build_parsed_factor_persistence_command(batch) + + assert command.issues == () + assert len(command.master_records) == expected_command["master_count"] + assert len(command.detail_records) == expected_command["detail_count"] + assert command.skipped_duplicate_count == expected_command["skipped_duplicate_count"] + master = command.master_records[0] + detail = command.detail_records[0] + expected_master = expected_command["master_record"] + expected_detail = expected_command["detail_record"] + assert master.source_family.value == expected_master["source_family"] + assert master.source_family_master_id == expected_master["source_family_master_id"] + assert master.source_document_id == expected_master["source_document_id"] + assert master.master_external_key == expected_master["master_external_key"] + assert master.lifecycle_status == expected_master["lifecycle_status"] + assert master.record_checksum_sha256 == expected_master["record_checksum_sha256"] + assert master.created_at == expected_master["created_at"] + assert master.updated_at == expected_master["updated_at"] + assert detail.source_family.value == expected_detail["source_family"] + assert detail.source_family_detail_id == expected_detail["source_family_detail_id"] + assert detail.source_family_master_id == expected_detail["source_family_master_id"] + assert detail.detail_external_key == expected_detail["detail_external_key"] + assert detail.factor_value == expected_detail["factor_value"] + assert detail.factor_unit == expected_detail["factor_unit"] + assert detail.lifecycle_status == expected_detail["lifecycle_status"] + assert detail.record_checksum_sha256 == expected_detail["record_checksum_sha256"] + assert detail.created_at == expected_detail["created_at"] + assert detail.updated_at == expected_detail["updated_at"] + + +def test_writer_persists_ghg_defra_and_ipcc_payloads_with_fake_repository() -> None: + repository = _FakeSourceFamilyRepository() + + for payload, expected_family in ( + (_ghg_payload(), SourceFamily.GHG), + (_defra_payload(), SourceFamily.DEFRA), + (_ipcc_payload(), SourceFamily.IPCC), + ): + result = persist_parsed_factor_records(payload, repository) + + assert result.status is ParsedFactorPersistenceStatus.DECLARED + assert result.persisted_master_count == 1 + assert result.persisted_detail_count == len(payload.records) + assert result.master_inserted_count == 1 + assert result.detail_inserted_count == len(payload.records) + assert result.final_status == "declared" + assert result.issues == () + assert result.command is not None + assert len(result.command.master_records) == 1 + assert len(result.command.detail_records) == len(payload.records) + assert all( + record.source_family is expected_family + for record in result.command.master_records + ) + + assert len(repository.calls) == 3 + + +def test_writer_deduplicates_identical_factor_identity_deterministically() -> None: + first = _defra_payload().records[0] + payload = create_parsed_raw_record_payload( + source_family="defra_desnz", + source_id="defra_desnz", + records=(first, first), + source_context={"artifact_reference": "artifact://defra"}, + ) + + command = build_parsed_factor_persistence_command(payload) + + assert command.issues == () + assert command.skipped_duplicate_count == 1 + assert len(command.master_records) == 1 + assert len(command.detail_records) == 1 + + +def test_writer_rejects_duplicate_factor_identity_with_different_content() -> None: + first = _defra_payload().records[0] + conflicting = replace( + first, + raw_fields={ + **dict(first.raw_fields), + "factor_value": Decimal("9.99"), + }, + ) + payload = create_parsed_raw_record_payload( + source_family="defra_desnz", + source_id="defra_desnz", + records=(first, conflicting), + ) + + command = build_parsed_factor_persistence_command(payload) + + assert any( + issue.code == "PARSED_FACTOR_PERSISTENCE_DUPLICATE_DETAIL_CONFLICT" + for issue in command.issues + ) + + +def test_writer_rejects_malformed_input_before_repository_call() -> None: + repository = _FakeSourceFamilyRepository() + malformed = ParsedRawRecordPayload( + source_family="defra_desnz", + source_id="defra_desnz", + records=( + ParsedRawRecord( + source_family="defra_desnz", + source_id="defra_desnz", + record_index=1, + raw_fields={ + "factor_id": "DEFRA-001", + "factor_value": Decimal("1.2"), + "unit": "kgco2e", + }, + ), + ), + ) + + result = persist_parsed_factor_records(malformed, repository) + + assert result.status is ParsedFactorPersistenceStatus.FAILED_VALIDATION + assert repository.calls == [] + assert result.issues[0].code == "PARSED_FACTOR_PERSISTENCE_MISSING_REQUIRED_FIELD" + assert result.issues[0].field_name == "records[1].source_document_id" + + +def _ghg_payload(): + result = parse_ghg_protocol_file_content( + _content_input( + source_family="ghg_protocol", + fixture_name="ghg_protocol/ghg_protocol_sample_factors.csv", + ) + ) + assert result.raw_record_payload is not None + return result.raw_record_payload + + +def _defra_payload(): + result = parse_defra_desnz_file_content( + _content_input( + source_family="defra_desnz", + fixture_name="defra_desnz/defra_desnz_normalized_factors.csv", + ) + ) + assert result.raw_record_payload is not None + return result.raw_record_payload + + +def _ipcc_payload(): + result = parse_ipcc_efdb_file_content( + _content_input( + source_family="ipcc_efdb", + fixture_name="ipcc_efdb/ipcc_efdb_sample_factors.csv", + ) + ) + assert result.raw_record_payload is not None + return result.raw_record_payload + + +def _content_input(*, source_family: str, fixture_name: str) -> ParserFileContentInput: + fixture_path = Path("tests/fixtures/source_documents") / fixture_name + return ParserFileContentInput( + source_family=source_family, + source_id=source_family, + content=fixture_path.read_text(encoding="utf-8"), + artifact_reference=str(fixture_path), + checksum_sha256="c" * 64, + content_type="text/csv", + format_hint="csv", + ) + + +def _parity_expectations() -> dict[str, object]: + with open(PARITY_EXPECTATIONS, encoding="utf-8") as fixture: + return json.load(fixture) diff --git a/tests/test_parser_adapter_readiness_report_contract.py b/tests/test_parser_adapter_readiness_report_contract.py index 2c2f6a09..49d542e6 100644 --- a/tests/test_parser_adapter_readiness_report_contract.py +++ b/tests/test_parser_adapter_readiness_report_contract.py @@ -140,15 +140,24 @@ def test_readiness_report_is_read_only() -> None: report.entries[0].capability.format_hint = "changed" # type: ignore[misc] -def test_readiness_report_is_metadata_only_and_contract_only() -> None: +def test_readiness_report_is_metadata_only_and_reflects_declared_capabilities() -> None: report = build_phase1_parser_adapter_readiness_report() + assert tuple(entry.readiness for entry in report.entries) == ( + "contract_only", + "contract_only", + "content_parser_ready", + ) + assert tuple( + entry.capability.supports_parser_execution for entry in report.entries + ) == (False, False, True) + assert tuple( + entry.capability.supports_content_inspection for entry in report.entries + ) == (False, False, True) + for entry in report.entries: - assert entry.readiness == "contract_only" assert entry.execution_mode == "dry_run" - assert entry.capability.supports_parser_execution is False assert entry.capability.supports_file_reads is False - assert entry.capability.supports_content_inspection is False assert not hasattr(entry, "parse") assert not hasattr(entry, "can_parse") assert not hasattr(entry.capability, "parse") diff --git a/tests/test_parser_adapter_registry_contract.py b/tests/test_parser_adapter_registry_contract.py index f1bca4a3..b1c79640 100644 --- a/tests/test_parser_adapter_registry_contract.py +++ b/tests/test_parser_adapter_registry_contract.py @@ -135,19 +135,26 @@ def test_phase1_parser_adapter_lookup_unknown_values_returns_none() -> None: assert get_phase1_parser_adapter_by_parser_key("unknown_parser", registry) is None -def test_phase1_parser_adapter_registry_stays_contract_only() -> None: +def test_phase1_parser_adapter_registry_reports_declared_capabilities() -> None: registry = create_phase1_parser_adapter_registry() for descriptor in registry.descriptors: assert descriptor.mode is SourceAcquisitionPlanMode.DRY_RUN - assert descriptor.capability.supports_parser_execution is False assert descriptor.capability.supports_file_reads is False - assert descriptor.capability.supports_content_inspection is False assert not hasattr(descriptor, "parse") assert not hasattr(descriptor, "can_parse") assert not hasattr(descriptor.capability, "parse") assert not hasattr(descriptor.capability, "can_parse") + assert tuple( + descriptor.capability.supports_parser_execution + for descriptor in registry.descriptors + ) == (False, False, True) + assert tuple( + descriptor.capability.supports_content_inspection + for descriptor in registry.descriptors + ) == (False, False, True) + def test_phase1_parser_adapter_registry_uses_safe_passive_identifiers() -> None: registry = create_phase1_parser_adapter_registry() diff --git a/tests/test_parser_contract_public_api.py b/tests/test_parser_contract_public_api.py index 512ca963..8775f672 100644 --- a/tests/test_parser_contract_public_api.py +++ b/tests/test_parser_contract_public_api.py @@ -36,6 +36,11 @@ "ParserRunContractValidationIssue", "ParserRunContractValidationResult", "ParserRunRequest", + "ParserRunRepository", + "ParserRunRepositoryIssue", + "ParserRunRepositoryPersistResult", + "ParserRunRepositoryPersistStatus", + "ParserRunRepositoryValidationResult", "ParserRunResult", "ParserRunStatus", "ParserRunSummary", @@ -49,6 +54,7 @@ "build_phase1_parser_adapter_readiness_report", "create_parser_normalized_output_batch", "create_parser_normalized_output_row", + "create_parser_run_repository_persist_result", "create_parser_run_request", "create_parser_run_result", "create_parser_validation_issue", @@ -66,6 +72,7 @@ "validate_parser_input_artifact", "validate_parser_normalized_output_batch", "validate_parser_normalized_output_row", + "validate_parser_run_repository_inputs", "validate_parser_run_request", "validate_parser_run_result", "validate_parser_validation_issue", @@ -182,6 +189,7 @@ def test_parser_contract_api_does_not_export_internal_module_names() -> None: assert "normalized_output_row_contract" not in contract_api.__all__ assert "validation_issue_contract" not in contract_api.__all__ assert "parser_run_contract" not in contract_api.__all__ + assert "run_repository_contract" not in contract_api.__all__ assert "dry_run_boundary_contract" not in contract_api.__all__ assert all(not name.startswith("_") for name in contract_api.__all__) diff --git a/tests/test_parser_input_mapping.py b/tests/test_parser_input_mapping.py index 63730c3f..b26220cd 100644 --- a/tests/test_parser_input_mapping.py +++ b/tests/test_parser_input_mapping.py @@ -16,6 +16,13 @@ Path(__file__).resolve().parents[0] / "fixtures" / "source_documents" / "defra_desnz" ) +EXPECTED_FIXTURE_FILE_NAMES = [ + "defra_desnz_malformed_factors.csv", + "defra_desnz_metadata.json", + "defra_desnz_normalized_factors.csv", + "defra_desnz_sample_factors.csv", +] + def test_mapping_can_be_created_from_no_documents() -> None: mapping = build_fixture_parser_input_mapping(()) @@ -34,12 +41,9 @@ def test_mapping_can_be_created_from_discovered_fixture_documents() -> None: mapping = build_fixture_parser_input_mapping(documents) - assert mapping.document_count == 2 + assert mapping.document_count == 4 assert mapping.source_family == SourceFamily.DEFRA_DESNZ - assert [entry.file_name for entry in mapping.entries] == [ - "defra_desnz_metadata.json", - "defra_desnz_sample_factors.csv", - ] + assert [entry.file_name for entry in mapping.entries] == EXPECTED_FIXTURE_FILE_NAMES def test_mapping_entry_ordering_is_deterministic() -> None: diff --git a/tests/test_parser_input_mapping_example.py b/tests/test_parser_input_mapping_example.py index dfb8f0c3..d39c8c63 100644 --- a/tests/test_parser_input_mapping_example.py +++ b/tests/test_parser_input_mapping_example.py @@ -29,7 +29,7 @@ def test_parser_input_mapping_example_returns_deterministic_fields() -> None: def test_parser_input_mapping_example_includes_expected_document_count() -> None: mapping = build_parser_input_mapping_example() - assert mapping["document_count"] == 2 + assert mapping["document_count"] == 4 assert mapping["warnings"] == () @@ -40,7 +40,9 @@ def test_parser_input_mapping_example_derives_file_names_and_extensions() -> Non (entry["file_name"], entry["file_extension"]) for entry in mapping["entries"] ] == [ + ("defra_desnz_malformed_factors.csv", ".csv"), ("defra_desnz_metadata.json", ".json"), + ("defra_desnz_normalized_factors.csv", ".csv"), ("defra_desnz_sample_factors.csv", ".csv"), ] @@ -76,7 +78,7 @@ def fail_open(*args, **kwargs): mapping = build_parser_input_mapping_example() - assert mapping["document_count"] == 2 + assert mapping["document_count"] == 4 def test_parser_input_mapping_example_does_not_use_real_source_data() -> None: @@ -95,6 +97,8 @@ def test_parser_input_mapping_example_does_not_change_mapping_behavior() -> None assert mapping["source_family"] == "defra_desnz" assert mapping["source_name"] == "fixture_parser_input_mapping" assert [entry["document_id"] for entry in mapping["entries"]] == [ + "defra_desnz:defra_desnz_malformed_factors.csv", "defra_desnz:defra_desnz_metadata.json", + "defra_desnz:defra_desnz_normalized_factors.csv", "defra_desnz:defra_desnz_sample_factors.csv", ] diff --git a/tests/test_parser_normalized_output_row_contract.py b/tests/test_parser_normalized_output_row_contract.py index 2ed3e076..ae58c1ba 100644 --- a/tests/test_parser_normalized_output_row_contract.py +++ b/tests/test_parser_normalized_output_row_contract.py @@ -284,6 +284,18 @@ def test_invalid_status_returns_invalid_result() -> None: assert _issue_codes(result) == ("PARSER_NORMALIZED_ROW_INVALID_STATUS",) +def test_reloaded_status_enum_value_remains_valid() -> None: + pre_reload_status = ParserNormalizedOutputRowStatus.VALIDATED + row = replace(_valid_row("defra_desnz"), status=pre_reload_status) + module_name = "carbonfactor_parser.parsers.normalized_output_row_contract" + sys.modules.pop(module_name, None) + reloaded_module = importlib.import_module(module_name) + + result = reloaded_module.validate_parser_normalized_output_row(row) + + assert result.is_valid is True + + def test_normalized_row_contract_is_read_only() -> None: row = _valid_row("ghg_protocol") batch = create_parser_normalized_output_batch((row,)) diff --git a/tests/test_parser_pipeline_summary.py b/tests/test_parser_pipeline_summary.py index 65a7d360..479f4d7b 100644 --- a/tests/test_parser_pipeline_summary.py +++ b/tests/test_parser_pipeline_summary.py @@ -21,6 +21,13 @@ Path(__file__).resolve().parents[0] / "fixtures" / "source_documents" / "defra_desnz" ) +EXPECTED_DEFRA_DESNZ_FIXTURE_SOURCE_NAMES = ( + "defra_desnz:defra_desnz_malformed_factors.csv", + "defra_desnz:defra_desnz_metadata.json", + "defra_desnz:defra_desnz_normalized_factors.csv", + "defra_desnz:defra_desnz_sample_factors.csv", +) + def _document( *, @@ -178,9 +185,9 @@ def test_summary_works_with_existing_fixture_only_pipeline_components() -> None: summary = summarize_parser_pipeline(documents, mapping, parser_result) - assert summary.discovered_document_count == 2 - assert summary.mapping_entry_count == 2 - assert summary.parser_record_count == 2 + assert summary.discovered_document_count == 4 + assert summary.mapping_entry_count == 4 + assert summary.parser_record_count == 4 assert summary.parser_warning_count == 0 assert summary.parser_error_count == 0 assert summary.has_discovered_documents is True @@ -188,10 +195,7 @@ def test_summary_works_with_existing_fixture_only_pipeline_components() -> None: assert summary.has_parser_records is True assert summary.is_clean is True assert summary.source_families == (SourceFamily.DEFRA_DESNZ,) - assert summary.source_names == ( - "defra_desnz:defra_desnz_metadata.json", - "defra_desnz:defra_desnz_sample_factors.csv", - ) + assert summary.source_names == EXPECTED_DEFRA_DESNZ_FIXTURE_SOURCE_NAMES def test_summary_does_not_perform_file_io(monkeypatch, tmp_path) -> None: diff --git a/tests/test_parser_pipeline_summary_example.py b/tests/test_parser_pipeline_summary_example.py index 85cccbd6..d26e2a97 100644 --- a/tests/test_parser_pipeline_summary_example.py +++ b/tests/test_parser_pipeline_summary_example.py @@ -35,7 +35,7 @@ def test_parser_pipeline_summary_example_returns_deterministic_fields() -> None: def test_parser_pipeline_summary_example_counts_are_consistent() -> None: result = build_parser_pipeline_summary_example() - assert result["discovered_document_count"] == 2 + assert result["discovered_document_count"] == 4 assert result["mapping_entry_count"] == result["discovered_document_count"] assert result["parser_record_count"] == result["mapping_entry_count"] @@ -63,7 +63,9 @@ def test_parser_pipeline_summary_example_includes_expected_source_metadata() -> assert result["source_families"] == ("defra_desnz",) assert result["source_names"] == ( + "defra_desnz:defra_desnz_malformed_factors.csv", "defra_desnz:defra_desnz_metadata.json", + "defra_desnz:defra_desnz_normalized_factors.csv", "defra_desnz:defra_desnz_sample_factors.csv", ) @@ -72,7 +74,7 @@ def test_parser_pipeline_summary_example_uses_artificial_fixture_directory() -> result = build_parser_pipeline_summary_example() assert FIXTURE_DIRECTORY.is_dir() - assert result["discovered_document_count"] == 2 + assert result["discovered_document_count"] == 4 def test_parser_pipeline_summary_example_does_not_read_file_contents(monkeypatch) -> None: @@ -83,7 +85,7 @@ def fail_open(*args, **kwargs): result = build_parser_pipeline_summary_example() - assert result["parser_record_count"] == 2 + assert result["parser_record_count"] == 4 def test_parser_pipeline_summary_example_does_not_use_real_source_data() -> None: @@ -100,6 +102,6 @@ def test_parser_pipeline_summary_example_does_not_use_real_source_data() -> None def test_parser_pipeline_summary_example_does_not_change_pipeline_behavior() -> None: result = build_parser_pipeline_summary_example() - assert result["discovered_document_count"] == 2 - assert result["mapping_entry_count"] == 2 - assert result["parser_record_count"] == 2 + assert result["discovered_document_count"] == 4 + assert result["mapping_entry_count"] == 4 + assert result["parser_record_count"] == 4 diff --git a/tests/test_parser_public_api.py b/tests/test_parser_public_api.py index 77028b25..da158d74 100644 --- a/tests/test_parser_public_api.py +++ b/tests/test_parser_public_api.py @@ -9,6 +9,10 @@ defra_desnz_parser, example_parser, example_source_specific_parser, + ghg_protocol_adapter, + ghg_protocol_content_parser, + ipcc_efdb_adapter, + ipcc_efdb_content_parser, execution_plan, execution_result, execution_runner, @@ -26,10 +30,15 @@ ArtificialParserAdapter, DEFAULT_PARSER_FILE_CONTENT_MAX_BYTES, DEFRA_DESNZ_MINIMAL_CONTENT_HEADER, + DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER, DefraDesnzParserAdapter, DefraDesnzParser, ExampleInMemoryParser, ExampleSourceSpecificParser, + GHGProtocolParserAdapter, + GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER, + IPCC_EFDB_NORMALIZED_CONTENT_HEADER, + IpccEfdbParserAdapter, NoopParserAdapter, ParserAdapter, ParserAdapterRegistry, @@ -70,6 +79,8 @@ load_parser_file_content_from_local_path, plan_parser_execution, parse_defra_desnz_file_content, + parse_ghg_protocol_file_content, + parse_ipcc_efdb_file_content, register_parser_adapter, resolve_parser_adapters, run_parser_execution, @@ -85,10 +96,15 @@ "ArtificialFixtureParser", "ArtificialParserAdapter", "DEFRA_DESNZ_MINIMAL_CONTENT_HEADER", + "DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER", "DefraDesnzParserAdapter", "DefraDesnzParser", "ExampleInMemoryParser", "ExampleSourceSpecificParser", + "GHGProtocolParserAdapter", + "GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER", + "IPCC_EFDB_NORMALIZED_CONTENT_HEADER", + "IpccEfdbParserAdapter", "DEFAULT_PARSER_FILE_CONTENT_MAX_BYTES", "NoopParserAdapter", "ParserAdapter", @@ -129,6 +145,8 @@ "load_parser_file_content_from_local_path", "plan_parser_execution", "parse_defra_desnz_file_content", + "parse_ghg_protocol_file_content", + "parse_ipcc_efdb_file_content", "register_parser_adapter", "resolve_parser_adapters", "run_parser_execution", @@ -146,12 +164,23 @@ "DEFRA_DESNZ_MINIMAL_CONTENT_HEADER": ( defra_desnz_content_parser.DEFRA_DESNZ_MINIMAL_CONTENT_HEADER ), + "DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER": ( + defra_desnz_content_parser.DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER + ), "DefraDesnzParserAdapter": defra_desnz_adapter.DefraDesnzParserAdapter, "DefraDesnzParser": defra_desnz_parser.DefraDesnzParser, "ExampleInMemoryParser": example_parser.ExampleInMemoryParser, "ExampleSourceSpecificParser": ( example_source_specific_parser.ExampleSourceSpecificParser ), + "GHGProtocolParserAdapter": ghg_protocol_adapter.GHGProtocolParserAdapter, + "GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER": ( + ghg_protocol_content_parser.GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER + ), + "IPCC_EFDB_NORMALIZED_CONTENT_HEADER": ( + ipcc_efdb_content_parser.IPCC_EFDB_NORMALIZED_CONTENT_HEADER + ), + "IpccEfdbParserAdapter": ipcc_efdb_adapter.IpccEfdbParserAdapter, "DEFAULT_PARSER_FILE_CONTENT_MAX_BYTES": ( file_content_loader.DEFAULT_PARSER_FILE_CONTENT_MAX_BYTES ), @@ -220,6 +249,12 @@ "parse_defra_desnz_file_content": ( defra_desnz_content_parser.parse_defra_desnz_file_content ), + "parse_ghg_protocol_file_content": ( + ghg_protocol_content_parser.parse_ghg_protocol_file_content + ), + "parse_ipcc_efdb_file_content": ( + ipcc_efdb_content_parser.parse_ipcc_efdb_file_content + ), "register_parser_adapter": adapter_registry.register_parser_adapter, "resolve_parser_adapters": adapter_registry.resolve_parser_adapters, "run_parser_execution": execution_runner.run_parser_execution, @@ -245,10 +280,21 @@ def test_expected_parser_public_symbols_import_from_package() -> None: "DEFRA_DESNZ_MINIMAL_CONTENT_HEADER": ( DEFRA_DESNZ_MINIMAL_CONTENT_HEADER ), + "DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER": ( + DEFRA_DESNZ_NORMALIZED_CONTENT_HEADER + ), "DefraDesnzParserAdapter": DefraDesnzParserAdapter, "DefraDesnzParser": DefraDesnzParser, "ExampleInMemoryParser": ExampleInMemoryParser, "ExampleSourceSpecificParser": ExampleSourceSpecificParser, + "GHGProtocolParserAdapter": GHGProtocolParserAdapter, + "GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER": ( + GHG_PROTOCOL_NORMALIZED_CONTENT_HEADER + ), + "IPCC_EFDB_NORMALIZED_CONTENT_HEADER": ( + IPCC_EFDB_NORMALIZED_CONTENT_HEADER + ), + "IpccEfdbParserAdapter": IpccEfdbParserAdapter, "DEFAULT_PARSER_FILE_CONTENT_MAX_BYTES": ( DEFAULT_PARSER_FILE_CONTENT_MAX_BYTES ), @@ -293,6 +339,8 @@ def test_expected_parser_public_symbols_import_from_package() -> None: ), "plan_parser_execution": plan_parser_execution, "parse_defra_desnz_file_content": parse_defra_desnz_file_content, + "parse_ghg_protocol_file_content": parse_ghg_protocol_file_content, + "parse_ipcc_efdb_file_content": parse_ipcc_efdb_file_content, "register_parser_adapter": register_parser_adapter, "resolve_parser_adapters": resolve_parser_adapters, "run_parser_execution": run_parser_execution, @@ -343,6 +391,8 @@ def test_parser_all_excludes_internal_module_names() -> None: assert "fixture_parser" not in parsers.__all__ assert "input_contract" not in parsers.__all__ assert "input_mapping" not in parsers.__all__ + assert "ipcc_efdb_adapter" not in parsers.__all__ + assert "ipcc_efdb_content_parser" not in parsers.__all__ assert "noop_adapter" not in parsers.__all__ assert "pipeline_summary" not in parsers.__all__ assert "raw_record" not in parsers.__all__ diff --git a/tests/test_parser_run_contract.py b/tests/test_parser_run_contract.py index 913d0794..d0c3ec26 100644 --- a/tests/test_parser_run_contract.py +++ b/tests/test_parser_run_contract.py @@ -15,6 +15,7 @@ create_phase1_parser_input_artifact, ) from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputRowStatus, create_parser_normalized_output_row, ) from carbonfactor_parser.parsers.parser_run_contract import ( @@ -549,6 +550,7 @@ def _row_for_artifact( return create_parser_normalized_output_row( artifact=artifact, row_id=row_id, + status=ParserNormalizedOutputRowStatus.VALIDATED, normalized_fields={ "activity_name": artifact.source_family, "unit": "kg", diff --git a/tests/test_parser_run_repository_contract.py b/tests/test_parser_run_repository_contract.py new file mode 100644 index 00000000..ec0c7dcc --- /dev/null +++ b/tests/test_parser_run_repository_contract.py @@ -0,0 +1,166 @@ +"""Tests for parser run repository contract metadata helpers.""" + +from __future__ import annotations + +from dataclasses import FrozenInstanceError +import inspect + +import pytest + +from carbonfactor_parser.parsers.contract_api import ( + ParserRunRepository, + ParserRunRepositoryIssue, + ParserRunRepositoryPersistStatus, + ParserRunStatus, + create_parser_run_repository_persist_result, + create_parser_run_request, + create_parser_run_result, + create_phase1_parser_input_artifact, + validate_parser_run_repository_inputs, +) + + +class _InMemoryRepository: + @property + def provider_name(self) -> str: + return "in_memory" + + def persist_runs(self, runs): + return create_parser_run_repository_persist_result( + provider_name=self.provider_name, + runs=runs, + ) + + +def _sample_run_result(): + artifact = create_phase1_parser_input_artifact( + source_family="defra_desnz", + artifact_reference="artifact://phase1/defra_desnz", + ) + request = create_parser_run_request( + source_family="defra_desnz", + artifacts=(artifact,), + ) + return create_parser_run_result( + request=request, + status=ParserRunStatus.DECLARED, + ) + + +def test_parser_run_repository_protocol_shape() -> None: + repository: ParserRunRepository = _InMemoryRepository() + + result = repository.persist_runs((_sample_run_result(),)) + + assert isinstance(repository, ParserRunRepository) + assert repository.provider_name == "in_memory" + assert result.status is ParserRunRepositoryPersistStatus.DECLARED + assert result.persisted_count == 1 + assert result.issues == () + + +def test_parser_run_repository_validation_requires_provider_name() -> None: + validation = validate_parser_run_repository_inputs( + provider_name="", + runs=(_sample_run_result(),), + ) + + assert validation.is_valid is False + assert validation.issues[0].code == "PARSER_RUN_REPOSITORY_MISSING_PROVIDER_NAME" + + +def test_parser_run_repository_validation_requires_run_instances() -> None: + validation = validate_parser_run_repository_inputs( + provider_name="in_memory", + runs=(object(),), + ) + + assert validation.is_valid is False + assert validation.issues[0].code == "PARSER_RUN_REPOSITORY_INVALID_RUN" + + +def test_parser_run_repository_persist_result_reports_validation_failure() -> None: + result = create_parser_run_repository_persist_result( + provider_name="", + runs=(object(),), + ) + + assert result.status is ParserRunRepositoryPersistStatus.FAILED_VALIDATION + assert result.persisted_count == 0 + assert len(result.issues) == 2 + + +def test_parser_run_repository_persist_result_snapshots_issue_collections() -> None: + issues = [ + ParserRunRepositoryIssue( + code="CUSTOM_REPOSITORY_WARNING", + message="custom issue", + field_name="runs", + severity="warning", + ), + ] + + result = create_parser_run_repository_persist_result( + provider_name="in_memory", + runs=(_sample_run_result(),), + issues=issues, + ) + issues.clear() + + assert result.status is ParserRunRepositoryPersistStatus.FAILED_VALIDATION + assert result.persisted_count == 0 + assert len(result.issues) == 1 + assert result.issues[0].code == "CUSTOM_REPOSITORY_WARNING" + + +def test_parser_run_repository_contract_values_are_immutable() -> None: + issue = ParserRunRepositoryIssue( + code="CUSTOM_REPOSITORY_WARNING", + message="custom issue", + field_name="runs", + ) + + with pytest.raises(FrozenInstanceError): + issue.code = "CHANGED" + + +def test_parser_run_repository_persist_status_values_are_deterministic() -> None: + assert tuple(status.value for status in ParserRunRepositoryPersistStatus) == ( + "declared", + "failed_validation", + ) + + +def test_parser_run_repository_contract_remains_runtime_passive() -> None: + public_members = { + name + for contract_type in ( + ParserRunRepository, + ParserRunRepositoryIssue, + ParserRunRepositoryPersistStatus, + ) + for name, _ in inspect.getmembers(contract_type) + if not name.startswith("_") + } + blocked_terms = ( + "db", + "sql", + "postgres", + "http", + "open", + "read_file", + "write", + "stat_file", + "exists", + "fetch", + "calculate", + "factor", + ) + + assert not any( + term in member.lower() + for member in public_members + for term in blocked_terms + ) + assert "parse" not in public_members + assert "execute" not in public_members diff --git a/tests/test_persistence_public_api.py b/tests/test_persistence_public_api.py index 97be8771..d470b893 100644 --- a/tests/test_persistence_public_api.py +++ b/tests/test_persistence_public_api.py @@ -13,13 +13,17 @@ postgresql_psycopg_session_adapter, postgresql_repository, postgresql_repository_disabled_execution_preview, + postgresql_runtime_config_gate, postgresql_runtime_execution_gate, postgresql_schema_bootstrap, postgresql_schema_bootstrap_planner, postgresql_schema_isolation_strategy, postgresql_transaction_policy, + parsed_factor_persistence_writer, repository, schema, + source_document_repository, + source_family_repository, ) from carbonfactor_parser.persistence import ( PersistenceInput, @@ -40,6 +44,22 @@ PersistenceRepository, PersistenceResult, PersistenceResultStatus, + ParsedFactorPersistenceCommand, + ParsedFactorPersistenceIssue, + ParsedFactorPersistenceStatus, + ParsedFactorPersistenceWriterResult, + SourceDocumentRepository, + SourceDocumentRepositoryIssue, + SourceDocumentRepositoryPersistResult, + SourceDocumentRepositoryPersistStatus, + SourceDocumentRepositoryValidationResult, + SourceFamilyDetailRecord, + SourceFamilyMasterRecord, + SourceFamilyRepository, + SourceFamilyRepositoryIssue, + SourceFamilyRepositoryPersistResult, + SourceFamilyRepositoryPersistStatus, + SourceFamilyRepositoryValidationResult, PsycopgPostgreSQLSessionAdapter, PsycopgPostgreSQLSessionAdapterBoundaryResult, PsycopgPostgreSQLSessionAdapterMetadata, @@ -98,6 +118,11 @@ PostgreSQLRepositoryRuntimeSafetyGateDescription, PostgreSQLRepositoryRuntimeSafetyGateIssue, PostgreSQLRepositoryRuntimeSafetyGateStatus, + PostgreSQLRuntimeConfigGate, + PostgreSQLRuntimeConfigGateDecision, + PostgreSQLRuntimeConfigGateDescription, + PostgreSQLRuntimeConfigGateIssue, + PostgreSQLRuntimeConfigGateStatus, PostgreSQLRuntimeExecutionGate, PostgreSQLRuntimeExecutionGateDecision, PostgreSQLRuntimeExecutionGateDescription, @@ -131,6 +156,7 @@ PostgreSQLTransactionPolicyValidationResult, PostgreSQLTransactionRuntimeBoundary, build_persistence_input_from_normalization_result, + build_parsed_factor_persistence_command, build_default_postgresql_transaction_policy, build_default_postgresql_idempotency_conflict_strategy, build_disabled_postgresql_execution_result, @@ -147,6 +173,8 @@ build_postgresql_transaction_plan, build_default_postgresql_schema_isolation_strategy, create_persistence_result, + create_source_document_repository_persist_result, + create_source_family_repository_persist_result, create_postgresql_connection_session_runtime_contract, create_postgresql_integration_test_boundary, create_postgresql_persistence_options, @@ -157,15 +185,21 @@ describe_postgresql_idempotency_conflict_strategy_boundary, describe_postgresql_repository_disabled_execution_preview, describe_postgresql_repository_runtime_safety_gate, + describe_postgresql_runtime_config_gate, describe_postgresql_runtime_execution_gate, describe_postgresql_schema_isolation_strategy, describe_postgresql_transaction_policy_boundary, evaluate_postgresql_integration_test_opt_in_config, + evaluate_postgresql_runtime_config_gate, evaluate_postgresql_runtime_execution_gate, evaluate_postgresql_repository_runtime_safety_gate, get_normalized_record_postgresql_schema, render_postgresql_ddl_preview, should_skip_postgresql_integration_tests, + source_family_repository_table_names, + persist_parsed_factor_records, + validate_source_document_repository_inputs, + validate_source_family_repository_inputs, validate_psycopg_session_adapter_boundary, validate_postgresql_connection_session_runtime_contract, validate_postgresql_persistence_options, @@ -195,6 +229,22 @@ "PersistenceRepository", "PersistenceResult", "PersistenceResultStatus", + "ParsedFactorPersistenceCommand", + "ParsedFactorPersistenceIssue", + "ParsedFactorPersistenceStatus", + "ParsedFactorPersistenceWriterResult", + "SourceDocumentRepository", + "SourceDocumentRepositoryIssue", + "SourceDocumentRepositoryPersistResult", + "SourceDocumentRepositoryPersistStatus", + "SourceDocumentRepositoryValidationResult", + "SourceFamilyDetailRecord", + "SourceFamilyMasterRecord", + "SourceFamilyRepository", + "SourceFamilyRepositoryIssue", + "SourceFamilyRepositoryPersistResult", + "SourceFamilyRepositoryPersistStatus", + "SourceFamilyRepositoryValidationResult", "PsycopgPostgreSQLSessionAdapter", "PsycopgPostgreSQLSessionAdapterBoundaryResult", "PsycopgPostgreSQLSessionAdapterMetadata", @@ -253,6 +303,11 @@ "PostgreSQLRepositoryRuntimeSafetyGateDescription", "PostgreSQLRepositoryRuntimeSafetyGateIssue", "PostgreSQLRepositoryRuntimeSafetyGateStatus", + "PostgreSQLRuntimeConfigGate", + "PostgreSQLRuntimeConfigGateDecision", + "PostgreSQLRuntimeConfigGateDescription", + "PostgreSQLRuntimeConfigGateIssue", + "PostgreSQLRuntimeConfigGateStatus", "PostgreSQLRuntimeExecutionGate", "PostgreSQLRuntimeExecutionGateDecision", "PostgreSQLRuntimeExecutionGateDescription", @@ -286,6 +341,7 @@ "PostgreSQLTransactionPolicyValidationResult", "PostgreSQLTransactionRuntimeBoundary", "build_persistence_input_from_normalization_result", + "build_parsed_factor_persistence_command", "build_default_postgresql_transaction_policy", "build_default_postgresql_idempotency_conflict_strategy", "build_disabled_postgresql_execution_result", @@ -302,6 +358,8 @@ "build_postgresql_transaction_plan", "build_default_postgresql_schema_isolation_strategy", "create_persistence_result", + "create_source_document_repository_persist_result", + "create_source_family_repository_persist_result", "create_postgresql_connection_session_runtime_contract", "create_postgresql_integration_test_boundary", "create_postgresql_persistence_options", @@ -312,15 +370,21 @@ "describe_postgresql_idempotency_conflict_strategy_boundary", "describe_postgresql_repository_disabled_execution_preview", "describe_postgresql_repository_runtime_safety_gate", + "describe_postgresql_runtime_config_gate", "describe_postgresql_runtime_execution_gate", "describe_postgresql_schema_isolation_strategy", "describe_postgresql_transaction_policy_boundary", + "evaluate_postgresql_runtime_config_gate", "evaluate_postgresql_runtime_execution_gate", "evaluate_postgresql_repository_runtime_safety_gate", "evaluate_postgresql_integration_test_opt_in_config", "get_normalized_record_postgresql_schema", "render_postgresql_ddl_preview", + "persist_parsed_factor_records", "should_skip_postgresql_integration_tests", + "source_family_repository_table_names", + "validate_source_document_repository_inputs", + "validate_source_family_repository_inputs", "validate_psycopg_session_adapter_boundary", "validate_postgresql_connection_session_runtime_contract", "validate_postgresql_persistence_options", @@ -365,6 +429,46 @@ "PersistenceRepository": repository.PersistenceRepository, "PersistenceResult": repository.PersistenceResult, "PersistenceResultStatus": repository.PersistenceResultStatus, + "ParsedFactorPersistenceCommand": ( + parsed_factor_persistence_writer.ParsedFactorPersistenceCommand + ), + "ParsedFactorPersistenceIssue": ( + parsed_factor_persistence_writer.ParsedFactorPersistenceIssue + ), + "ParsedFactorPersistenceStatus": ( + parsed_factor_persistence_writer.ParsedFactorPersistenceStatus + ), + "ParsedFactorPersistenceWriterResult": ( + parsed_factor_persistence_writer.ParsedFactorPersistenceWriterResult + ), + "SourceDocumentRepository": source_document_repository.SourceDocumentRepository, + "SourceDocumentRepositoryIssue": ( + source_document_repository.SourceDocumentRepositoryIssue + ), + "SourceDocumentRepositoryPersistResult": ( + source_document_repository.SourceDocumentRepositoryPersistResult + ), + "SourceDocumentRepositoryPersistStatus": ( + source_document_repository.SourceDocumentRepositoryPersistStatus + ), + "SourceDocumentRepositoryValidationResult": ( + source_document_repository.SourceDocumentRepositoryValidationResult + ), + "SourceFamilyDetailRecord": source_family_repository.SourceFamilyDetailRecord, + "SourceFamilyMasterRecord": source_family_repository.SourceFamilyMasterRecord, + "SourceFamilyRepository": source_family_repository.SourceFamilyRepository, + "SourceFamilyRepositoryIssue": ( + source_family_repository.SourceFamilyRepositoryIssue + ), + "SourceFamilyRepositoryPersistResult": ( + source_family_repository.SourceFamilyRepositoryPersistResult + ), + "SourceFamilyRepositoryPersistStatus": ( + source_family_repository.SourceFamilyRepositoryPersistStatus + ), + "SourceFamilyRepositoryValidationResult": ( + source_family_repository.SourceFamilyRepositoryValidationResult + ), "PsycopgPostgreSQLSessionAdapter": ( postgresql_psycopg_session_adapter.PsycopgPostgreSQLSessionAdapter ), @@ -558,6 +662,21 @@ "PostgreSQLRepositoryRuntimeSafetyGateStatus": ( postgresql_repository.PostgreSQLRepositoryRuntimeSafetyGateStatus ), + "PostgreSQLRuntimeConfigGate": ( + postgresql_runtime_config_gate.PostgreSQLRuntimeConfigGate + ), + "PostgreSQLRuntimeConfigGateDecision": ( + postgresql_runtime_config_gate.PostgreSQLRuntimeConfigGateDecision + ), + "PostgreSQLRuntimeConfigGateDescription": ( + postgresql_runtime_config_gate.PostgreSQLRuntimeConfigGateDescription + ), + "PostgreSQLRuntimeConfigGateIssue": ( + postgresql_runtime_config_gate.PostgreSQLRuntimeConfigGateIssue + ), + "PostgreSQLRuntimeConfigGateStatus": ( + postgresql_runtime_config_gate.PostgreSQLRuntimeConfigGateStatus + ), "PostgreSQLRuntimeExecutionGate": ( postgresql_runtime_execution_gate.PostgreSQLRuntimeExecutionGate ), @@ -662,6 +781,9 @@ "build_persistence_input_from_normalization_result": ( input.build_persistence_input_from_normalization_result ), + "build_parsed_factor_persistence_command": ( + parsed_factor_persistence_writer.build_parsed_factor_persistence_command + ), "build_default_postgresql_transaction_policy": ( postgresql_transaction_policy.build_default_postgresql_transaction_policy ), @@ -718,6 +840,12 @@ .build_default_postgresql_schema_isolation_strategy ), "create_persistence_result": repository.create_persistence_result, + "create_source_document_repository_persist_result": ( + source_document_repository.create_source_document_repository_persist_result + ), + "create_source_family_repository_persist_result": ( + source_family_repository.create_source_family_repository_persist_result + ), "create_postgresql_connection_session_runtime_contract": ( postgresql_connection_session_contract .create_postgresql_connection_session_runtime_contract @@ -755,6 +883,10 @@ "describe_postgresql_repository_runtime_safety_gate": ( postgresql_repository.describe_postgresql_repository_runtime_safety_gate ), + "describe_postgresql_runtime_config_gate": ( + postgresql_runtime_config_gate + .describe_postgresql_runtime_config_gate + ), "describe_postgresql_runtime_execution_gate": ( postgresql_runtime_execution_gate .describe_postgresql_runtime_execution_gate @@ -770,6 +902,10 @@ "evaluate_postgresql_integration_test_opt_in_config": ( integration_test_boundary.evaluate_postgresql_integration_test_opt_in_config ), + "evaluate_postgresql_runtime_config_gate": ( + postgresql_runtime_config_gate + .evaluate_postgresql_runtime_config_gate + ), "evaluate_postgresql_runtime_execution_gate": ( postgresql_runtime_execution_gate .evaluate_postgresql_runtime_execution_gate @@ -784,6 +920,18 @@ "should_skip_postgresql_integration_tests": ( integration_test_boundary.should_skip_postgresql_integration_tests ), + "source_family_repository_table_names": ( + source_family_repository.source_family_repository_table_names + ), + "persist_parsed_factor_records": ( + parsed_factor_persistence_writer.persist_parsed_factor_records + ), + "validate_source_document_repository_inputs": ( + source_document_repository.validate_source_document_repository_inputs + ), + "validate_source_family_repository_inputs": ( + source_family_repository.validate_source_family_repository_inputs + ), "validate_psycopg_session_adapter_boundary": ( postgresql_psycopg_session_adapter .validate_psycopg_session_adapter_boundary @@ -843,6 +991,34 @@ def test_expected_persistence_public_symbols_import_from_package() -> None: "PersistenceRepository": PersistenceRepository, "PersistenceResult": PersistenceResult, "PersistenceResultStatus": PersistenceResultStatus, + "ParsedFactorPersistenceCommand": ParsedFactorPersistenceCommand, + "ParsedFactorPersistenceIssue": ParsedFactorPersistenceIssue, + "ParsedFactorPersistenceStatus": ParsedFactorPersistenceStatus, + "ParsedFactorPersistenceWriterResult": ParsedFactorPersistenceWriterResult, + "SourceDocumentRepository": SourceDocumentRepository, + "SourceDocumentRepositoryIssue": SourceDocumentRepositoryIssue, + "SourceDocumentRepositoryPersistResult": ( + SourceDocumentRepositoryPersistResult + ), + "SourceDocumentRepositoryPersistStatus": ( + SourceDocumentRepositoryPersistStatus + ), + "SourceDocumentRepositoryValidationResult": ( + SourceDocumentRepositoryValidationResult + ), + "SourceFamilyDetailRecord": SourceFamilyDetailRecord, + "SourceFamilyMasterRecord": SourceFamilyMasterRecord, + "SourceFamilyRepository": SourceFamilyRepository, + "SourceFamilyRepositoryIssue": SourceFamilyRepositoryIssue, + "SourceFamilyRepositoryPersistResult": ( + SourceFamilyRepositoryPersistResult + ), + "SourceFamilyRepositoryPersistStatus": ( + SourceFamilyRepositoryPersistStatus + ), + "SourceFamilyRepositoryValidationResult": ( + SourceFamilyRepositoryValidationResult + ), "PsycopgPostgreSQLSessionAdapter": PsycopgPostgreSQLSessionAdapter, "PsycopgPostgreSQLSessionAdapterBoundaryResult": ( PsycopgPostgreSQLSessionAdapterBoundaryResult @@ -957,6 +1133,11 @@ def test_expected_persistence_public_symbols_import_from_package() -> None: "PostgreSQLRepositoryRuntimeSafetyGateStatus": ( PostgreSQLRepositoryRuntimeSafetyGateStatus ), + "PostgreSQLRuntimeConfigGate": PostgreSQLRuntimeConfigGate, + "PostgreSQLRuntimeConfigGateDecision": PostgreSQLRuntimeConfigGateDecision, + "PostgreSQLRuntimeConfigGateDescription": PostgreSQLRuntimeConfigGateDescription, + "PostgreSQLRuntimeConfigGateIssue": PostgreSQLRuntimeConfigGateIssue, + "PostgreSQLRuntimeConfigGateStatus": PostgreSQLRuntimeConfigGateStatus, "PostgreSQLRuntimeExecutionGate": PostgreSQLRuntimeExecutionGate, "PostgreSQLRuntimeExecutionGateDecision": ( PostgreSQLRuntimeExecutionGateDecision @@ -1026,6 +1207,9 @@ def test_expected_persistence_public_symbols_import_from_package() -> None: "build_persistence_input_from_normalization_result": ( build_persistence_input_from_normalization_result ), + "build_parsed_factor_persistence_command": ( + build_parsed_factor_persistence_command + ), "build_default_postgresql_transaction_policy": ( build_default_postgresql_transaction_policy ), @@ -1066,6 +1250,12 @@ def test_expected_persistence_public_symbols_import_from_package() -> None: build_default_postgresql_schema_isolation_strategy ), "create_persistence_result": create_persistence_result, + "create_source_document_repository_persist_result": ( + create_source_document_repository_persist_result + ), + "create_source_family_repository_persist_result": ( + create_source_family_repository_persist_result + ), "create_postgresql_connection_session_runtime_contract": ( create_postgresql_connection_session_runtime_contract ), @@ -1096,6 +1286,9 @@ def test_expected_persistence_public_symbols_import_from_package() -> None: "describe_postgresql_repository_runtime_safety_gate": ( describe_postgresql_repository_runtime_safety_gate ), + "describe_postgresql_runtime_config_gate": ( + describe_postgresql_runtime_config_gate + ), "describe_postgresql_runtime_execution_gate": ( describe_postgresql_runtime_execution_gate ), @@ -1105,6 +1298,9 @@ def test_expected_persistence_public_symbols_import_from_package() -> None: "describe_postgresql_transaction_policy_boundary": ( describe_postgresql_transaction_policy_boundary ), + "evaluate_postgresql_runtime_config_gate": ( + evaluate_postgresql_runtime_config_gate + ), "evaluate_postgresql_runtime_execution_gate": ( evaluate_postgresql_runtime_execution_gate ), @@ -1118,9 +1314,17 @@ def test_expected_persistence_public_symbols_import_from_package() -> None: get_normalized_record_postgresql_schema ), "render_postgresql_ddl_preview": render_postgresql_ddl_preview, + "persist_parsed_factor_records": persist_parsed_factor_records, "should_skip_postgresql_integration_tests": ( should_skip_postgresql_integration_tests ), + "source_family_repository_table_names": source_family_repository_table_names, + "validate_source_document_repository_inputs": ( + validate_source_document_repository_inputs + ), + "validate_source_family_repository_inputs": ( + validate_source_family_repository_inputs + ), "validate_psycopg_session_adapter_boundary": ( validate_psycopg_session_adapter_boundary ), diff --git a/tests/test_phase1_ingestion_orchestrator.py b/tests/test_phase1_ingestion_orchestrator.py new file mode 100644 index 00000000..77b19f5b --- /dev/null +++ b/tests/test_phase1_ingestion_orchestrator.py @@ -0,0 +1,531 @@ +from __future__ import annotations + +import json +import logging +from decimal import Decimal + +from carbonfactor_parser.parsers.input_artifact_contract import ( + create_phase1_parser_input_artifact, +) +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputRowStatus, + create_parser_normalized_output_row, +) +from carbonfactor_parser.parsers.parser_run_contract import ( + ParserRunResult, + ParserRunStatus, + create_parser_run_request, + create_parser_run_result, +) +from carbonfactor_parser.parsers.run_repository_contract import ( + create_parser_run_repository_persist_result, +) +from carbonfactor_parser.persistence.source_document_repository import ( + SourceDocumentRepositoryIssue, + create_source_document_repository_persist_result, +) +from carbonfactor_parser.persistence.source_family_repository import ( + SourceFamilyDetailRecord, + SourceFamilyMasterRecord, + create_source_family_repository_persist_result, +) +from carbonfactor_parser.persistence.postgresql_runtime_config_gate import ( + PostgreSQLRuntimeConfigGate, + evaluate_postgresql_runtime_config_gate, +) +from carbonfactor_parser.persistence.postgresql_schema_bootstrap import ( + build_postgresql_phase1_schema_bootstrap_report, +) +from carbonfactor_parser.source_acquisition.discovery_candidate_contract import ( + SourceDiscoveryCandidate, + SourceDiscoveryCandidateResult, +) +from carbonfactor_parser.source_acquisition.download_artifact_contract import ( + create_source_download_artifact_from_candidate, +) +from carbonfactor_parser.source_acquisition.phase1_ingestion_orchestrator import ( + PHASE1_SOURCE_FAMILIES, + Phase1IngestionExecutionMode, + Phase1IngestionFamilyStatus, + Phase1IngestionOrchestratorDependencies, + Phase1IngestionOrchestratorRequest, + Phase1IngestionRunStatus, + run_phase1_ingestion_orchestrator, +) +from carbonfactor_parser.source_acquisition.phase1_observability import ( + PHASE1_OPERATIONAL_LOGGER_NAME, +) +from carbonfactor_parser.source_acquisition.run_contract import ( + SourceAcquisitionRunSummary, + SourceAcquisitionRunResult, + SourceAcquisitionRunStatus, +) +from carbonfactor_parser.source_acquisition.run_repository_contract import ( + create_source_acquisition_run_repository_persist_result, +) + + +def test_orchestrator_runs_selected_phase1_families_end_to_end_sequentially() -> None: + dependencies = _dependencies( + { + source_family: _FakeSourceRuntime(source_family) + for source_family in PHASE1_SOURCE_FAMILIES + } + ) + + result = run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=("ghg", "defra_desnz", "ipcc_efdb"), + run_id="phase1-run-001", + ), + dependencies, + ) + + assert result.status is Phase1IngestionRunStatus.COMPLETED + assert result.selected_source_families == PHASE1_SOURCE_FAMILIES + assert [family.source_family for family in result.family_results] == list( + PHASE1_SOURCE_FAMILIES + ) + assert all( + family.status is Phase1IngestionFamilyStatus.COMPLETED + for family in result.family_results + ) + assert result.summary.completed_family_count == 3 + assert result.summary.failed_family_count == 0 + assert result.summary.source_candidate_count == 3 + assert result.summary.source_artifact_count == 3 + assert result.summary.parser_run_count == 3 + assert result.summary.parsed_factor_row_count == 3 + assert result.summary.persisted_source_run_count == 3 + assert result.summary.persisted_source_document_count == 3 + assert result.summary.persisted_parser_run_count == 3 + assert result.summary.persisted_master_count == 3 + assert result.summary.persisted_detail_count == 3 + assert result.failures == () + + +def test_orchestrator_emits_correlation_friendly_operational_logs(caplog) -> None: + dependencies = _dependencies({"ghg_protocol": _FakeSourceRuntime("ghg_protocol")}) + + with caplog.at_level(logging.INFO, logger=PHASE1_OPERATIONAL_LOGGER_NAME): + run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=("ghg_protocol",), + run_id="phase1-run-009", + correlation_id="correlation-009", + ), + dependencies, + ) + + events = [json.loads(record.message) for record in caplog.records] + + assert tuple(event["event"] for event in events) == ( + "phase1_ingestion_orchestrator_started", + "phase1_source_family_started", + "phase1_source_family_completed", + "phase1_ingestion_orchestrator_completed", + ) + family_completed = events[2] + assert family_completed["run_id"] == "phase1-run-009" + assert family_completed["correlation_id"] == "correlation-009" + assert family_completed["source_family"] == "ghg_protocol" + assert family_completed["source_key"] == "ghg_protocol" + assert family_completed["parser"]["accepted_row_count"] == 1 + assert family_completed["parser"]["validation_issue_count"] == 0 + assert family_completed["parser"]["failure_count"] == 0 + assert family_completed["persistence"] == { + "parsed_factor_detail_count": 1, + "parsed_factor_master_count": 1, + "parser_run_count": 1, + "source_document_count": 1, + "source_run_count": 1, + } + assert family_completed["documents"] == [ + { + "checksum_sha256": None, + "document_id": "ghg_protocol-artifact", + "source_family": "ghg_protocol", + "source_key": "ghg_protocol", + }, + ] + + +def test_orchestrator_failure_log_uses_structured_reason_codes(caplog) -> None: + dependencies = _dependencies( + { + "defra_desnz": _FakeSourceRuntime( + "defra_desnz", + parser_status=ParserRunStatus.FAILED, + ), + } + ) + + with caplog.at_level(logging.INFO, logger=PHASE1_OPERATIONAL_LOGGER_NAME): + run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=("defra_desnz",), + run_id="phase1-run-010", + ), + dependencies, + ) + + family_completed = [ + json.loads(record.message) + for record in caplog.records + if json.loads(record.message)["event"] == "phase1_source_family_completed" + ][0] + + assert family_completed["status"] == "failed_parser" + assert family_completed["failures"] == [ + { + "code": "PHASE1_INGESTION_PARSER_FAILED", + "field_name": "parser_run_result.status", + "message": "Parser run returned failed status.", + "severity": "error", + "source_family": "defra_desnz", + "source_key": "defra_desnz", + "stage": "parser", + }, + ] + + +def test_orchestrator_only_runs_explicitly_selected_source_family() -> None: + runtimes = { + source_family: _FakeSourceRuntime(source_family) + for source_family in PHASE1_SOURCE_FAMILIES + } + result = run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=("defra",), + run_id="phase1-run-002", + ), + _dependencies(runtimes), + ) + + assert result.status is Phase1IngestionRunStatus.COMPLETED + assert result.selected_source_families == ("defra_desnz",) + assert result.family_results[0].acquisition_result is not None + assert result.family_results[0].acquisition_result.run_id == "phase1-run-002" + assert runtimes["defra_desnz"].calls == ("discover", "download", "parse") + assert runtimes["ghg_protocol"].calls == () + assert runtimes["ipcc_efdb"].calls == () + + +def test_duplicate_source_family_selection_is_idempotent() -> None: + runtime = _FakeSourceRuntime("ipcc_efdb") + + result = run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=("ipcc", "ipcc_efdb"), + run_id="phase1-run-008", + ), + _dependencies({"ipcc_efdb": runtime}), + ) + + assert result.status is Phase1IngestionRunStatus.COMPLETED + assert result.selected_source_families == ("ipcc_efdb",) + assert len(result.family_results) == 1 + assert runtime.calls == ("discover", "download", "parse") + + +def test_partial_failure_is_deterministic_per_source_family() -> None: + result = run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=("ghg_protocol", "defra_desnz", "ipcc_efdb"), + run_id="phase1-run-003", + ), + _dependencies( + { + "ghg_protocol": _FakeSourceRuntime("ghg_protocol"), + "defra_desnz": _FakeSourceRuntime( + "defra_desnz", + parser_status=ParserRunStatus.FAILED, + ), + "ipcc_efdb": _FakeSourceRuntime("ipcc_efdb"), + } + ), + ) + + assert result.status is Phase1IngestionRunStatus.COMPLETED_WITH_FAILURES + assert tuple(family.status for family in result.family_results) == ( + Phase1IngestionFamilyStatus.COMPLETED, + Phase1IngestionFamilyStatus.FAILED_PARSER, + Phase1IngestionFamilyStatus.COMPLETED, + ) + assert result.summary.completed_family_count == 2 + assert result.summary.failed_family_count == 1 + assert result.summary.failure_count == 1 + assert result.failures[0].source_family == "defra_desnz" + assert result.failures[0].stage == "parser" + assert result.failures[0].code == "PHASE1_INGESTION_PARSER_FAILED" + + +def test_bounded_parallel_is_declared_but_not_enabled_by_default() -> None: + result = run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=("ghg_protocol",), + run_id="phase1-run-004", + execution_mode=Phase1IngestionExecutionMode.BOUNDED_PARALLEL, + max_parallelism=2, + ), + _dependencies({"ghg_protocol": _FakeSourceRuntime("ghg_protocol")}), + ) + + assert result.status is Phase1IngestionRunStatus.NOT_EXECUTABLE + assert result.family_results == () + assert result.failures[0].code == "PHASE1_INGESTION_BOUNDED_PARALLEL_NOT_ENABLED" + + +def test_postgresql_runtime_readiness_blocks_before_source_execution() -> None: + runtime = _FakeSourceRuntime("ghg_protocol") + result = run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=("ghg_protocol",), + run_id="phase1-run-006", + runtime_config_decision=evaluate_postgresql_runtime_config_gate( + PostgreSQLRuntimeConfigGate(requested=True), + ), + ), + _dependencies({"ghg_protocol": runtime}), + ) + + assert result.status is Phase1IngestionRunStatus.NOT_EXECUTABLE + assert result.family_results == () + assert runtime.calls == () + assert result.failures[0].stage == "postgresql_runtime_config" + assert result.failures[0].code == "POSTGRESQL_RUNTIME_CONFIG_BLOCKED" + + +def test_postgresql_schema_bootstrap_missing_tables_blocks_execution() -> None: + runtime = _FakeSourceRuntime("ghg_protocol") + result = run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=("ghg_protocol",), + run_id="phase1-run-007", + schema_bootstrap_report=build_postgresql_phase1_schema_bootstrap_report(), + ), + _dependencies({"ghg_protocol": runtime}), + ) + + assert result.status is Phase1IngestionRunStatus.NOT_EXECUTABLE + assert result.family_results == () + assert runtime.calls == () + assert result.failures[0].stage == "postgresql_schema_bootstrap" + assert result.failures[0].code == "PHASE1_INGESTION_POSTGRESQL_SCHEMA_NOT_READY" + + +def test_repository_failure_stops_family_at_deterministic_stage() -> None: + result = run_phase1_ingestion_orchestrator( + Phase1IngestionOrchestratorRequest( + source_families=("ipcc_efdb",), + run_id="phase1-run-005", + ), + _dependencies( + {"ipcc_efdb": _FakeSourceRuntime("ipcc_efdb")}, + fail_source_documents=True, + ), + ) + + family = result.family_results[0] + assert result.status is Phase1IngestionRunStatus.FAILED + assert family.status is ( + Phase1IngestionFamilyStatus.FAILED_SOURCE_DOCUMENT_PERSISTENCE + ) + assert family.parser_run_result is None + assert result.failures[0].stage == "source_document_persistence" + assert result.failures[0].code == "SOURCE_DOCUMENT_REPOSITORY_TEST_FAILURE" + + +class _FakeSourceRuntime: + def __init__( + self, + source_family: str, + *, + parser_status: ParserRunStatus = ParserRunStatus.COMPLETED, + ) -> None: + self.source_family = source_family + self.parser_status = parser_status + self.calls: tuple[str, ...] = () + + def discover( + self, + source_family: str, + request: Phase1IngestionOrchestratorRequest, + ) -> SourceDiscoveryCandidateResult: + self.calls = (*self.calls, "discover") + return SourceDiscoveryCandidateResult( + candidates=( + SourceDiscoveryCandidate( + source_family=source_family, + source_key=source_family, + candidate_id=f"{source_family}-candidate", + title=f"{source_family} fixture", + reference_uri=f"fixture://{source_family}/source.csv", + artifact_kind="csv", + reporting_year=2024, + content_type="text/csv", + extension=".csv", + version_label="fixture", + ), + ) + ) + + def download( + self, + source_family: str, + discovery_result: SourceDiscoveryCandidateResult, + request: Phase1IngestionOrchestratorRequest, + ) -> SourceAcquisitionRunResult: + self.calls = (*self.calls, "download") + artifact = create_source_download_artifact_from_candidate( + discovery_result.candidates[0], + artifact_id=f"{source_family}-artifact", + local_reference=f"fixture://{source_family}/downloaded.csv", + content_type="text/csv", + extension=".csv", + ) + return SourceAcquisitionRunResult( + source_family=source_family, + source_key=source_family, + status=SourceAcquisitionRunStatus.COMPLETED, + candidates=discovery_result.candidates, + artifacts=(artifact,), + issues=(), + summary=SourceAcquisitionRunSummary( + candidate_count=1, + artifact_count=1, + issue_count=0, + info_count=0, + warning_count=0, + error_count=0, + ), + run_id=request.run_id, + version_label="fixture", + ) + + def parse( + self, + source_family: str, + acquisition_result: SourceAcquisitionRunResult, + request: Phase1IngestionOrchestratorRequest, + ) -> ParserRunResult: + self.calls = (*self.calls, "parse") + artifact = create_phase1_parser_input_artifact( + source_family=source_family, + artifact_reference=acquisition_result.artifacts[0].local_reference, + reporting_year=2024, + ) + parser_request = create_parser_run_request( + source_family=source_family, + artifacts=(artifact,), + run_id=f"{request.run_id}-{source_family}-parser", + correlation_id=request.correlation_id, + ) + rows = () + if self.parser_status is not ParserRunStatus.FAILED: + rows = ( + create_parser_normalized_output_row( + artifact=artifact, + row_id=f"{source_family}-row-001", + status=ParserNormalizedOutputRowStatus.VALIDATED, + normalized_fields={ + "source_family": source_family, + "source_id": source_family, + "source_document_id": f"{source_family}-artifact", + "source_year": 2024, + "source_version": "fixture", + "factor_id": f"{source_family}-factor", + "factor_name": f"{source_family} factor", + "factor_value": Decimal("1.0"), + "factor_unit": "kg CO2e", + "unit": "kg CO2e", + }, + ), + ) + return create_parser_run_result( + request=parser_request, + status=self.parser_status, + rows=rows, + ) + + +class _FakeSourceRunRepository: + @property + def provider_name(self) -> str: + return "fake_source_runs" + + def persist_runs(self, runs): + return create_source_acquisition_run_repository_persist_result( + provider_name=self.provider_name, + runs=tuple(runs), + ) + + +class _FakeSourceDocumentRepository: + def __init__(self, *, fail: bool = False) -> None: + self.fail = fail + + @property + def provider_name(self) -> str: + return "fake_source_documents" + + def persist_source_documents(self, records): + issues = () + if self.fail: + issues = ( + SourceDocumentRepositoryIssue( + code="SOURCE_DOCUMENT_REPOSITORY_TEST_FAILURE", + message="source document persistence failed", + field_name="records", + ), + ) + return create_source_document_repository_persist_result( + provider_name=self.provider_name, + records=tuple(records), + issues=issues, + ) + + +class _FakeParserRunRepository: + @property + def provider_name(self) -> str: + return "fake_parser_runs" + + def persist_runs(self, runs): + return create_parser_run_repository_persist_result( + provider_name=self.provider_name, + runs=tuple(runs), + ) + + +class _FakeSourceFamilyRepository: + @property + def provider_name(self) -> str: + return "fake_source_family" + + def persist_source_family_records( + self, + master_records: tuple[SourceFamilyMasterRecord, ...], + detail_records: tuple[SourceFamilyDetailRecord, ...], + ): + return create_source_family_repository_persist_result( + provider_name=self.provider_name, + master_records=master_records, + detail_records=detail_records, + ) + + +def _dependencies( + runtimes, + *, + fail_source_documents: bool = False, +) -> Phase1IngestionOrchestratorDependencies: + return Phase1IngestionOrchestratorDependencies( + source_runtimes=runtimes, + source_run_repository=_FakeSourceRunRepository(), + source_document_repository=_FakeSourceDocumentRepository( + fail=fail_source_documents, + ), + parser_run_repository=_FakeParserRunRepository(), + parsed_factor_repository=_FakeSourceFamilyRepository(), + ) diff --git a/tests/test_phase1_observability.py b/tests/test_phase1_observability.py new file mode 100644 index 00000000..2cf3624d --- /dev/null +++ b/tests/test_phase1_observability.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path + +from carbonfactor_parser.persistence.postgresql_options import ( + create_postgresql_persistence_options, +) +from carbonfactor_parser.source_acquisition.phase1_observability import ( + PHASE1_OPERATIONAL_LOGGER_NAME, + REDACTED, + emit_phase1_operational_event, + redact_diagnostic_value, + summarize_postgresql_options_for_diagnostics, +) + + +PARITY_FIXTURE_PATH = ( + Path(__file__).parent + / "fixtures/parity/phase1_operational_diagnostics_expectations.json" +) + + +def _parity_expectations() -> dict[str, object]: + return json.loads(PARITY_FIXTURE_PATH.read_text(encoding="utf-8")) + + +def test_postgresql_options_diagnostics_redact_sensitive_runtime_values() -> None: + options = create_postgresql_persistence_options( + host="db.internal.example", + port=5432, + database="carbonops_prod", + username="service_user", + password_set=True, + ssl_mode="require", + application_name="carbonops-phase1", + connect_timeout_seconds=10, + ) + + summary = summarize_postgresql_options_for_diagnostics(options) + + assert summary == { + "application_name": REDACTED, + "connect_timeout_seconds": 10, + "database": REDACTED, + "host": REDACTED, + "password_set": True, + "port": 5432, + "ssl_mode": "require", + "username": REDACTED, + } + + +def test_redaction_removes_secret_fields_and_connection_userinfo() -> None: + value = { + "password": "super-secret", + "connectionString": "Host=db;Username=svc;" + "Password" + "=raw-secret", + "apiKey": "api-secret", + "nested": { + "message": ( + "failed dsn=postgresql://svc:secret@db.internal/carbonops " + "connectionString" + "=postgresql://svc:raw-secret@db.internal/carbonops " + "token" + "=" + "abc123" + ), + }, + "safe_count": 3, + } + + redacted = redact_diagnostic_value("payload", value) + + assert redacted == { + "apiKey": REDACTED, + "connectionString": REDACTED, + "nested": { + "message": ( + f"failed dsn={REDACTED} " + "connectionString" + f"={REDACTED} token" + "=" + REDACTED + ), + }, + "password": REDACTED, + "safe_count": 3, + } + + +def test_operational_event_log_shape_is_stable_json( + caplog, +) -> None: + logger = logging.getLogger(PHASE1_OPERATIONAL_LOGGER_NAME) + + with caplog.at_level(logging.INFO, logger=PHASE1_OPERATIONAL_LOGGER_NAME): + event = emit_phase1_operational_event( + "phase1_test_event", + { + "z_count": 2, + "a_context": {"source_family": "ghg_protocol"}, + }, + logger=logger, + ) + + assert event == { + "a_context": {"source_family": "ghg_protocol"}, + "event": "phase1_test_event", + "z_count": 2, + } + assert json.loads(caplog.records[-1].message) == event + assert caplog.records[-1].message == ( + '{"a_context":{"source_family":"ghg_protocol"},' + '"event":"phase1_test_event","z_count":2}' + ) + + +def test_phase1_operational_diagnostics_shared_parity_shape() -> None: + expectations = _parity_expectations() + + assert expectations["request_keys"] == [ + "correlation_id", + "execution_mode", + "max_degree_of_parallelism", + "run_id", + "source_families", + ] + assert expectations["family_keys"] == [ + "correlation_id", + "documents", + "failures", + "parser", + "persistence", + "run_id", + "source_family", + "source_key", + "status", + ] + assert expectations["document_keys"] == [ + "checksum_sha256", + "document_id", + "source_family", + "source_key", + ] + assert expectations["parser_keys"] == [ + "accepted_row_count", + "failure_count", + "result_status", + "run_id", + "validation_issue_count", + ] + assert expectations["failure_keys"] == [ + "code", + "field_name", + "message", + "severity", + "source_family", + "source_key", + "stage", + ] + assert expectations["summary_keys"] == [ + "completed_family_count", + "failed_family_count", + "failure_count", + "parsed_factor_row_count", + "parser_run_count", + "persisted_detail_count", + "persisted_master_count", + "persisted_parser_run_count", + "persisted_source_document_count", + "persisted_source_run_count", + "requested_family_count", + "source_artifact_count", + "source_candidate_count", + ] + assert expectations["orchestrator_event_names"] == [ + "phase1_ingestion_orchestrator_started", + "phase1_source_family_completed", + "phase1_ingestion_orchestrator_completed", + ] + assert expectations["service_host_event_names"] == [ + "phase1_service_host_starting", + "phase1_service_host_started", + "phase1_service_host_scheduled_run_started", + "phase1_service_host_scheduled_run_completed", + "phase1_service_host_scheduled_run_skipped", + ] + assert expectations["redacted"] == REDACTED diff --git a/tests/test_phase1_service_host.py b/tests/test_phase1_service_host.py new file mode 100644 index 00000000..c0dc6cc3 --- /dev/null +++ b/tests/test_phase1_service_host.py @@ -0,0 +1,290 @@ +from __future__ import annotations + +import json +import logging + +import pytest + +from carbonfactor_parser.persistence.postgresql_options import ( + create_postgresql_persistence_options, +) +from carbonfactor_parser.persistence.postgresql_schema_bootstrap import ( + PostgreSQLSchemaBootstrapMode, + build_postgresql_phase1_schema_bootstrap_report, +) +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + get_required_table_names, +) +from carbonfactor_parser.source_acquisition.phase1_ingestion_orchestrator import ( + Phase1IngestionOrchestratorRequest, + Phase1IngestionOrchestratorResult, + Phase1IngestionRunStatus, + Phase1IngestionRunSummary, +) +from carbonfactor_parser.source_acquisition.phase1_service_host import ( + Phase1ScheduledIngestionServiceHost, + Phase1ScheduledRunStatus, + Phase1ServiceHostConfig, + Phase1ServiceHostStatus, + validate_phase1_service_host_config, +) +from carbonfactor_parser.source_acquisition.phase1_observability import ( + PHASE1_OPERATIONAL_LOGGER_NAME, + REDACTED, +) + + +def test_service_host_validates_required_postgresql_runtime_config() -> None: + config = Phase1ServiceHostConfig( + source_families=("ghg_protocol",), + postgresql_options=create_postgresql_persistence_options( + host="localhost", + port=5432, + database="carbonops", + username="carbonops", + password_set=False, + ), + ) + + issues = validate_phase1_service_host_config(config) + + assert [issue.code for issue in issues] == [ + "PHASE1_SERVICE_HOST_POSTGRESQL_PASSWORD_NOT_CONFIRMED", + ] + assert issues[0].field_name == "postgresql_options.password_set" + + +def test_service_host_startup_checks_phase1_schema_before_ready() -> None: + checker = _FakeSchemaBootstrapChecker(present=False) + host = Phase1ScheduledIngestionServiceHost( + _config(), + schema_bootstrap_checker=checker, + orchestrator_runner=_FakeOrchestratorRunner(), + ) + + result = host.start() + + assert result.status is Phase1ServiceHostStatus.BLOCKED + assert host.status is Phase1ServiceHostStatus.BLOCKED + assert checker.calls == ( + (PostgreSQLSchemaBootstrapMode.CHECK_ONLY, True), + ) + assert result.schema_bootstrap_report is not None + assert result.schema_bootstrap_report.missing_table_names + assert result.issues[0].code == "PHASE1_SERVICE_HOST_POSTGRESQL_SCHEMA_NOT_READY" + + +def test_service_host_startup_logs_redacted_runtime_config(caplog) -> None: + host = Phase1ScheduledIngestionServiceHost( + _config(), + schema_bootstrap_checker=_FakeSchemaBootstrapChecker(present=True), + orchestrator_runner=_FakeOrchestratorRunner(), + ) + + with caplog.at_level(logging.INFO, logger=PHASE1_OPERATIONAL_LOGGER_NAME): + host.start() + + starting = json.loads(caplog.records[0].message) + + assert starting["event"] == "phase1_service_host_starting" + assert starting["postgresql_options"] == { + "application_name": None, + "connect_timeout_seconds": None, + "database": REDACTED, + "host": REDACTED, + "password_set": True, + "port": 5432, + "ssl_mode": None, + "username": REDACTED, + } + assert "localhost" not in caplog.records[0].message + assert "carbonops" not in caplog.records[0].message + + +def test_scheduled_trigger_runs_orchestrator_for_selected_source_families() -> None: + runner = _FakeOrchestratorRunner() + host = Phase1ScheduledIngestionServiceHost( + _config(source_families=("defra", "ipcc_efdb"), run_id_prefix="phase1-test"), + schema_bootstrap_checker=_FakeSchemaBootstrapChecker(present=True), + orchestrator_runner=runner, + ) + + startup = host.start() + result = host.trigger_scheduled_run() + + assert startup.is_ready + assert result.status is Phase1ScheduledRunStatus.STARTED + assert result.run_id == "phase1-test-000001" + assert runner.requests[0].source_families == ("defra", "ipcc_efdb") + assert runner.requests[0].schema_bootstrap_report is not None + assert runner.requests[0].schema_bootstrap_report.missing_table_names == () + assert result.orchestrator_result is not None + assert result.orchestrator_result.status is Phase1IngestionRunStatus.COMPLETED + assert host.status is Phase1ServiceHostStatus.READY + + +def test_scheduled_trigger_skips_overlapping_run() -> None: + host_holder: dict[str, Phase1ScheduledIngestionServiceHost] = {} + nested_result_holder = {} + + def runner( + request: Phase1IngestionOrchestratorRequest, + ) -> Phase1IngestionOrchestratorResult: + nested_result_holder["result"] = host_holder["host"].trigger_scheduled_run() + return _orchestrator_result(request) + + host = Phase1ScheduledIngestionServiceHost( + _config(), + schema_bootstrap_checker=_FakeSchemaBootstrapChecker(present=True), + orchestrator_runner=runner, + ) + host_holder["host"] = host + + host.start() + result = host.trigger_scheduled_run() + + assert result.status is Phase1ScheduledRunStatus.STARTED + nested_result = nested_result_holder["result"] + assert nested_result.status is Phase1ScheduledRunStatus.SKIPPED_ALREADY_RUNNING + assert nested_result.issues[0].code == ( + "PHASE1_SERVICE_HOST_RUN_ALREADY_IN_PROGRESS" + ) + assert host.status is Phase1ServiceHostStatus.READY + + +def test_graceful_shutdown_blocks_new_runs_and_stops_after_active_run() -> None: + host_holder: dict[str, Phase1ScheduledIngestionServiceHost] = {} + nested_result_holder = {} + + def runner( + request: Phase1IngestionOrchestratorRequest, + ) -> Phase1IngestionOrchestratorResult: + shutdown_status = host_holder["host"].request_shutdown() + nested_result = host_holder["host"].trigger_scheduled_run() + nested_result_holder["shutdown_status"] = shutdown_status + nested_result_holder["result"] = nested_result + return _orchestrator_result(request) + + host = Phase1ScheduledIngestionServiceHost( + _config(), + schema_bootstrap_checker=_FakeSchemaBootstrapChecker(present=True), + orchestrator_runner=runner, + ) + host_holder["host"] = host + + host.start() + result = host.trigger_scheduled_run() + after_shutdown_result = host.trigger_scheduled_run() + + assert result.status is Phase1ScheduledRunStatus.STARTED + assert nested_result_holder["shutdown_status"] is ( + Phase1ServiceHostStatus.SHUTDOWN_REQUESTED + ) + assert nested_result_holder["result"].status is ( + Phase1ScheduledRunStatus.SKIPPED_SHUTTING_DOWN + ) + assert host.status is Phase1ServiceHostStatus.STOPPED + assert after_shutdown_result.status is Phase1ScheduledRunStatus.SKIPPED_SHUTTING_DOWN + + +def test_scheduled_runner_error_releases_overlap_guard_and_returns_ready() -> None: + failed_once = False + + def runner( + request: Phase1IngestionOrchestratorRequest, + ) -> Phase1IngestionOrchestratorResult: + nonlocal failed_once + if not failed_once: + failed_once = True + raise RuntimeError(f"boom: {request.run_id}") + return _orchestrator_result(request) + + host = Phase1ScheduledIngestionServiceHost( + _config(), + schema_bootstrap_checker=_FakeSchemaBootstrapChecker(present=True), + orchestrator_runner=runner, + ) + + host.start() + with pytest.raises(RuntimeError, match="boom: phase1-scheduled-000001"): + host.trigger_scheduled_run() + + assert host.status is Phase1ServiceHostStatus.READY + follow_up = host.trigger_scheduled_run() + assert follow_up.status is Phase1ScheduledRunStatus.STARTED + assert follow_up.run_id == "phase1-scheduled-000002" + + +class _FakeSchemaBootstrapChecker: + def __init__(self, *, present: bool) -> None: + self.present = present + self.calls: tuple[tuple[PostgreSQLSchemaBootstrapMode, bool], ...] = () + + def __call__( + self, + mode: PostgreSQLSchemaBootstrapMode, + fail_on_missing: bool, + ): + self.calls = (*self.calls, (mode, fail_on_missing)) + present_table_names = get_required_table_names() if self.present else () + return build_postgresql_phase1_schema_bootstrap_report( + mode=mode, + present_table_names=present_table_names, + fail_on_missing=fail_on_missing, + ) + + +class _FakeOrchestratorRunner: + def __init__(self) -> None: + self.requests: tuple[Phase1IngestionOrchestratorRequest, ...] = () + + def __call__( + self, + request: Phase1IngestionOrchestratorRequest, + ) -> Phase1IngestionOrchestratorResult: + self.requests = (*self.requests, request) + return _orchestrator_result(request) + + +def _config( + *, + source_families: tuple[str, ...] = ("ghg_protocol",), + run_id_prefix: str = "phase1-scheduled", +) -> Phase1ServiceHostConfig: + return Phase1ServiceHostConfig( + source_families=source_families, + run_id_prefix=run_id_prefix, + postgresql_options=create_postgresql_persistence_options( + host="localhost", + port=5432, + database="carbonops", + username="carbonops", + password_set=True, + ), + ) + + +def _orchestrator_result( + request: Phase1IngestionOrchestratorRequest, +) -> Phase1IngestionOrchestratorResult: + return Phase1IngestionOrchestratorResult( + status=Phase1IngestionRunStatus.COMPLETED, + request=request, + selected_source_families=request.source_families, + family_results=(), + summary=Phase1IngestionRunSummary( + requested_family_count=len(request.source_families), + completed_family_count=len(request.source_families), + failed_family_count=0, + source_candidate_count=0, + source_artifact_count=0, + parser_run_count=0, + parsed_factor_row_count=0, + persisted_source_run_count=0, + persisted_source_document_count=0, + persisted_parser_run_count=0, + persisted_master_count=0, + persisted_detail_count=0, + failure_count=0, + ), + ) diff --git a/tests/test_postgresql_ddl_renderer.py b/tests/test_postgresql_ddl_renderer.py index ff64198f..506512f9 100644 --- a/tests/test_postgresql_ddl_renderer.py +++ b/tests/test_postgresql_ddl_renderer.py @@ -39,9 +39,9 @@ def _rendered_table_constraint_and_index_identifiers() -> tuple[str, ...]: identifiers: list[str] = [table.table_name for table in rendered.tables] for statement in rendered.statements: - identifiers.extend(re.findall(r"\bCREATE TABLE ([a-z][a-z0-9_]*)", statement)) + identifiers.extend(re.findall(r"\bCREATE TABLE (?:IF NOT EXISTS )?([a-z][a-z0-9_]*)", statement)) identifiers.extend(re.findall(r"\bCONSTRAINT ([a-z][a-z0-9_]*)", statement)) - identifiers.extend(re.findall(r"\bCREATE (?:UNIQUE )?INDEX ([a-z][a-z0-9_]*)", statement)) + identifiers.extend(re.findall(r"\bCREATE (?:UNIQUE )?INDEX (?:IF NOT EXISTS )?([a-z][a-z0-9_]*)", statement)) return tuple(identifiers) @@ -74,6 +74,8 @@ def test_required_create_table_statements_are_rendered() -> None: "source_documents", "parser_runs", "schema_bootstrap_states", + "source_family_year_states", + "normalized_factor_records", "ghg_emission_factor_masters", "ghg_emission_factor_details", "defra_emission_factor_masters", @@ -82,7 +84,7 @@ def test_required_create_table_statements_are_rendered() -> None: "ipcc_emission_factor_details", ) for table_name in required_tables: - assert f"CREATE TABLE {table_name}" in statements + assert f"CREATE TABLE IF NOT EXISTS {table_name}" in statements def test_primary_key_foreign_key_unique_and_index_statements_are_rendered() -> None: @@ -92,7 +94,15 @@ def test_primary_key_foreign_key_unique_and_index_statements_are_rendered() -> N assert "PRIMARY KEY" in statements assert "REFERENCES ghg_emission_factor_masters (ghg_emission_factor_master_id)" in statements assert "CONSTRAINT uq_source_documents_family_uri_checksum UNIQUE" in statements - assert "CREATE INDEX idx_ingestion_runs_run_status ON ingestion_runs (run_status);" in statements + assert ( + "CONSTRAINT uq_defra_emission_factor_masters_family_year_version_key " + "UNIQUE (source_family, source_year, source_version, master_external_key)" + ) in statements + assert "CREATE INDEX IF NOT EXISTS idx_ingestion_runs_run_status ON ingestion_runs (run_status);" in statements + assert ( + "CREATE INDEX IF NOT EXISTS idx_ghg_emission_factor_masters_source_year " + "ON ghg_emission_factor_masters (source_family, source_year, source_version);" + ) in statements def test_renderer_output_is_deterministic() -> None: @@ -138,7 +148,7 @@ def test_known_long_foreign_key_and_index_names_are_shortened_deterministically( assert expected_fk_name != long_fk_name assert expected_index_name != long_index_name assert f"CONSTRAINT {expected_fk_name} FOREIGN KEY" in statements - assert f"CREATE INDEX {expected_index_name} ON defra_emission_factor_details" in statements + assert f"CREATE INDEX IF NOT EXISTS {expected_index_name} ON defra_emission_factor_details" in statements assert long_fk_name not in statements assert long_index_name not in statements @@ -172,7 +182,12 @@ def test_forbidden_name_fragments_do_not_appear() -> None: forbidden = ("temp", "test", "fake", "sample", "manual", "json_input") statements = "\n".join(render_postgresql_phase1_schema_ddl().statements) lowered = statements.lower() - assert not any(fragment in lowered for fragment in forbidden) + identifiers = re.findall(r"[a-z][a-z0-9_]*", lowered) + assert not any( + identifier == fragment or identifier.startswith(f"{fragment}_") + for identifier in identifiers + for fragment in forbidden + ) def test_identifiers_follow_lowercase_snake_case() -> None: @@ -200,8 +215,8 @@ def test_different_long_identifiers_with_same_visible_prefix_do_not_collapse() - first_rendered = _render_identifier(first_name, "index") second_rendered = _render_identifier(second_name, "index") assert first_rendered != second_rendered - assert f"CREATE INDEX {first_rendered} ON good_table (id);" in statements - assert f"CREATE INDEX {second_rendered} ON good_table (id);" in statements + assert f"CREATE INDEX IF NOT EXISTS {first_rendered} ON good_table (id);" in statements + assert f"CREATE INDEX IF NOT EXISTS {second_rendered} ON good_table (id);" in statements def test_structured_renderer_rejects_unknown_unique_and_foreign_key_columns() -> None: diff --git a/tests/test_postgresql_execution.py b/tests/test_postgresql_execution.py new file mode 100644 index 00000000..546aed32 --- /dev/null +++ b/tests/test_postgresql_execution.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from carbonfactor_parser.persistence.postgresql_execution import ( + commit, + execute, + fetchone, + rollback, +) + + +class _RecordingConnection: + def __init__(self) -> None: + self.calls: list[tuple[object, ...]] = [] + self.committed = False + self.rolled_back = False + + def execute(self, *args: object) -> object: + self.calls.append(args) + return "cursor" + + def commit(self) -> None: + self.committed = True + + def rollback(self) -> None: + self.rolled_back = True + + +class _ConnectionWithoutTransactionMethods: + pass + + +class _RecordingCursor: + def __init__(self) -> None: + self.called = False + + def fetchone(self) -> tuple[str]: + self.called = True + return ("row",) + + +def test_execute_calls_connection_execute_without_parameters_when_none() -> None: + connection = _RecordingConnection() + + result = execute(connection, "SELECT 1") + + assert result == "cursor" + assert connection.calls == [("SELECT 1",)] + + +def test_execute_calls_connection_execute_with_parameters_when_provided() -> None: + connection = _RecordingConnection() + parameters = ("value",) + + result = execute(connection, "SELECT %s", parameters) + + assert result == "cursor" + assert connection.calls == [("SELECT %s", parameters)] + + +def test_fetchone_calls_cursor_fetchone() -> None: + cursor = _RecordingCursor() + + result = fetchone(cursor) + + assert result == ("row",) + assert cursor.called is True + + +def test_commit_noops_when_method_missing() -> None: + commit(_ConnectionWithoutTransactionMethods()) + + +def test_commit_calls_connection_commit_when_present() -> None: + connection = _RecordingConnection() + + commit(connection) + + assert connection.committed is True + + +def test_rollback_noops_when_method_missing() -> None: + rollback(_ConnectionWithoutTransactionMethods()) + + +def test_rollback_calls_connection_rollback_when_present() -> None: + connection = _RecordingConnection() + + rollback(connection) + + assert connection.rolled_back is True diff --git a/tests/test_postgresql_normalized_factor_repository.py b/tests/test_postgresql_normalized_factor_repository.py new file mode 100644 index 00000000..ba1be305 --- /dev/null +++ b/tests/test_postgresql_normalized_factor_repository.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +from decimal import Decimal +import os +import uuid + +import pytest + +from carbonfactor_parser.parsers.input_artifact_contract import ( + create_phase1_parser_input_artifact, +) +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputBatch, + ParserNormalizedOutputRowStatus, + create_parser_normalized_output_batch, + create_parser_normalized_output_row, +) +from carbonfactor_parser.persistence import ( + POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR, + POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR, +) +from carbonfactor_parser.persistence.postgresql_normalized_factor_repository import ( + NORMALIZED_FACTOR_RECORDS_TABLE_NAME, + PostgreSQLNormalizedFactorInsertStatus, + PostgreSQLNormalizedFactorRuntimeRepository, + insert_postgresql_normalized_factor_records, +) +from carbonfactor_parser.persistence.postgresql_runtime_config import ( + POSTGRESQL_RUNTIME_DSN_ENV_VAR, + load_postgresql_runtime_config, +) +from carbonfactor_parser.persistence.postgresql_runtime_schema_bootstrap import ( + bootstrap_postgresql_phase1_schema, +) +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + get_required_table_names, +) + + +class _FakeCursor: + def __init__(self, row: tuple[object, ...] | None = None) -> None: + self._row = row + + def fetchone(self) -> tuple[object, ...] | None: + return self._row + + def fetchall(self) -> list[tuple[object, ...]]: + return [] + + +class _FakeConnection: + def __init__(self, *, fail_with: Exception | None = None) -> None: + self.fail_with = fail_with + self.seen_idempotency_keys: set[str] = set() + self.statements: list[tuple[str, object | None]] = [] + self.commit_count = 0 + self.rollback_count = 0 + + def execute(self, statement: str, parameters: object | None = None) -> _FakeCursor: + self.statements.append((statement, parameters)) + if self.fail_with is not None: + raise self.fail_with + key = parameters[1] # type: ignore[index] + if str(key) in self.seen_idempotency_keys: + return _FakeCursor(None) + self.seen_idempotency_keys.add(str(key)) + return _FakeCursor(("inserted",)) + + def commit(self) -> None: + self.commit_count += 1 + + def rollback(self) -> None: + self.rollback_count += 1 + + +def test_runtime_repository_inserts_normalized_factor_records() -> None: + connection = _FakeConnection() + repository = PostgreSQLNormalizedFactorRuntimeRepository(connection) + + summary = repository.insert_normalized_factor_records(_batch()) + + assert summary.status is PostgreSQLNormalizedFactorInsertStatus.INSERTED + assert summary.attempted == 1 + assert summary.inserted == 1 + assert summary.skipped_duplicate == 0 + assert summary.failed == 0 + assert summary.validation_error_count == 0 + assert connection.commit_count == 1 + statement, parameters = connection.statements[0] + assert f"INSERT INTO {NORMALIZED_FACTOR_RECORDS_TABLE_NAME}" in statement + assert "ON CONFLICT (idempotency_key_sha256) DO NOTHING" in statement + assert parameters is not None + assert parameters[2] == "defra_desnz" # type: ignore[index] + assert parameters[4] == 2024 # type: ignore[index] + assert parameters[13] == Decimal("0.20705") # type: ignore[index] + assert parameters[14] == "kWh" # type: ignore[index] + assert "local-only-run-001" in parameters # type: ignore[operator] + + +def test_runtime_repository_repeated_insert_is_idempotent() -> None: + connection = _FakeConnection() + repository = PostgreSQLNormalizedFactorRuntimeRepository(connection) + + first = repository.insert_normalized_factor_records(_batch()) + second = repository.insert_normalized_factor_records(_batch()) + + assert first.inserted == 1 + assert first.skipped_duplicate == 0 + assert second.status is PostgreSQLNormalizedFactorInsertStatus.INSERTED + assert second.attempted == 1 + assert second.inserted == 0 + assert second.skipped_duplicate == 1 + assert len(connection.statements) == 2 + + +def test_runtime_repository_reports_validation_failure_without_database_call() -> None: + connection = _FakeConnection() + repository = PostgreSQLNormalizedFactorRuntimeRepository(connection) + malformed_batch = _batch(normalized_fields={"factor_id": "DEFRA-2024-ELEC"}) + + summary = repository.insert_normalized_factor_records(malformed_batch) + + assert summary.status is PostgreSQLNormalizedFactorInsertStatus.FAILED_VALIDATION + assert summary.attempted == 1 + assert summary.inserted == 0 + assert summary.failed == 1 + assert summary.validation_error_count >= 1 + assert [issue.code for issue in summary.issues] == [ + "POSTGRESQL_NORMALIZED_FACTOR_MISSING_FACTOR_VALUE", + "POSTGRESQL_NORMALIZED_FACTOR_MISSING_FACTOR_UNIT", + ] + assert connection.statements == [] + + +def test_runtime_repository_redacts_database_errors() -> None: + private_dsn = "postgresql://carbonops:secret@example.invalid:5432/carbonops" + connection = _FakeConnection( + fail_with=RuntimeError( + f"could not connect dsn={private_dsn} password=secret" + ), + ) + repository = PostgreSQLNormalizedFactorRuntimeRepository(connection) + + summary = repository.insert_normalized_factor_records(_batch()) + + assert summary.status is PostgreSQLNormalizedFactorInsertStatus.FAILED_DATABASE + assert summary.failed == 1 + assert summary.inserted == 0 + assert connection.rollback_count == 1 + message = summary.issues[0].message + assert "secret" not in message + assert private_dsn not in message + assert "password=***" in message + + +def test_runtime_insert_with_missing_config_fails_closed() -> None: + config_result = load_postgresql_runtime_config({}) + called = False + + def connection_factory(_config): + nonlocal called + called = True + return _FakeConnection() + + summary = insert_postgresql_normalized_factor_records( + _batch(), + config_result=config_result, + connection_factory=connection_factory, + ) + + assert summary.status is PostgreSQLNormalizedFactorInsertStatus.FAILED_VALIDATION + assert summary.attempted == 1 + assert summary.failed == 1 + assert summary.validation_error_count == 4 + assert called is False + + +def test_runtime_insert_with_ready_config_uses_explicit_connection_factory() -> None: + config_result = load_postgresql_runtime_config( + {POSTGRESQL_RUNTIME_DSN_ENV_VAR: "postgresql://example.invalid/carbonops"}, + ) + connection = _FakeConnection() + + summary = insert_postgresql_normalized_factor_records( + _batch(), + config_result=config_result, + connection_factory=lambda _config: connection, + ) + + assert summary.status is PostgreSQLNormalizedFactorInsertStatus.INSERTED + assert summary.inserted == 1 + assert len(connection.statements) == 1 + + +def test_phase1_bootstrap_includes_normalized_factor_runtime_table() -> None: + assert NORMALIZED_FACTOR_RECORDS_TABLE_NAME in get_required_table_names() + + +@pytest.mark.postgresql_integration +def test_docker_postgresql_normalized_factor_insert_integration() -> None: + if os.getenv(POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR) != "1": + pytest.skip("PostgreSQL integration test opt-in is not enabled.") + dsn = os.getenv(POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR) + if not dsn: + pytest.skip("PostgreSQL integration test DSN was not provided.") + + import psycopg + + schema_name = f"carbonops_ph012_{uuid.uuid4().hex}" + with psycopg.connect(dsn) as connection: + connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") + connection.execute(f"SET search_path TO {schema_name}") + bootstrap = bootstrap_postgresql_phase1_schema(connection) + repository = PostgreSQLNormalizedFactorRuntimeRepository(connection) + + first = repository.insert_normalized_factor_records(_batch()) + second = repository.insert_normalized_factor_records(_batch()) + cursor = connection.execute( + "SELECT COUNT(*) FROM normalized_factor_records", + ) + + assert bootstrap.missing_table_names == () + assert first.inserted == 1 + assert second.inserted == 0 + assert second.skipped_duplicate == 1 + assert cursor.fetchone()[0] == 1 + + +def _batch( + *, + normalized_fields: dict[str, object] | None = None, +) -> ParserNormalizedOutputBatch: + artifact = create_phase1_parser_input_artifact( + source_family="defra_desnz", + artifact_reference="artifact://defra/conversion-factors-2024.csv", + checksum_sha256="a" * 64, + reporting_year=2024, + ) + fields = { + "source_family": "defra_desnz", + "source_id": "defra_desnz", + "source_year": 2024, + "source_version": "conversion-factors-2024", + "source_checksum_sha256": "a" * 64, + "source_document_id": "defra-document-2024", + "factor_id": "DEFRA-2024-ELEC", + "factor_name": "Electricity generated", + "factor_value": Decimal("0.20705"), + "factor_unit": "kWh", + "unit": "kWh", + "run_id": "local-only-run-001", + } + if normalized_fields is not None: + fields = normalized_fields + return create_parser_normalized_output_batch( + ( + create_parser_normalized_output_row( + artifact=artifact, + row_id="defra-row-001", + source_row_number=2, + status=ParserNormalizedOutputRowStatus.VALIDATED, + normalized_fields=fields, + ), + ), + ) diff --git a/tests/test_postgresql_persisted_parity_validation.py b/tests/test_postgresql_persisted_parity_validation.py new file mode 100644 index 00000000..d34afe49 --- /dev/null +++ b/tests/test_postgresql_persisted_parity_validation.py @@ -0,0 +1,323 @@ +from __future__ import annotations + +from dataclasses import replace +import hashlib +import os +from pathlib import Path +import subprocess +import uuid + +import pytest + +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputBatch, +) +from carbonfactor_parser.persistence import ( + POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR, + POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR, +) +from carbonfactor_parser.persistence.postgresql_runtime_schema_bootstrap import ( + bootstrap_postgresql_phase1_schema, +) +from carbonfactor_parser.persistence.postgresql_source_family_repository import ( + PostgreSQLSourceFamilyRuntimeRepository, +) +from carbonfactor_parser.persistence.postgresql_year_state_repository import ( + PostgreSQLSourceFamilyYearStateRepository, +) +from carbonfactor_parser.pipeline.defra_desnz_production_e2e import ( + DEFRA_DESNZ_SOURCE_FAMILY, + DefraDesnzProductionParserBoundary, +) +from carbonfactor_parser.pipeline.ghg_protocol_production_e2e import ( + GHG_PROTOCOL_SOURCE_FAMILY, + GHGProtocolProductionParserBoundary, +) +from carbonfactor_parser.pipeline.ipcc_efdb_production_e2e import ( + IPCC_EFDB_SOURCE_FAMILY, + IpccEfdbProductionParserBoundary, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EDownloadedArtifact, +) + + +PERSISTED_PARITY_OPT_IN_ENV_VAR = "CARBONOPS_RUN_PERSISTED_PARITY_VALIDATION" +DOTNET_POSTGRESQL_SCHEMA_ENV_VAR = "CARBONOPS_DOTNET_POSTGRESQL_TEST_SCHEMA" +DOTNET_POSTGRESQL_DSN_ENV_VAR = "CARBONOPS_DOTNET_POSTGRESQL_TEST_DSN" +DOTNET_POSTGRESQL_OPT_IN_ENV_VAR = "CARBONOPS_RUN_DOTNET_POSTGRESQL_INTEGRATION" +PARITY_SOURCE_VERSION = "prod009-prod010-parity" +PARITY_RUN_ID = "prod009-prod010-parity" +SOURCE_FAMILIES = ( + GHG_PROTOCOL_SOURCE_FAMILY, + DEFRA_DESNZ_SOURCE_FAMILY, + IPCC_EFDB_SOURCE_FAMILY, +) + + +def test_persisted_parity_validation_is_opt_in_by_default() -> None: + assert not _parity_enabled({}) + + +@pytest.mark.postgresql_integration +def test_postgresql_persisted_output_matches_dotnet_baseline_when_enabled( + tmp_path: Path, +) -> None: + if not _parity_enabled(os.environ): + pytest.skip("Persisted parity validation opt-in is not enabled.") + dsn = os.getenv(POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR) + if not dsn: + pytest.skip("PostgreSQL integration test DSN was not provided.") + + import psycopg + + repository_root = Path(__file__).resolve().parents[1] + python_schema = f"carbonops_prod010_py_{uuid.uuid4().hex}" + dotnet_schema = f"carbonops_prod010_dotnet_{uuid.uuid4().hex}" + + with psycopg.connect(dsn) as connection: + _create_schema(connection, python_schema) + _create_schema(connection, dotnet_schema) + _populate_python_schema(connection, python_schema) + + _populate_dotnet_schema(repository_root, dsn, dotnet_schema) + + with psycopg.connect(dsn) as connection: + python_snapshot = _snapshot_schema(connection, python_schema) + dotnet_snapshot = _snapshot_schema(connection, dotnet_schema) + + assert python_snapshot == dotnet_snapshot + + +def _parity_enabled(environment: dict[str, str] | os._Environ[str]) -> bool: + return environment.get(PERSISTED_PARITY_OPT_IN_ENV_VAR) == "1" + + +def _create_schema(connection: object, schema_name: str) -> None: + connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") + connection.commit() + + +def _populate_python_schema(connection: object, schema_name: str) -> None: + connection.execute(f"SET search_path TO {schema_name}") + bootstrap_postgresql_phase1_schema(connection) + repository = PostgreSQLSourceFamilyRuntimeRepository(connection) + year_state = PostgreSQLSourceFamilyYearStateRepository(connection) + + for source_family in SOURCE_FAMILIES: + batch = _rewritten_batch(_parse_fixture(source_family)) + first = repository.insert_normalized_factor_records(batch) + second = repository.insert_normalized_factor_records(batch) + assert first.master_inserted > 0 + assert first.detail_inserted > 0 + assert second.master_inserted == 0 + assert second.detail_inserted == 0 + assert second.master_skipped > 0 + assert second.detail_skipped > 0 + year_state.record_ingested_year(source_family, 2024) + year_state.record_ingested_year(source_family, 2024) + + +def _populate_dotnet_schema(repository_root: Path, dsn: str, schema_name: str) -> None: + environment = dict(os.environ) + environment[POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR] = "1" + environment[DOTNET_POSTGRESQL_OPT_IN_ENV_VAR] = "1" + environment[DOTNET_POSTGRESQL_DSN_ENV_VAR] = dsn + environment[DOTNET_POSTGRESQL_SCHEMA_ENV_VAR] = schema_name + result = subprocess.run( + [ + "dotnet", + "test", + "tests/dotnet/CarbonOps.Parser.Contracts.Tests/" + "CarbonOps.Parser.Contracts.Tests.csproj", + "--configuration", + "Release", + "--no-restore", + "--filter", + "FullyQualifiedName~PersistedParityFixtureBaselineWhenEnabled", + ], + cwd=repository_root, + env=environment, + check=False, + capture_output=True, + text=True, + timeout=180, + ) + assert result.returncode == 0, _redacted_dotnet_output(result) + + +def _parse_fixture(source_family: str) -> ParserNormalizedOutputBatch: + fixture_path = _fixture_path(source_family) + artifact = ProductionE2EDownloadedArtifact( + source_family=source_family, + source_year=2024, + artifact_reference=str(fixture_path), + checksum_sha256=hashlib.sha256(fixture_path.read_bytes()).hexdigest(), + content_type="text/csv", + format_hint="csv", + ) + parser = { + GHG_PROTOCOL_SOURCE_FAMILY: GHGProtocolProductionParserBoundary(), + DEFRA_DESNZ_SOURCE_FAMILY: DefraDesnzProductionParserBoundary(), + IPCC_EFDB_SOURCE_FAMILY: IpccEfdbProductionParserBoundary(), + }[source_family] + batch = parser.parse(artifact) + assert batch.row_count > 0 + return batch + + +def _rewritten_batch(batch: ParserNormalizedOutputBatch) -> ParserNormalizedOutputBatch: + rows = [] + for row in batch.rows: + fields = { + key: value + for key, value in dict(row.normalized_fields).items() + if key + not in { + "source_year", + "source_version", + "run_id", + "provenance_checksum_value", + "master_external_key", + "source_family_master_id", + "source_family_detail_id", + } + } + factor_id = str(fields.get("factor_id") or row.row_id) + fields.update( + { + "source_year": "2024", + "source_version": PARITY_SOURCE_VERSION, + "run_id": PARITY_RUN_ID, + "provenance_checksum_value": _artifact_checksum(row.artifact_reference), + "master_external_key": f"2024:{PARITY_SOURCE_VERSION}:{factor_id}", + } + ) + rows.append( + replace( + row, + normalized_fields=tuple(sorted(fields.items(), key=lambda item: item[0])), + reporting_year=2024, + ) + ) + return ParserNormalizedOutputBatch(rows=tuple(rows)) + + +def _snapshot_schema(connection: object, schema_name: str) -> dict[str, object]: + connection.execute(f"SET search_path TO {schema_name}") + return { + "year_state": _fetchall( + connection, + """ + SELECT + source_family, + max(ingested_year) AS latest_year, + max(ingested_year) + 1 AS next_target_year, + count(*) AS state_rows + FROM source_family_year_states + GROUP BY source_family + ORDER BY source_family + """, + ), + "families": { + source_family: _source_family_snapshot(connection, source_family) + for source_family in SOURCE_FAMILIES + }, + } + + +def _source_family_snapshot(connection: object, source_family: str) -> dict[str, object]: + prefix = { + GHG_PROTOCOL_SOURCE_FAMILY: "ghg", + DEFRA_DESNZ_SOURCE_FAMILY: "defra", + IPCC_EFDB_SOURCE_FAMILY: "ipcc", + }[source_family] + master_table = f"{prefix}_emission_factor_masters" + detail_table = f"{prefix}_emission_factor_details" + master_id = f"{prefix}_emission_factor_master_id" + return { + "counts": _fetchall( + connection, + f""" + SELECT + (SELECT count(*) FROM {master_table} WHERE source_version = %s), + (SELECT count(*) + FROM {detail_table} d + JOIN {master_table} m ON m.{master_id} = d.{master_id} + WHERE m.source_version = %s) + """, + (PARITY_SOURCE_VERSION, PARITY_SOURCE_VERSION), + ), + "masters": _fetchall( + connection, + f""" + SELECT source_family, source_year, source_version, master_external_key, status + FROM {master_table} + WHERE source_version = %s + ORDER BY source_family, source_year, source_version, master_external_key + """, + (PARITY_SOURCE_VERSION,), + ), + "details": _fetchall( + connection, + f""" + SELECT + m.source_family, + m.source_year, + m.source_version, + m.master_external_key, + d.detail_external_key, + d.factor_id, + d.factor_name, + d.factor_value::text, + d.factor_unit, + d.status + FROM {detail_table} d + JOIN {master_table} m ON m.{master_id} = d.{master_id} + WHERE m.source_version = %s + ORDER BY + m.source_family, + m.source_year, + m.source_version, + m.master_external_key, + d.detail_external_key + """, + (PARITY_SOURCE_VERSION,), + ), + } + + +def _fetchall( + connection: object, + statement: str, + parameters: tuple[object, ...] | None = None, +) -> tuple[tuple[object, ...], ...]: + cursor = connection.execute(statement, parameters) + return tuple(tuple(row) for row in cursor.fetchall()) + + +def _fixture_path(source_family: str) -> Path: + family_directory, file_name = { + GHG_PROTOCOL_SOURCE_FAMILY: ("ghg_protocol", "ghg_protocol_sample_factors.csv"), + DEFRA_DESNZ_SOURCE_FAMILY: ("defra_desnz", "defra_desnz_normalized_factors.csv"), + IPCC_EFDB_SOURCE_FAMILY: ("ipcc_efdb", "ipcc_efdb_sample_factors.csv"), + }[source_family] + return ( + Path(__file__).resolve().parent + / "fixtures" + / "source_documents" + / family_directory + / file_name + ) + + +def _artifact_checksum(artifact_reference: str) -> str: + return hashlib.sha256(Path(artifact_reference).read_bytes()).hexdigest() + + +def _redacted_dotnet_output(result: subprocess.CompletedProcess[str]) -> str: + rendered = "\n".join([result.stdout, result.stderr]) + dsn = os.getenv(POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR) + if dsn: + rendered = rendered.replace(dsn, "") + return rendered diff --git a/tests/test_postgresql_phase1_schema_contract.py b/tests/test_postgresql_phase1_schema_contract.py new file mode 100644 index 00000000..8d96752e --- /dev/null +++ b/tests/test_postgresql_phase1_schema_contract.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import re + +from carbonfactor_parser.persistence.postgresql_ddl_renderer import render_postgresql_phase1_schema_ddl +from carbonfactor_parser.persistence.postgresql_schema_catalog import get_required_table_names + + +def _sql() -> str: + return "\n".join(render_postgresql_phase1_schema_ddl().statements) + + +def test_run_history_tables_are_required_phase1_tables() -> None: + required_table_names = set(get_required_table_names()) + + assert "parser_ingestion_runs" in required_table_names + assert "parser_ingestion_source_results" in required_table_names + assert "parser_ingestion_issues" in required_table_names + + +def test_run_history_schema_uses_additive_idempotent_sql() -> None: + statements = _sql() + + for table_name in ( + "parser_ingestion_runs", + "parser_ingestion_source_results", + "parser_ingestion_issues", + ): + assert f"CREATE TABLE IF NOT EXISTS {table_name}" in statements + assert "CREATE INDEX IF NOT EXISTS idx_parser_ingestion_issues_run_id" in statements + assert "CREATE INDEX IF NOT EXISTS idx_parser_ingestion_issues_family_year" in statements + assert "CREATE INDEX IF NOT EXISTS idx_parser_ingestion_issues_code" in statements + + +def test_run_history_schema_contains_expected_columns_defaults_and_foreign_keys() -> None: + statements = _sql() + + assert "run_id text NOT NULL" in statements + assert "started_at timestamp with time zone NOT NULL" in statements + assert "trigger_type text DEFAULT 'operator' NOT NULL" in statements + assert "enabled_source_families jsonb DEFAULT '[]'::jsonb NOT NULL" in statements + assert "metadata jsonb DEFAULT '{}'::jsonb NOT NULL" in statements + assert "created_at timestamp with time zone DEFAULT now() NOT NULL" in statements + assert "REFERENCES parser_ingestion_runs (run_id)" in statements + + +def test_run_history_source_result_unique_contract_is_rendered() -> None: + statements = " ".join(_sql().split()) + + assert ( + "CONSTRAINT uq_parser_ingestion_source_results_run_family_year " + "UNIQUE (run_id, source_family, target_year)" + ) in statements + + +def test_run_history_schema_has_no_destructive_statements() -> None: + statements = _sql().lower() + + assert not re.search(r"\bdrop\b", statements) + assert not re.search(r"\btruncate\b", statements) + assert not re.search(r"\bdelete\s+from\b", statements) diff --git a/tests/test_postgresql_runtime_config_gate.py b/tests/test_postgresql_runtime_config_gate.py new file mode 100644 index 00000000..8dbbce59 --- /dev/null +++ b/tests/test_postgresql_runtime_config_gate.py @@ -0,0 +1,184 @@ +"""Tests for PostgreSQL runtime configuration gate metadata boundary.""" + +import builtins +import inspect +import os +import sqlite3 +import urllib.request + +import carbonfactor_parser.persistence.postgresql_runtime_config_gate as gate_module +from carbonfactor_parser.persistence import ( + PostgreSQLRuntimeConfigGate, + PostgreSQLRuntimeConfigGateDecision, + PostgreSQLRuntimeConfigGateDescription, + PostgreSQLRuntimeConfigGateIssue, + PostgreSQLRuntimeConfigGateStatus, + describe_postgresql_runtime_config_gate, + evaluate_postgresql_runtime_config_gate, +) + + +def test_runtime_config_gate_description_is_side_effect_free() -> None: + description = describe_postgresql_runtime_config_gate() + + assert description.default_status is PostgreSQLRuntimeConfigGateStatus.DISABLED + assert description.disabled_by_default is True + assert description.accepts_caller_intent is True + assert description.loads_environment is False + assert description.loads_config_files is False + assert description.loads_credentials is False + assert description.opens_connection is False + assert description.runs_sql is False + + +def test_runtime_config_gate_defaults_to_disabled() -> None: + decision = evaluate_postgresql_runtime_config_gate() + + assert decision.status is PostgreSQLRuntimeConfigGateStatus.DISABLED + assert decision.requested is False + assert decision.config_loading_enabled is False + assert decision.runtime_enabled is False + assert decision.loads_environment is False + assert decision.loads_config_files is False + assert decision.loads_credentials is False + assert decision.required_future_components == ( + "postgresql_implementation_safety_gate", + "postgresql_persistence_options_contract", + "explicit_runtime_configuration_opt_in", + "approved_secret_source", + ) + assert [issue.code for issue in decision.issues] == [ + "POSTGRESQL_RUNTIME_CONFIG_DISABLED_BY_DEFAULT", + ] + + +def test_runtime_config_gate_returns_blocked_when_requested_but_incomplete() -> None: + decision = evaluate_postgresql_runtime_config_gate( + PostgreSQLRuntimeConfigGate( + requested=True, + safety_gate_approved=True, + options_contract_available=False, + explicit_runtime_opt_in=True, + secret_source_approved=False, + ) + ) + + assert decision.status is PostgreSQLRuntimeConfigGateStatus.BLOCKED + assert decision.required_future_components == ( + "postgresql_persistence_options_contract", + "approved_secret_source", + ) + + +def test_runtime_config_gate_reports_not_enabled_even_when_metadata_ready() -> None: + decision = evaluate_postgresql_runtime_config_gate( + PostgreSQLRuntimeConfigGate( + requested=True, + safety_gate_approved=True, + options_contract_available=True, + explicit_runtime_opt_in=True, + secret_source_approved=True, + ) + ) + + assert decision.status is PostgreSQLRuntimeConfigGateStatus.NOT_ENABLED + assert decision.required_future_components == () + assert decision.config_loading_enabled is False + assert decision.runtime_enabled is False + assert [issue.code for issue in decision.issues] == [ + "POSTGRESQL_RUNTIME_CONFIG_NOT_ENABLED", + ] + + +def test_runtime_config_gate_status_values_are_stable_wire_names() -> None: + assert [status.value for status in PostgreSQLRuntimeConfigGateStatus] == [ + "disabled", + "blocked", + "not_enabled", + ] + + +def test_runtime_config_gate_decision_snapshots_collection_inputs() -> None: + required_components = ["component"] + notes = ["note"] + issues = [PostgreSQLRuntimeConfigGateIssue("CODE", "message")] + + decision = PostgreSQLRuntimeConfigGateDecision( + status=PostgreSQLRuntimeConfigGateStatus.BLOCKED, + requested=True, + reason="reason", + config_loading_enabled=False, + runtime_enabled=False, + loads_environment=False, + loads_config_files=False, + loads_credentials=False, + required_future_components=required_components, + safe_operational_notes=notes, + issues=issues, + ) + required_components.clear() + notes.clear() + issues.clear() + + assert decision.required_future_components == ("component",) + assert decision.safe_operational_notes == ("note",) + assert [issue.code for issue in decision.issues] == ["CODE"] + + +def test_runtime_config_gate_description_snapshots_notes() -> None: + notes = ["note"] + + description = PostgreSQLRuntimeConfigGateDescription( + default_status=PostgreSQLRuntimeConfigGateStatus.DISABLED, + disabled_by_default=True, + accepts_caller_intent=True, + loads_environment=False, + loads_config_files=False, + loads_credentials=False, + opens_connection=False, + runs_sql=False, + notes=notes, + ) + notes.clear() + + assert description.notes == ("note",) + + +def test_runtime_config_gate_has_no_external_side_effects(monkeypatch) -> None: + def fail_side_effect(*args, **kwargs): + raise AssertionError("runtime config gate must not touch external state") + + monkeypatch.setattr(builtins, "open", fail_side_effect) + monkeypatch.setattr(os, "getenv", fail_side_effect) + monkeypatch.setattr(urllib.request, "urlopen", fail_side_effect) + monkeypatch.setattr(sqlite3, "connect", fail_side_effect) + + decision = evaluate_postgresql_runtime_config_gate( + PostgreSQLRuntimeConfigGate(requested=True), + ) + + assert decision.config_loading_enabled is False + assert decision.runtime_enabled is False + assert decision.loads_environment is False + assert decision.loads_config_files is False + assert decision.loads_credentials is False + + +def test_runtime_config_gate_module_has_no_driver_or_runtime_calls() -> None: + source = inspect.getsource(gate_module) + lower_source = source.lower() + + assert "import psycopg" not in source + assert "from psycopg" not in source + assert "asyncpg" not in lower_source + assert "sqlalchemy" not in lower_source + assert "create_engine" not in source + assert "psycopg.connect" not in source + assert "connect(" not in source + assert "cursor(" not in source + assert "execute(" not in source + assert "commit(" not in source + assert "rollback(" not in source + assert "begin(" not in source + assert "os.environ" not in source + assert "getenv" not in source diff --git a/tests/test_postgresql_runtime_year_state.py b/tests/test_postgresql_runtime_year_state.py new file mode 100644 index 00000000..76932300 --- /dev/null +++ b/tests/test_postgresql_runtime_year_state.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import os +import uuid + +import pytest + +from carbonfactor_parser.persistence import ( + POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR, + POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR, +) +from carbonfactor_parser.persistence.postgresql_runtime_config import ( + POSTGRESQL_RUNTIME_DATABASE_ENV_VAR, + POSTGRESQL_RUNTIME_DSN_ENV_VAR, + POSTGRESQL_RUNTIME_HOST_ENV_VAR, + POSTGRESQL_RUNTIME_INITIAL_YEAR_ENV_VAR, + POSTGRESQL_RUNTIME_PASSWORD_ENV_VAR, + POSTGRESQL_RUNTIME_USERNAME_ENV_VAR, + PostgreSQLRuntimeConfigStatus, + load_postgresql_runtime_config, + load_postgresql_runtime_config_from_environment, +) +from carbonfactor_parser.persistence.postgresql_runtime import ( + PostgreSQLRuntimeStartupBlockedError, + start_postgresql_runtime, +) +from carbonfactor_parser.persistence.postgresql_runtime_schema_bootstrap import ( + bootstrap_postgresql_phase1_schema, +) +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + SourceFamily, + get_required_table_names, +) +from carbonfactor_parser.persistence.postgresql_year_state_repository import ( + PostgreSQLSourceFamilyYearStateRepository, + SourceFamilyYearState, +) + + +class FakeCursor: + def __init__(self, rows: list[tuple[object, ...]]) -> None: + self._rows = rows + + def fetchone(self) -> tuple[object, ...] | None: + return self._rows[0] if self._rows else None + + def fetchall(self) -> list[tuple[object, ...]]: + return self._rows + + +class FakeConnection: + def __init__( + self, + *, + latest_years: dict[str, int | None] | None = None, + present_tables: tuple[str, ...] = (), + ) -> None: + self.latest_years = latest_years or {} + self.present_tables = set(present_tables) + self.statements: list[tuple[str, object | None]] = [] + self.commit_count = 0 + + def execute( + self, + statement: str, + parameters: object | None = None, + ) -> FakeCursor: + self.statements.append((statement, parameters)) + normalized = " ".join(statement.split()).lower() + + if "from source_family_year_states" in normalized and "max" in normalized: + family = parameters[0] # type: ignore[index] + return FakeCursor([(self.latest_years.get(str(family)),)]) + + if "from information_schema.tables" in normalized: + return FakeCursor( + [(table_name,) for table_name in sorted(self.present_tables)], + ) + + if normalized.startswith("create table if not exists"): + table_name = normalized.split("create table if not exists ", maxsplit=1)[1] + table_name = table_name.split(" ", maxsplit=1)[0] + self.present_tables.add(table_name) + + if normalized.startswith("insert into source_family_year_states"): + family = parameters[1] # type: ignore[index] + year = parameters[2] # type: ignore[index] + current = self.latest_years.get(str(family)) + self.latest_years[str(family)] = max( + int(year), + int(current) if current is not None else int(year), + ) + + return FakeCursor([]) + + def commit(self) -> None: + self.commit_count += 1 + + +def test_runtime_config_fails_closed_when_required_db_config_is_missing() -> None: + result = load_postgresql_runtime_config({}) + + assert result.status is PostgreSQLRuntimeConfigStatus.BLOCKED + assert result.config is None + assert [issue.code for issue in result.issues] == [ + "POSTGRESQL_RUNTIME_CONFIG_MISSING_HOST", + "POSTGRESQL_RUNTIME_CONFIG_MISSING_DATABASE", + "POSTGRESQL_RUNTIME_CONFIG_MISSING_USERNAME", + "POSTGRESQL_RUNTIME_CONFIG_MISSING_PASSWORD", + ] + + +def test_runtime_startup_fails_closed_when_config_is_blocked() -> None: + result = load_postgresql_runtime_config({}) + + with pytest.raises(PostgreSQLRuntimeStartupBlockedError) as raised: + start_postgresql_runtime(result) + + assert "POSTGRESQL_RUNTIME_CONFIG_MISSING_HOST" in str(raised.value) + + +def test_runtime_config_accepts_explicit_dsn_without_exposing_value() -> None: + private_dsn = "postgresql://user:secret@example.invalid:5432/carbonops" + result = load_postgresql_runtime_config( + { + POSTGRESQL_RUNTIME_DSN_ENV_VAR: private_dsn, + POSTGRESQL_RUNTIME_INITIAL_YEAR_ENV_VAR: "2025", + }, + ) + + assert result.status is PostgreSQLRuntimeConfigStatus.READY + assert result.config is not None + assert result.config.uses_dsn is True + assert result.config.initial_year == 2025 + assert private_dsn not in repr(result) + + +def test_runtime_config_accepts_explicit_field_configuration() -> None: + result = load_postgresql_runtime_config( + { + POSTGRESQL_RUNTIME_HOST_ENV_VAR: "localhost", + POSTGRESQL_RUNTIME_DATABASE_ENV_VAR: "carbonops", + POSTGRESQL_RUNTIME_USERNAME_ENV_VAR: "carbonops", + POSTGRESQL_RUNTIME_PASSWORD_ENV_VAR: "local-only-password", + }, + ) + + assert result.status is PostgreSQLRuntimeConfigStatus.READY + assert result.config is not None + assert result.config.host == "localhost" + assert result.config.port == 5432 + assert result.config.initial_year == 2024 + assert result.config.password_configured is True + assert "local-only-password" not in repr(result) + + +def test_runtime_config_can_read_environment_mapping_when_called() -> None: + result = load_postgresql_runtime_config_from_environment( + { + POSTGRESQL_RUNTIME_DSN_ENV_VAR: "postgresql://example.invalid/db", + }, + ) + + assert result.status is PostgreSQLRuntimeConfigStatus.READY + assert result.loaded_from_environment is True + assert result.loaded_from_explicit_values is False + + +def test_year_state_returns_initial_year_when_no_data_exists() -> None: + repository = PostgreSQLSourceFamilyYearStateRepository(FakeConnection()) + + assert repository.latest_ingested_year(SourceFamily.GHG) is None + assert repository.next_target_year(SourceFamily.GHG) == 2024 + assert repository.get_year_state(SourceFamily.GHG) == SourceFamilyYearState( + source_family=SourceFamily.GHG, + latest_year=None, + next_year=2024, + initial_year=2024, + ) + + +def test_year_state_returns_latest_and_next_year_for_existing_state() -> None: + repository = PostgreSQLSourceFamilyYearStateRepository( + FakeConnection(latest_years={"defra": 2026}), + initial_year=2023, + ) + + assert repository.latest_ingested_year("defra") == 2026 + assert repository.next_target_year("defra") == 2027 + assert repository.get_year_state("defra") == SourceFamilyYearState( + source_family=SourceFamily.DEFRA, + latest_year=2026, + next_year=2027, + initial_year=2023, + ) + + +def test_record_ingested_year_is_idempotent_for_source_family_year() -> None: + connection = FakeConnection() + repository = PostgreSQLSourceFamilyYearStateRepository(connection) + + repository.record_ingested_year(SourceFamily.IPCC, 2024) + repository.record_ingested_year(SourceFamily.IPCC, 2024) + + insert_statements = [ + statement + for statement, _parameters in connection.statements + if "INSERT INTO source_family_year_states" in statement + ] + assert len(insert_statements) == 2 + assert "ON CONFLICT (source_family, ingested_year)" in insert_statements[0] + assert repository.next_target_year(SourceFamily.IPCC) == 2025 + assert connection.commit_count == 2 + + +def test_runtime_schema_bootstrap_creates_missing_tables_idempotently() -> None: + connection = FakeConnection() + + first = bootstrap_postgresql_phase1_schema(connection) + second = bootstrap_postgresql_phase1_schema(connection) + + assert set(first.required_table_names) == set(get_required_table_names()) + assert "source_family_year_states" in first.required_table_names + assert first.missing_table_names == () + assert set(first.created_table_names) == set(get_required_table_names()) + assert second.missing_table_names == () + assert second.created_table_names == () + assert connection.commit_count == 2 + assert any( + "CREATE TABLE IF NOT EXISTS source_family_year_states" in statement + for statement, _parameters in connection.statements + ) + assert any( + "CREATE TABLE IF NOT EXISTS ghg_emission_factor_masters" in statement + and "source_year integer NOT NULL" in statement + and "artifact_checksum_sha256 text" in statement + for statement, _parameters in connection.statements + ) + assert any( + "CREATE TABLE IF NOT EXISTS ipcc_emission_factor_details" in statement + and "raw_fields jsonb NOT NULL" in statement + for statement, _parameters in connection.statements + ) + assert any( + "CREATE INDEX IF NOT EXISTS idx_source_family_year_states_family_year" + in statement + for statement, _parameters in connection.statements + ) + + +@pytest.mark.postgresql_integration +def test_docker_postgresql_schema_bootstrap_and_year_state_integration() -> None: + if os.getenv(POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR) != "1": + pytest.skip("PostgreSQL integration test opt-in is not enabled.") + dsn = os.getenv(POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR) + if not dsn: + pytest.skip("PostgreSQL integration test DSN was not provided.") + + import psycopg + + schema_name = f"carbonops_ph011_{uuid.uuid4().hex}" + with psycopg.connect(dsn) as connection: + connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") + connection.execute(f"SET search_path TO {schema_name}") + + first = bootstrap_postgresql_phase1_schema(connection) + second = bootstrap_postgresql_phase1_schema(connection) + repository = PostgreSQLSourceFamilyYearStateRepository(connection) + + assert first.missing_table_names == () + assert second.missing_table_names == () + assert repository.next_target_year(SourceFamily.DEFRA) == 2024 + + repository.record_ingested_year(SourceFamily.GHG, 2024) + repository.record_ingested_year(SourceFamily.GHG, 2025) + + assert repository.latest_ingested_year(SourceFamily.GHG) == 2025 + assert repository.next_target_year(SourceFamily.GHG) == 2026 diff --git a/tests/test_postgresql_schema_bootstrap_planner.py b/tests/test_postgresql_schema_bootstrap_planner.py index 1db4b08f..2378550a 100644 --- a/tests/test_postgresql_schema_bootstrap_planner.py +++ b/tests/test_postgresql_schema_bootstrap_planner.py @@ -70,6 +70,11 @@ def test_schema_bootstrap_plan_contains_required_phase1_metadata() -> None: "source_documents", "parser_runs", "schema_bootstrap_states", + "parser_ingestion_runs", + "parser_ingestion_source_results", + "parser_ingestion_issues", + "source_family_year_states", + "normalized_factor_records", "ghg_emission_factor_masters", "ghg_emission_factor_details", "defra_emission_factor_masters", @@ -135,8 +140,19 @@ def test_schema_bootstrap_plan_orders_tables_deterministically() -> None: "ingestion_runs", "source_documents", "parser_runs", + "parser_ingestion_issues", + "parser_ingestion_issues", + "parser_ingestion_issues", + "source_family_year_states", + "normalized_factor_records", + "ghg_emission_factor_masters", + "ghg_emission_factor_masters", "ghg_emission_factor_details", + "defra_emission_factor_masters", + "defra_emission_factor_masters", "defra_emission_factor_details", + "ipcc_emission_factor_masters", + "ipcc_emission_factor_masters", "ipcc_emission_factor_details", ) @@ -215,7 +231,7 @@ def test_schema_bootstrap_idempotency_verification_detects_duplicate_definitions sorted( ( duplicate_table_sql(duplicate_table), - "CREATE INDEX idx_duplicate_business_key ON duplicate_table (business_key);", + "CREATE INDEX IF NOT EXISTS idx_duplicate_business_key ON duplicate_table (business_key);", ) ) ) diff --git a/tests/test_postgresql_schema_catalog.py b/tests/test_postgresql_schema_catalog.py index 24e785e6..cf5aa212 100644 --- a/tests/test_postgresql_schema_catalog.py +++ b/tests/test_postgresql_schema_catalog.py @@ -19,6 +19,11 @@ "source_documents", "parser_runs", "schema_bootstrap_states", + "parser_ingestion_runs", + "parser_ingestion_source_results", + "parser_ingestion_issues", + "source_family_year_states", + "normalized_factor_records", ) EXPECTED_SOURCE_FAMILY_TABLE_NAMES = { diff --git a/tests/test_postgresql_schema_ddl.py b/tests/test_postgresql_schema_ddl.py index e65ffd3f..c5d62bca 100644 --- a/tests/test_postgresql_schema_ddl.py +++ b/tests/test_postgresql_schema_ddl.py @@ -46,7 +46,7 @@ def _create_table_names(sql_statements: tuple[str, ...]) -> tuple[str, ...]: names: list[str] = [] for statement in sql_statements: - match = re.match(r"CREATE TABLE ([a-z][a-z0-9_]*) \(", statement) + match = re.match(r"CREATE TABLE (?:IF NOT EXISTS )?([a-z][a-z0-9_]*) \(", statement) if match is not None: names.append(match.group(1)) return tuple(names) @@ -54,7 +54,7 @@ def _create_table_names(sql_statements: tuple[str, ...]) -> tuple[str, ...]: def _sql_for_table(table_name: str) -> str: for statement in render_postgresql_phase1_create_table_ddl(): - if statement.startswith(f"CREATE TABLE {table_name} "): + if statement.startswith(f"CREATE TABLE IF NOT EXISTS {table_name} ") or statement.startswith(f"CREATE TABLE {table_name} "): return statement raise AssertionError(f"CREATE TABLE statement not rendered for {table_name}") @@ -75,9 +75,29 @@ def test_rendered_sql_includes_table_names_and_representative_columns() -> None: assert "run_status text NOT NULL" in _sql_for_table("ingestion_runs") assert "source_document_id uuid NOT NULL" in _sql_for_table("source_documents") assert "source_document_uri text NOT NULL" in _sql_for_table("source_documents") + assert "source_family text NOT NULL" in _sql_for_table( + "ghg_emission_factor_masters" + ) + assert "source_year integer NOT NULL" in _sql_for_table( + "ghg_emission_factor_masters" + ) + assert "source_version text NOT NULL" in _sql_for_table( + "ghg_emission_factor_masters" + ) + assert "artifact_checksum_sha256 text" in _sql_for_table( + "ghg_emission_factor_masters" + ) + assert "archive_reference text" in _sql_for_table( + "ghg_emission_factor_masters" + ) + assert "run_id text" in _sql_for_table("ghg_emission_factor_masters") + assert "status text NOT NULL" in _sql_for_table("ghg_emission_factor_masters") assert "master_external_key text NOT NULL" in _sql_for_table( "ghg_emission_factor_masters" ) + assert "raw_fields jsonb NOT NULL" in _sql_for_table( + "defra_emission_factor_details" + ) assert "factor_value numeric NOT NULL" in _sql_for_table( "defra_emission_factor_details" ) @@ -97,10 +117,18 @@ def test_unique_foreign_key_and_index_fragments_follow_catalog_metadata() -> Non "FOREIGN KEY (source_document_id) " "REFERENCES source_documents (source_document_id)" ) in statements + assert ( + "CONSTRAINT uq_ghg_emission_factor_masters_family_year_version_key " + "UNIQUE (source_family, source_year, source_version, master_external_key)" + ) in statements assert len(index_statements) == expected_index_count assert ( "ON defra_emission_factor_details (defra_emission_factor_master_id);" ) in "\n".join(index_statements) + assert ( + "ON ipcc_emission_factor_masters " + "(source_family, source_year, source_version);" + ) in "\n".join(index_statements) def test_output_ordering_is_deterministic_across_repeated_calls() -> None: @@ -113,8 +141,22 @@ def test_output_ordering_is_deterministic_across_repeated_calls() -> None: def test_output_excludes_forbidden_non_contract_name_fragments() -> None: rendered_sql = "\n".join(render_postgresql_phase1_schema_ddl()).lower() + identifiers = re.findall(r"[a-z][a-z0-9_]*", rendered_sql) + + assert not any( + identifier == fragment or identifier.startswith(f"{fragment}_") + for identifier in identifiers + for fragment in FORBIDDEN_NAME_FRAGMENTS + ) + + +def test_schema_bootstrap_ddl_is_additive_only() -> None: + rendered_sql = "\n".join(render_postgresql_phase1_schema_ddl()).upper() - assert not any(fragment in rendered_sql for fragment in FORBIDDEN_NAME_FRAGMENTS) + assert "DROP " not in rendered_sql + assert "TRUNCATE " not in rendered_sql + assert "DELETE " not in rendered_sql + assert "ALTER TABLE" not in rendered_sql def test_importing_schema_ddl_does_not_import_runtime_heavy_libraries() -> None: diff --git a/tests/test_postgresql_source_family_repository.py b/tests/test_postgresql_source_family_repository.py new file mode 100644 index 00000000..92c5c8b5 --- /dev/null +++ b/tests/test_postgresql_source_family_repository.py @@ -0,0 +1,616 @@ +from __future__ import annotations + +from dataclasses import replace +from decimal import Decimal +import json +import os +from pathlib import Path +import uuid + +import pytest + +from carbonfactor_parser.parsers.file_content_input import ParserFileContentInput +from carbonfactor_parser.parsers.input_artifact_contract import ( + create_phase1_parser_input_artifact, +) +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputRowStatus, + create_parser_normalized_output_batch, + create_parser_normalized_output_row, +) +from carbonfactor_parser.parsers.defra_desnz_content_parser import ( + parse_defra_desnz_file_content, +) +from carbonfactor_parser.parsers.ghg_protocol_content_parser import ( + parse_ghg_protocol_file_content, +) +from carbonfactor_parser.parsers.ipcc_efdb_content_parser import ( + parse_ipcc_efdb_file_content, +) +from carbonfactor_parser.persistence import ( + POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR, + POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR, +) +from carbonfactor_parser.persistence.parsed_factor_persistence_writer import ( + ParsedFactorPersistenceStatus, + build_parsed_factor_persistence_command, + persist_parsed_factor_records, +) +from carbonfactor_parser.persistence.postgresql_runtime_schema_bootstrap import ( + bootstrap_postgresql_phase1_schema, +) +from carbonfactor_parser.persistence.postgresql_schema_catalog import ( + source_family_postgresql_value, +) +from carbonfactor_parser.persistence.postgresql_source_family_sql import ( + detail_insert_sql, + master_insert_sql, +) +from carbonfactor_parser.persistence.postgresql_source_family_ids import ( + detail_uuid, + ingestion_run_uuid, + master_uuid, + source_document_uuid, +) +from carbonfactor_parser.persistence.postgresql_source_family_parameters import ( + detail_parameters, + json_payload, + json_safe, + master_parameters, +) +from carbonfactor_parser.persistence.postgresql_source_family_repository import ( + PostgreSQLSourceFamilyRuntimeRepository, + PostgreSQLSourceSpecificFactorInsertStatus, +) +from carbonfactor_parser.persistence.postgresql_source_family_upserts import ( + ensure_ingestion_run, + ensure_source_document, +) + + +class _FakeCursor: + def __init__(self, row: tuple[object, ...] | None = None) -> None: + self._row = row + + def fetchone(self) -> tuple[object, ...] | None: + return self._row + + +class _FakeConnection: + def __init__(self) -> None: + self.statements: list[tuple[str, object | None]] = [] + self.master_keys: set[tuple[object, ...]] = set() + self.detail_keys: set[tuple[object, ...]] = set() + self.commit_count = 0 + self.rollback_count = 0 + + def execute(self, statement: str, parameters: object | None = None) -> _FakeCursor: + self.statements.append((statement, parameters)) + normalized = " ".join(statement.split()).lower() + if "_emission_factor_masters" in normalized and normalized.startswith("insert"): + assert isinstance(parameters, tuple) + key = (parameters[1], parameters[2], parameters[3], parameters[8]) + if key in self.master_keys: + return _FakeCursor() + self.master_keys.add(key) + return _FakeCursor((parameters[0],)) + if "_emission_factor_details" in normalized and normalized.startswith("insert"): + assert isinstance(parameters, tuple) + key = (parameters[1], parameters[2]) + if key in self.detail_keys: + return _FakeCursor() + self.detail_keys.add(key) + return _FakeCursor((parameters[0],)) + return _FakeCursor() + + def commit(self) -> None: + self.commit_count += 1 + + def rollback(self) -> None: + self.rollback_count += 1 + + + +def test_postgresql_source_family_stable_uuid_helpers_match_legacy_payloads() -> None: + command = build_parsed_factor_persistence_command(_payload("ghg_protocol")) + master = command.master_records[0] + detail = command.detail_records[0] + + assert source_document_uuid(master) == _legacy_stable_uuid( + "source_document", + "ghg_protocol", + master.source_document_id, + ) + ingestion_source = master.ingestion_run_id or master.run_id + if ingestion_source is None: + ingestion_source = ( + f"ghg_protocol:{master.source_year}:{master.source_version}" + ) + assert ingestion_run_uuid(master) == _legacy_stable_uuid( + "ingestion_run", + "ghg_protocol", + ingestion_source, + ) + assert master_uuid(master.source_family, master.source_family_master_id) == ( + _legacy_stable_uuid( + "master", + "ghg_protocol", + master.source_family_master_id, + ) + ) + assert detail_uuid(detail.source_family, detail.source_family_detail_id) == ( + _legacy_stable_uuid( + "detail", + "ghg_protocol", + detail.source_family_detail_id, + ) + ) + assert source_document_uuid(master) == source_document_uuid(master) + assert ingestion_run_uuid(master) == ingestion_run_uuid(master) + + +def test_postgresql_source_family_ingestion_run_uuid_fallback_is_deterministic() -> None: + command = build_parsed_factor_persistence_command(_payload("defra_desnz")) + master = replace(command.master_records[0], ingestion_run_id=None, run_id=None) + + expected = _legacy_stable_uuid( + "ingestion_run", + "defra_desnz", + f"defra_desnz:{master.source_year}:{master.source_version}", + ) + + assert ingestion_run_uuid(master) == expected + assert ingestion_run_uuid(master) == ingestion_run_uuid(master) + + +def test_postgresql_source_family_ensure_ingestion_run_upsert_compatibility() -> None: + connection = _FakeConnection() + command = build_parsed_factor_persistence_command(_payload("ipcc_efdb")) + master = replace( + command.master_records[0], + source_year=2025, + source_version="ipcc-2025", + source_release="release-a", + ingestion_run_id=None, + run_id="cycle-run-001", + ) + + ensure_ingestion_run(connection, master) + + assert len(connection.statements) == 1 + statement, parameters = connection.statements[0] + sql = _normalized_sql(statement) + assert "INSERT INTO ingestion_runs" in sql + assert "ingestion_run_id" in sql + assert "run_status" in sql + assert "VALUES (%s, %s, NOW(), NOW())" in sql + assert "ON CONFLICT (ingestion_run_id) DO NOTHING" in sql + assert parameters == (str(ingestion_run_uuid(master)), "completed") + assert ingestion_run_uuid(master) == _legacy_stable_uuid( + "ingestion_run", + source_family_postgresql_value(master.source_family), + master.run_id, + ) + assert master.source_year == 2025 + assert master.source_version == "ipcc-2025" + assert master.source_release == "release-a" + + +def test_postgresql_source_family_ensure_source_document_upsert_compatibility() -> None: + connection = _FakeConnection() + command = build_parsed_factor_persistence_command(_payload("ghg_protocol")) + master = replace( + command.master_records[0], + artifact_reference=None, + artifact_checksum_sha256=None, + ) + + ensure_source_document(connection, master) + + assert len(connection.statements) == 1 + statement, parameters = connection.statements[0] + sql = _normalized_sql(statement) + assert "INSERT INTO source_documents" in sql + assert "source_document_id" in sql + assert "ingestion_run_id" in sql + assert "source_family" in sql + assert "source_document_uri" in sql + assert "source_checksum_sha256" in sql + assert "acquisition_status" in sql + assert "VALUES (%s, %s, %s, %s, %s, %s, NOW(), NOW(), NOW())" in sql + assert ( + "ON CONFLICT (source_family, source_document_uri, source_checksum_sha256)" + in sql + ) + assert "DO NOTHING" in sql + assert parameters == ( + str(source_document_uuid(master)), + str(ingestion_run_uuid(master)), + source_family_postgresql_value(master.source_family), + master.source_document_id, + "checksum-unavailable", + "downloaded", + ) + + +def test_postgresql_source_family_ensure_source_document_uses_artifact_values() -> None: + connection = _FakeConnection() + command = build_parsed_factor_persistence_command(_payload("defra_desnz")) + master = command.master_records[0] + + ensure_source_document(connection, master) + + _statement, parameters = connection.statements[0] + assert isinstance(parameters, tuple) + assert parameters[3] == master.artifact_reference + assert parameters[4] == master.artifact_checksum_sha256 + assert parameters[2] == source_family_postgresql_value(master.source_family) + + +@pytest.mark.parametrize( + ("source_family", "master_table", "master_id"), + ( + ("ghg_protocol", "ghg_emission_factor_masters", "ghg_emission_factor_master_id"), + ("defra_desnz", "defra_emission_factor_masters", "defra_emission_factor_master_id"), + ("ipcc_efdb", "ipcc_emission_factor_masters", "ipcc_emission_factor_master_id"), + ), +) +def test_postgresql_source_family_master_insert_sql_compatibility( + source_family: str, + master_table: str, + master_id: str, +) -> None: + sql = _normalized_sql(master_insert_sql(source_family)) + + assert f"INSERT INTO {master_table}" in sql + assert ( + "ON CONFLICT (source_family, source_year, source_version, master_external_key)" + in sql + ) + assert f"RETURNING {master_id}" in sql + assert "metadata" in sql + assert "%s::jsonb" in sql + + +@pytest.mark.parametrize( + ("source_family", "detail_table", "master_id", "detail_id"), + ( + ( + "ghg_protocol", + "ghg_emission_factor_details", + "ghg_emission_factor_master_id", + "ghg_emission_factor_detail_id", + ), + ( + "defra_desnz", + "defra_emission_factor_details", + "defra_emission_factor_master_id", + "defra_emission_factor_detail_id", + ), + ( + "ipcc_efdb", + "ipcc_emission_factor_details", + "ipcc_emission_factor_master_id", + "ipcc_emission_factor_detail_id", + ), + ), +) +def test_postgresql_source_family_detail_insert_sql_compatibility( + source_family: str, + detail_table: str, + master_id: str, + detail_id: str, +) -> None: + sql = _normalized_sql(detail_insert_sql(source_family)) + + assert f"INSERT INTO {detail_table}" in sql + assert f"ON CONFLICT ({master_id}, detail_external_key)" in sql + assert f"RETURNING {detail_id}" in sql + assert "raw_fields" in sql + assert "normalized_fields" in sql + assert sql.count("%s::jsonb") == 2 + + +def test_postgresql_source_family_master_parameters_compatibility() -> None: + command = build_parsed_factor_persistence_command(_payload("ghg_protocol")) + master = command.master_records[0] + + parameters = master_parameters(master) + + assert len(parameters) == 18 + assert parameters[0] == str( + master_uuid(master.source_family, master.source_family_master_id) + ) + assert parameters[1] == "ghg_protocol" + assert parameters[2] == master.source_year + assert parameters[3] == master.source_version + assert parameters[5] == str(source_document_uuid(master)) + assert parameters[6] == str(ingestion_run_uuid(master)) + assert parameters[8] == master.master_external_key + assert parameters[16] == master.record_checksum_sha256 + assert parameters[17] == json.dumps( + json_safe(master.metadata), sort_keys=True, separators=(",", ":") + ) + assert ": " not in str(parameters[17]) + assert ", " not in str(parameters[17]) + + +def test_postgresql_source_family_detail_parameters_compatibility() -> None: + command = build_parsed_factor_persistence_command(_payload("defra_desnz")) + detail = command.detail_records[0] + + parameters = detail_parameters(detail) + + assert len(parameters) == 12 + assert parameters[0] == str( + detail_uuid(detail.source_family, detail.source_family_detail_id) + ) + assert parameters[1] == str( + master_uuid(detail.source_family, detail.source_family_master_id) + ) + assert parameters[2] == detail.detail_external_key + assert parameters[3] == detail.source_row_number + assert parameters[4] == detail.factor_id + assert parameters[6] == str(Decimal(str(detail.factor_value))) + assert parameters[9] == detail.record_checksum_sha256 + assert parameters[10] == json.dumps( + json_safe(detail.raw_fields), sort_keys=True, separators=(",", ":") + ) + assert parameters[11] == json.dumps( + json_safe(detail.normalized_fields), sort_keys=True, separators=(",", ":") + ) + assert ": " not in str(parameters[10]) + assert ", " not in str(parameters[10]) + assert ": " not in str(parameters[11]) + assert ", " not in str(parameters[11]) + + +def test_postgresql_source_family_json_helpers_preserve_legacy_behavior() -> None: + payload = { + 2: (Decimal("1.20"), [Decimal("3.40"), {"b": 2, "a": Decimal("5")}]), + "10": "ten", + "a": None, + } + + safe_payload = json_safe(payload) + + assert safe_payload == { + "10": "ten", + "2": ["1.20", ["3.40", {"a": "5", "b": 2}]], + "a": None, + } + assert json_payload(payload) == ( + '{"10":"ten","2":["1.20",["3.40",{"a":"5","b":2}]],"a":null}' + ) + + +def test_postgresql_source_family_repository_inserts_and_skips_idempotently() -> None: + connection = _FakeConnection() + repository = PostgreSQLSourceFamilyRuntimeRepository(connection) + payload = _payload("defra_desnz") + + first = persist_parsed_factor_records(payload, repository) + second = persist_parsed_factor_records(payload, repository) + + assert first.status is ParsedFactorPersistenceStatus.DECLARED + assert first.persisted_master_count == 1 + assert first.persisted_detail_count == len(payload.records) + assert first.skipped_master_count == 0 + assert second.persisted_master_count == 0 + assert second.persisted_detail_count == 0 + assert second.skipped_master_count == 1 + assert second.skipped_detail_count == len(payload.records) + assert connection.commit_count == 2 + assert any( + "INSERT INTO defra_emission_factor_masters" in statement + for statement, _parameters in connection.statements + ) + assert any( + "INSERT INTO defra_emission_factor_details" in statement + for statement, _parameters in connection.statements + ) + + +def test_writer_result_exposes_master_detail_summary_counts() -> None: + connection = _FakeConnection() + repository = PostgreSQLSourceFamilyRuntimeRepository(connection) + payload = _payload("ghg_protocol") + + first = persist_parsed_factor_records(payload, repository) + second = persist_parsed_factor_records(payload, repository) + + assert first.master_inserted_count == 1 + assert first.detail_inserted_count == len(payload.records) + assert second.master_skipped_count == 1 + assert second.detail_skipped_count == len(payload.records) + assert second.validation_failure_count == 0 + assert second.final_status == "declared" + + +def test_insert_normalized_factor_records_returns_user_visible_counts() -> None: + connection = _FakeConnection() + repository = PostgreSQLSourceFamilyRuntimeRepository(connection) + artifact = create_phase1_parser_input_artifact( + source_family="ghg_protocol", + artifact_reference="artifact://ghg/factors.csv", + reporting_year=2024, + ) + batch = create_parser_normalized_output_batch( + ( + create_parser_normalized_output_row( + artifact=artifact, + row_id="ghg-row-001", + source_row_number=2, + status=ParserNormalizedOutputRowStatus.VALIDATED, + normalized_fields={ + "source_year": 2024, + "source_version": "ghg-2024", + "factor_id": "GHG-001", + "factor_value": Decimal("1.25"), + "factor_unit": "kgco2e", + "source_checksum_sha256": "c" * 64, + }, + ), + ) + ) + + first = repository.insert_normalized_factor_records(batch) + second = repository.insert_normalized_factor_records(batch) + + assert first.status is PostgreSQLSourceSpecificFactorInsertStatus.INSERTED + assert first.master_inserted == 1 + assert first.detail_inserted == 1 + assert first.validation_error_count == 0 + assert second.master_skipped == 1 + assert second.detail_skipped == 1 + assert second.skipped_duplicate == 1 + + +@pytest.mark.parametrize( + ("source_family", "master_table", "detail_table"), + ( + ( + "ghg_protocol", + "ghg_emission_factor_masters", + "ghg_emission_factor_details", + ), + ( + "defra_desnz", + "defra_emission_factor_masters", + "defra_emission_factor_details", + ), + ( + "ipcc_efdb", + "ipcc_emission_factor_masters", + "ipcc_emission_factor_details", + ), + ), +) +def test_repository_targets_all_source_specific_table_pairs( + source_family: str, + master_table: str, + detail_table: str, +) -> None: + connection = _FakeConnection() + repository = PostgreSQLSourceFamilyRuntimeRepository(connection) + + result = persist_parsed_factor_records(_payload(source_family), repository) + + assert result.persisted_master_count == 1 + assert result.persisted_detail_count >= 1 + assert any(f"INSERT INTO {master_table}" in s for s, _ in connection.statements) + assert any(f"INSERT INTO {detail_table}" in s for s, _ in connection.statements) + + +@pytest.mark.postgresql_integration +def test_docker_postgresql_source_specific_master_detail_tables_integration() -> None: + if os.getenv(POSTGRESQL_INTEGRATION_TEST_OPT_IN_ENV_VAR) != "1": + pytest.skip("PostgreSQL integration test opt-in is not enabled.") + dsn = os.getenv(POSTGRESQL_INTEGRATION_TEST_DSN_ENV_VAR) + if not dsn: + pytest.skip("PostgreSQL integration test DSN was not provided.") + + import psycopg + + schema_name = f"carbonops_ph020_{uuid.uuid4().hex}" + with psycopg.connect(dsn) as connection: + connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") + connection.execute(f"SET search_path TO {schema_name}") + bootstrap_postgresql_phase1_schema(connection) + repository = PostgreSQLSourceFamilyRuntimeRepository(connection) + + for source_family in ("ghg_protocol", "defra_desnz", "ipcc_efdb"): + first = persist_parsed_factor_records(_payload(source_family), repository) + second = persist_parsed_factor_records(_payload(source_family), repository) + assert first.persisted_master_count == 1 + assert first.persisted_detail_count >= 1 + assert second.skipped_master_count == 1 + assert second.skipped_detail_count >= 1 + + table_counts = { + table_name: connection.execute( + f"SELECT COUNT(*) FROM {table_name}", + ).fetchone()[0] + for table_name in ( + "ghg_emission_factor_masters", + "ghg_emission_factor_details", + "defra_emission_factor_masters", + "defra_emission_factor_details", + "ipcc_emission_factor_masters", + "ipcc_emission_factor_details", + ) + } + assert all(count > 0 for count in table_counts.values()) + + +def _payload(source_family: str): + if source_family == "ghg_protocol": + result = parse_ghg_protocol_file_content(_content_input(source_family)) + elif source_family == "defra_desnz": + result = parse_defra_desnz_file_content(_content_input(source_family)) + elif source_family == "ipcc_efdb": + result = parse_ipcc_efdb_file_content(_content_input(source_family)) + else: + raise ValueError(source_family) + assert result.raw_record_payload is not None + return result.raw_record_payload + + +def _content_input(source_family: str) -> ParserFileContentInput: + fixture_name = { + "ghg_protocol": "ghg_protocol/ghg_protocol_sample_factors.csv", + "defra_desnz": "defra_desnz/defra_desnz_normalized_factors.csv", + "ipcc_efdb": "ipcc_efdb/ipcc_efdb_sample_factors.csv", + }[source_family] + fixture_path = Path("tests/fixtures/source_documents") / fixture_name + return ParserFileContentInput( + source_family=source_family, + source_id=source_family, + content=fixture_path.read_text(encoding="utf-8"), + artifact_reference=str(fixture_path), + checksum_sha256="c" * 64, + content_type="text/csv", + format_hint="csv", + ) + + +def test_source_family_repository_redacts_database_errors() -> None: + private_dsn = "postgresql://carbonops:secret@example.invalid:5432/carbonops" + connection = _FailingConnection( + RuntimeError(f"could not connect dsn={private_dsn} password=secret token=abc") + ) + repository = PostgreSQLSourceFamilyRuntimeRepository(connection) + + command = build_parsed_factor_persistence_command(_payload("ghg_protocol")) + + result = repository.persist_source_family_records( + command.master_records, + command.detail_records, + ) + + assert result.status.value == "failed_database" + assert connection.rollback_count == 1 + message = result.issues[0].message + assert "secret" not in message + assert private_dsn not in message + assert "password=***" in message + assert "token=***" in message + + +class _FailingConnection(_FakeConnection): + def __init__(self, exc: Exception) -> None: + super().__init__() + self._exc = exc + + def execute(self, statement: str, parameters: object | None = None) -> _FakeCursor: + raise self._exc + + + +def _legacy_stable_uuid(*values: object) -> uuid.UUID: + payload = json.dumps(tuple(str(value) for value in values), separators=(",", ":")) + return uuid.uuid5(uuid.NAMESPACE_URL, payload) + + +def _normalized_sql(statement: str) -> str: + return " ".join(statement.split()) diff --git a/tests/test_production_config_boundary.py b/tests/test_production_config_boundary.py new file mode 100644 index 00000000..c7a3ada2 --- /dev/null +++ b/tests/test_production_config_boundary.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import builtins +import inspect +import os +import sqlite3 +import urllib.request + +from carbonfactor_parser.persistence.production_config_boundary import ( + PRODUCTION_CONFIG_REQUIRED_ENV_VARS, + PRODUCTION_CONFIG_SECRET_ENV_VARS, + describe_production_config_boundary, + validate_production_config_mapping, +) + + +def _valid_config(**overrides: str | None) -> dict[str, str | None]: + values: dict[str, str | None] = { + "CARBONOPS_PARSER_ENV": "production", + "CARBONOPS_PARSER_DATABASE_PROVIDER": "postgres", + "CARBONOPS_PARSER_POSTGRES_HOST": "db.internal.example", + "CARBONOPS_PARSER_POSTGRES_PORT": "5432", + "CARBONOPS_PARSER_POSTGRES_DATABASE": "carbonops_parser", + "CARBONOPS_PARSER_POSTGRES_USERNAME": "carbonops_runtime", + "CARBONOPS_PARSER_POSTGRES_PASSWORD": "runtime-secret-not-returned", + "CARBONOPS_PARSER_POSTGRES_SCHEMA": "carbonops", + "CARBONOPS_PARSER_RAW_ARCHIVE_PATH": "/var/lib/carbonops/raw", + "CARBONOPS_PARSER_LOG_LEVEL": "info", + } + values.update(overrides) + return values + + +def test_production_config_boundary_documents_aligned_required_env_vars() -> None: + description = describe_production_config_boundary() + + assert description.required_env_vars == PRODUCTION_CONFIG_REQUIRED_ENV_VARS + assert description.secret_env_vars == PRODUCTION_CONFIG_SECRET_ENV_VARS + assert description.provider == "postgres" + assert description.loads_environment is False + assert description.loads_config_files is False + assert description.loads_credentials is False + assert description.logs_secret_values is False + assert "CARBONOPS_PARSER_POSTGRES_PASSWORD" in description.secret_env_vars + + +def test_valid_production_config_mapping_passes_without_returning_secret() -> None: + result = validate_production_config_mapping(_valid_config()) + + assert result.is_valid + assert result.issues == () + + +def test_missing_required_production_keys_fail_closed_with_safe_messages() -> None: + result = validate_production_config_mapping( + _valid_config( + CARBONOPS_PARSER_POSTGRES_PASSWORD=" ", + CARBONOPS_PARSER_RAW_ARCHIVE_PATH=None, + ), + ) + + assert not result.is_valid + assert [issue.field_name for issue in result.issues] == [ + "CARBONOPS_PARSER_POSTGRES_PASSWORD", + "CARBONOPS_PARSER_RAW_ARCHIVE_PATH", + ] + rendered = repr(result) + assert "runtime-secret-not-returned" not in rendered + assert "Password" + "=" not in rendered + + +def test_invalid_values_fail_with_actionable_key_only_messages() -> None: + result = validate_production_config_mapping( + _valid_config( + CARBONOPS_PARSER_DATABASE_PROVIDER="mysql", + CARBONOPS_PARSER_POSTGRES_PORT="not-a-port", + CARBONOPS_PARSER_LOG_LEVEL="verbose", + CARBONOPS_PARSER_POSTGRES_CONNECTION_STRING=( + "Host=db;Username=svc;" + "Password" + "=raw-secret" + ), + ), + ) + + assert [issue.code for issue in result.issues] == [ + "PRODUCTION_CONFIG_UNSUPPORTED_DATABASE_PROVIDER", + "PRODUCTION_CONFIG_INVALID_POSTGRES_PORT", + "PRODUCTION_CONFIG_INVALID_LOG_LEVEL", + "PRODUCTION_CONFIG_RAW_CONNECTION_STRING_NOT_ALLOWED", + ] + rendered = repr(result) + assert "mysql" not in rendered + assert "not-a-port" not in rendered + assert "verbose" not in rendered + assert "raw-secret" not in rendered + + +def test_production_config_validation_has_no_external_side_effects(monkeypatch) -> None: + def fail_side_effect(*args, **kwargs): + raise AssertionError("production config validation must not touch external state") + + monkeypatch.setattr(builtins, "open", fail_side_effect) + monkeypatch.setattr(os, "getenv", fail_side_effect) + monkeypatch.setattr(urllib.request, "urlopen", fail_side_effect) + monkeypatch.setattr(sqlite3, "connect", fail_side_effect) + + result = validate_production_config_mapping(_valid_config()) + + assert result.is_valid + + +def test_production_config_module_does_not_load_environment_or_connect() -> None: + import carbonfactor_parser.persistence.production_config_boundary as module + + source = inspect.getsource(module) + lower_source = source.lower() + + assert "os.environ" not in source + assert "getenv" not in source + assert "open(" not in source + assert "connect(" not in source + assert "psycopg" not in lower_source + assert "sqlalchemy" not in lower_source diff --git a/tests/test_production_e2e_year_orchestrator.py b/tests/test_production_e2e_year_orchestrator.py new file mode 100644 index 00000000..fac227ff --- /dev/null +++ b/tests/test_production_e2e_year_orchestrator.py @@ -0,0 +1,353 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from carbonfactor_parser.parsers.normalized_output_row_contract import ( + ParserNormalizedOutputBatch, + ParserNormalizedOutputRow, + ParserNormalizedOutputRowStatus, + create_parser_normalized_output_batch, +) +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + PRODUCTION_E2E_SOURCE_FAMILIES, + ProductionE2EDownloadedArtifact, + ProductionE2ESourceYearDiscoveryRequest, + ProductionE2ESourceYearDiscoveryResult, + ProductionE2ESourceYearDiscoveryStatus, + ProductionE2ESourceYearDownloadResult, + ProductionE2ESourceYearDownloadStatus, + ProductionE2EValidationResult, + ProductionE2EValidationStatus, + ProductionE2EYearFamilyStatus, + ProductionE2EYearOrchestratorDependencies, + ProductionE2EYearOrchestratorRequest, + ProductionE2EYearRunStatus, + ProductionE2EYearSelectionStatus, + run_production_e2e_year_orchestrator, +) + + +def test_first_run_without_existing_data_targets_default_2024() -> None: + year_state = _FakeYearStateRepository() + adapter = _FakeSourceAdapter("ghg_protocol", available_years=(2024,)) + parser = _FakeParser() + insert_repository = _FakeInsertRepository() + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="production-e2e-run-001", + enabled_source_families=("ghg_protocol",), + ), + _dependencies( + year_state, + {"ghg_protocol": adapter}, + {"ghg_protocol": parser}, + insert_repository, + ), + ) + + family = result.family_results[0] + assert result.status is ProductionE2EYearRunStatus.COMPLETED + assert family.status is ProductionE2EYearFamilyStatus.COMPLETED + assert family.year_state.latest_year is None + assert family.year_state.target_year == 2024 + assert family.year_state.selection_status is ( + ProductionE2EYearSelectionStatus.INITIAL_YEAR_SELECTED + ) + assert adapter.discovery_requests[0].target_year == 2024 + assert parser.parsed_artifacts[0].source_year == 2024 + assert insert_repository.inserted_batches[0].row_count == 1 + assert year_state.recorded_years == (("ghg", 2024),) + + +def test_next_run_after_2024_targets_2025() -> None: + year_state = _FakeYearStateRepository({"defra": 2024}) + adapter = _FakeSourceAdapter("defra_desnz", available_years=(2025,)) + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="production-e2e-run-002", + enabled_source_families=("defra_desnz",), + ), + _dependencies(year_state, {"defra_desnz": adapter}), + ) + + family = result.family_results[0] + assert family.status is ProductionE2EYearFamilyStatus.COMPLETED + assert family.year_state.latest_year == 2024 + assert family.year_state.target_year == 2025 + assert family.year_state.selection_status is ( + ProductionE2EYearSelectionStatus.NEXT_YEAR_SELECTED + ) + assert adapter.discovery_requests[0].target_year == 2025 + assert year_state.recorded_years == (("defra", 2025),) + + +def test_next_run_after_2025_targets_2026() -> None: + year_state = _FakeYearStateRepository({"ipcc": 2025}) + adapter = _FakeSourceAdapter("ipcc_efdb", available_years=(2026,)) + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="production-e2e-run-003", + enabled_source_families=("ipcc_efdb",), + ), + _dependencies(year_state, {"ipcc_efdb": adapter}), + ) + + family = result.family_results[0] + assert result.status is ProductionE2EYearRunStatus.COMPLETED + assert family.year_state.latest_year == 2025 + assert family.year_state.target_year == 2026 + assert adapter.discovery_requests[0].target_year == 2026 + assert year_state.recorded_years == (("ipcc", 2026),) + + +def test_next_run_after_2026_reports_2027_unavailable_as_safe_noop() -> None: + year_state = _FakeYearStateRepository({"ghg": 2026}) + adapter = _FakeSourceAdapter("ghg_protocol", available_years=()) + parser = _FakeParser() + insert_repository = _FakeInsertRepository() + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest( + run_id="production-e2e-run-004", + enabled_source_families=("ghg_protocol",), + ), + _dependencies( + year_state, + {"ghg_protocol": adapter}, + {"ghg_protocol": parser}, + insert_repository, + ), + ) + + family = result.family_results[0] + assert result.status is ProductionE2EYearRunStatus.COMPLETED + assert family.status is ProductionE2EYearFamilyStatus.NO_AVAILABLE_SOURCE_YEAR + assert family.year_state.latest_year == 2026 + assert family.year_state.target_year == 2027 + assert family.discovery_result is not None + assert family.discovery_result.status is ( + ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR + ) + assert parser.parsed_artifacts == () + assert insert_repository.inserted_batches == () + assert year_state.recorded_years == () + assert result.summary.no_available_source_year_count == 1 + assert result.summary.failed_family_count == 0 + + +def test_all_three_source_families_are_represented_in_one_run_summary() -> None: + year_state = _FakeYearStateRepository() + adapters = { + source_family: _FakeSourceAdapter(source_family, available_years=(2024,)) + for source_family in PRODUCTION_E2E_SOURCE_FAMILIES + } + parsers = { + source_family: _FakeParser() + for source_family in PRODUCTION_E2E_SOURCE_FAMILIES + } + insert_repository = _FakeInsertRepository() + + result = run_production_e2e_year_orchestrator( + ProductionE2EYearOrchestratorRequest(run_id="production-e2e-run-005"), + _dependencies(year_state, adapters, parsers, insert_repository), + ) + + assert result.status is ProductionE2EYearRunStatus.COMPLETED + assert result.selected_source_families == PRODUCTION_E2E_SOURCE_FAMILIES + assert tuple(family.source_family for family in result.family_results) == ( + "ghg_protocol", + "defra_desnz", + "ipcc_efdb", + ) + assert tuple(family.year_state.target_year for family in result.family_results) == ( + 2024, + 2024, + 2024, + ) + assert result.summary.requested_family_count == 3 + assert result.summary.completed_family_count == 3 + assert result.summary.no_available_source_year_count == 0 + assert result.summary.failed_family_count == 0 + assert result.summary.parsed_row_count == 3 + assert result.summary.attempted_insert_count == 3 + assert result.summary.inserted_count == 3 + assert year_state.recorded_years == ( + ("ghg", 2024), + ("defra", 2024), + ("ipcc", 2024), + ) + + +class _FakeYearStateRepository: + def __init__(self, latest_years: dict[str, int] | None = None) -> None: + self.latest_years = dict(latest_years or {}) + self._recorded_years: list[tuple[str, int]] = [] + + @property + def recorded_years(self) -> tuple[tuple[str, int], ...]: + return tuple(self._recorded_years) + + def latest_ingested_year(self, source_family: str) -> int | None: + return self.latest_years.get(source_family) + + def record_ingested_year(self, source_family: str, ingested_year: int) -> None: + self.latest_years[source_family] = ingested_year + self._recorded_years.append((source_family, ingested_year)) + + +class _FakeSourceAdapter: + def __init__(self, source_family: str, *, available_years: tuple[int, ...]) -> None: + self._source_family = source_family + self.available_years = set(available_years) + self._discovery_requests: list[ProductionE2ESourceYearDiscoveryRequest] = [] + + @property + def source_family(self) -> str: + return self._source_family + + @property + def discovery_requests( + self, + ) -> tuple[ProductionE2ESourceYearDiscoveryRequest, ...]: + return tuple(self._discovery_requests) + + def discover_target_year( + self, + request: ProductionE2ESourceYearDiscoveryRequest, + ) -> ProductionE2ESourceYearDiscoveryResult: + self._discovery_requests.append(request) + if request.target_year not in self.available_years: + return ProductionE2ESourceYearDiscoveryResult( + status=ProductionE2ESourceYearDiscoveryStatus.NO_AVAILABLE_SOURCE_YEAR, + source_family=request.source_family, + target_year=request.target_year, + reason_code="target_year_not_published", + ) + return ProductionE2ESourceYearDiscoveryResult( + status=ProductionE2ESourceYearDiscoveryStatus.SOURCE_YEAR_AVAILABLE, + source_family=request.source_family, + target_year=request.target_year, + artifact_reference=( + f"local://{request.source_family}/{request.target_year}.csv" + ), + ) + + def download_target_year( + self, + discovery_result: ProductionE2ESourceYearDiscoveryResult, + ) -> ProductionE2ESourceYearDownloadResult: + assert discovery_result.artifact_reference is not None + return ProductionE2ESourceYearDownloadResult( + status=ProductionE2ESourceYearDownloadStatus.DOWNLOADED, + source_family=discovery_result.source_family, + target_year=discovery_result.target_year, + artifact=ProductionE2EDownloadedArtifact( + source_family=discovery_result.source_family, + source_year=discovery_result.target_year, + artifact_reference=discovery_result.artifact_reference, + checksum_sha256="fake-checksum", + content_type="text/csv", + format_hint="csv", + ), + ) + + +class _FakeParser: + def __init__(self) -> None: + self._parsed_artifacts: list[ProductionE2EDownloadedArtifact] = [] + + @property + def parsed_artifacts(self) -> tuple[ProductionE2EDownloadedArtifact, ...]: + return tuple(self._parsed_artifacts) + + def parse( + self, + artifact: ProductionE2EDownloadedArtifact, + ) -> ParserNormalizedOutputBatch: + self._parsed_artifacts.append(artifact) + return create_parser_normalized_output_batch( + ( + ParserNormalizedOutputRow( + source_family=artifact.source_family, + source_key=artifact.source_family, + parser_key=f"{artifact.source_family}_parser", + artifact_reference=artifact.artifact_reference, + row_id=f"{artifact.source_family}-{artifact.source_year}-1", + normalized_fields=( + ("source_family", artifact.source_family), + ("source_id", artifact.source_family), + ("factor_id", "factor-1"), + ("factor_name", "Example factor"), + ("factor_value", "1.0"), + ("unit", "kg CO2e"), + ), + status=ParserNormalizedOutputRowStatus.VALIDATED, + reporting_year=artifact.source_year, + ), + ) + ) + + +class _FakeValidationBoundary: + def validate( + self, + batch: ParserNormalizedOutputBatch, + ) -> ProductionE2EValidationResult: + return ProductionE2EValidationResult( + status=ProductionE2EValidationStatus.VALIDATED, + diagnostic_count=0, + blocking_error_count=0, + warning_count=0, + ) + + +@dataclass(frozen=True) +class _FakeInsertResult: + status: str + attempted: int + inserted: int + skipped_duplicate: int = 0 + failed: int = 0 + validation_error_count: int = 0 + + +class _FakeInsertRepository: + def __init__(self) -> None: + self._inserted_batches: list[ParserNormalizedOutputBatch] = [] + + @property + def inserted_batches(self) -> tuple[ParserNormalizedOutputBatch, ...]: + return tuple(self._inserted_batches) + + def insert_normalized_factor_records( + self, + batch: ParserNormalizedOutputBatch, + ) -> _FakeInsertResult: + self._inserted_batches.append(batch) + return _FakeInsertResult( + status="inserted", + attempted=batch.row_count, + inserted=batch.row_count, + ) + + +def _dependencies( + year_state: _FakeYearStateRepository, + adapters: dict[str, _FakeSourceAdapter], + parsers: dict[str, _FakeParser] | None = None, + insert_repository: _FakeInsertRepository | None = None, +) -> ProductionE2EYearOrchestratorDependencies: + active_parsers = parsers or { + source_family: _FakeParser() + for source_family in PRODUCTION_E2E_SOURCE_FAMILIES + } + return ProductionE2EYearOrchestratorDependencies( + year_state_repository=year_state, + source_adapters=adapters, + parser_boundaries=active_parsers, + validation_boundary=_FakeValidationBoundary(), + insert_repository=insert_repository or _FakeInsertRepository(), + ) diff --git a/tests/test_production_packaging_operator_runbook.py b/tests/test_production_packaging_operator_runbook.py new file mode 100644 index 00000000..84a78561 --- /dev/null +++ b/tests/test_production_packaging_operator_runbook.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RUNBOOK_PATH = REPOSITORY_ROOT / "docs" / "production-packaging-operator-runbook.md" + + +def test_production_packaging_operator_runbook_covers_operator_flow() -> None: + runbook = RUNBOOK_PATH.read_text(encoding="utf-8") + + for heading in ( + "## Install", + "## Configure", + "## Validate", + "## Run", + "## PostgreSQL Readiness", + "## Scheduling", + "## Stop And Rerun", + "## Troubleshooting", + "## Production Validation Checklist", + ): + assert heading in runbook + + assert "Local fixture/dry-run" in runbook + assert "Local PostgreSQL smoke" in runbook + assert "Production PostgreSQL" in runbook + + +def test_runbook_documents_python_and_dotnet_entrypoint_alignment() -> None: + runbook = RUNBOOK_PATH.read_text(encoding="utf-8") + normalized = " ".join(runbook.split()).lower() + + assert "carbonops-parser" in runbook + assert "carbonops-source-acquisition" in runbook + assert "src/dotnet/CarbonOps.Parser.sln" in runbook + assert ".net scheduled-worker baseline" in normalized + assert "ingestion_status=not_implemented" in normalized + assert "`run-once` command is still a safe" in runbook + assert "not-yet-implemented placeholder" in normalized + assert "Final Project Production-Ready Verdict" in runbook + assert "CARBONOPS_POSTGRESQL_PASSWORD" in runbook + assert "CARBONOPS_POSTGRESQL_DSN" in runbook + assert "avoid in production because it is easier to leak" in runbook + + +def test_runbook_safe_commands_do_not_require_production_secrets() -> None: + runbook = RUNBOOK_PATH.read_text(encoding="utf-8") + normalized = " ".join(runbook.split()) + + safe_command_markers = ( + "carbonops-source-acquisition validate", + "carbonops-parser local-dry-run", + "carbonops-parser validate-ingestion-config", + "python -m pytest", + "git diff --check", + ) + + for marker in safe_command_markers: + assert marker in runbook + + assert "without opening PostgreSQL" in runbook + assert "Do not put passwords, tokens, private DSNs, or real credentials" in normalized + assert "postgresql_password_configured=True" in runbook + + +def test_runbook_documents_production_commands_and_required_config() -> None: + runbook = RUNBOOK_PATH.read_text(encoding="utf-8") + + assert ( + "carbonops-parser run-ingestion \\\n" + " --config /etc/carbonops-parser/ingestion.production.json \\\n" + " --cycles 1" + ) in runbook + assert "CARBONOPS_POSTGRESQL_HOST" in runbook + assert "CARBONOPS_POSTGRESQL_PORT" in runbook + assert "CARBONOPS_POSTGRESQL_DATABASE" in runbook + assert "CARBONOPS_POSTGRESQL_USERNAME" in runbook + assert "CARBONOPS_POSTGRESQL_PASSWORD" in runbook + assert "CARBONOPS_POSTGRESQL_APPLICATION_NAME" in runbook + assert "CARBONOPS_POSTGRESQL_INITIAL_YEAR" in runbook + assert "`archive_root`" in runbook + assert "`enabled_source_families`" in runbook + assert "`source_years...artifact_url`" in runbook + + +def test_runbook_documents_postgresql_readiness_queries_and_cron() -> None: + runbook = RUNBOOK_PATH.read_text(encoding="utf-8") + normalized = " ".join(runbook.split()) + + for table_name in ( + "source_family_year_states", + "ghg_emission_factor_masters", + "ghg_emission_factor_details", + "defra_emission_factor_masters", + "defra_emission_factor_details", + "ipcc_emission_factor_masters", + "ipcc_emission_factor_details", + ): + assert table_name in runbook + + assert "CREATE TABLE IF NOT EXISTS" in normalized + assert "CREATE INDEX IF NOT EXISTS" in normalized + assert "SELECT source_family, max(ingested_year)" in runbook + assert "Supported production scheduling is cron" in runbook + assert "CARBONOPS_POSTGRESQL_PASSWORD_FILE" in runbook diff --git a/tests/test_production_parity_contract.py b/tests/test_production_parity_contract.py new file mode 100644 index 00000000..b85e7e54 --- /dev/null +++ b/tests/test_production_parity_contract.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +CONTRACT_PATH = REPOSITORY_ROOT / "docs" / "production-parity-contract.md" + + +def test_production_parity_contract_defines_runtime_verdicts() -> None: + contract = CONTRACT_PATH.read_text(encoding="utf-8") + normalized = " ".join(contract.split()) + + assert "Project-level production-ready: yes" in contract + assert "Python runtime production path: yes" in contract + assert ".NET runtime parity path: yes" in normalized + assert "The .NET `run-once` command remains a fail-closed placeholder" in contract + + +def test_production_parity_contract_covers_required_behavior() -> None: + contract = CONTRACT_PATH.read_text(encoding="utf-8") + + for marker in ( + "same PostgreSQL schema/model", + "GHG Protocol", + "DEFRA/DESNZ", + "IPCC EFDB", + "The default initial year is `2024`", + "latest_successful_imported_year + 1", + "`no_available_source_year`", + "Idempotency And Reruns", + "Redaction And Secret Handling", + "Operator Expectations", + "Production Validation Expectations", + ): + assert marker in contract + + +def test_production_parity_contract_lists_dotnet_follow_up_sequence() -> None: + contract = CONTRACT_PATH.read_text(encoding="utf-8") + + for marker in ( + ".NET service/scheduled-worker entrypoint", + ".NET production config loader and redaction", + ".NET PostgreSQL schema bootstrap and year-state", + ".NET source discovery/download/parsing orchestration", + ".NET source-specific master/detail insert", + ".NET idempotency and rerun behavior", + ".NET Docker PostgreSQL E2E tests", + "Python/.NET parity validation", + "Final project production-ready verdict", + "Final Project Production-Ready Verdict", + ): + assert marker in contract diff --git a/tests/test_production_rc_verification.py b/tests/test_production_rc_verification.py new file mode 100644 index 00000000..cdfbbfb6 --- /dev/null +++ b/tests/test_production_rc_verification.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPOSITORY_ROOT / "scripts" / "production_rc_verification.py" + +spec = importlib.util.spec_from_file_location("production_rc_verification", SCRIPT_PATH) +assert spec is not None +production_rc_verification = importlib.util.module_from_spec(spec) +assert spec.loader is not None +sys.modules[spec.name] = production_rc_verification +spec.loader.exec_module(production_rc_verification) + + +def test_production_rc_default_path_passes_without_live_or_database_modes() -> None: + report = production_rc_verification.build_production_rc_verification_report( + root=REPOSITORY_ROOT, + python_bin=sys.executable, + env={}, + ) + + assert report.passed + assert report.mode == "dry-run" + assert report.destructive_operations_enabled is False + assert report.live_source_calls_enabled is False + assert report.database_connections_enabled is False + assert [check.name for check in report.checks] == [ + "production config validation", + "schema bootstrap readiness", + "service entrypoint availability", + "orchestrator dry-run behavior", + "diagnostics redaction", + "CI release gate status", + ] + + +def test_production_rc_blocks_integration_mode_without_explicit_opt_in() -> None: + report = production_rc_verification.build_production_rc_verification_report( + root=REPOSITORY_ROOT, + python_bin=sys.executable, + mode="integration", + env={}, + ) + + assert not report.passed + assert report.checks[0].name == "integration mode guardrail" + assert report.checks[0].status == "failed" + assert "explicit opt-in" in report.checks[0].message + + +def test_production_rc_blocks_live_mode_without_explicit_opt_in() -> None: + report = production_rc_verification.build_production_rc_verification_report( + root=REPOSITORY_ROOT, + python_bin=sys.executable, + mode="live", + env={}, + ) + + assert not report.passed + assert report.checks[0].name == "live mode guardrail" + assert report.checks[0].status == "failed" + assert "default non-destructive dry-run path" in report.checks[0].message + + +def test_production_rc_report_does_not_render_secret_values() -> None: + report = production_rc_verification.build_production_rc_verification_report( + root=REPOSITORY_ROOT, + python_bin=sys.executable, + env={}, + ) + rendered = production_rc_verification.render_report( + report, + output_format="json", + ) + + assert "external-secret-present" not in rendered + assert "secret-db" not in rendered + assert "secret_database" not in rendered + assert "secret_user" not in rendered + assert "raw-secret" not in rendered + assert "raw-token" not in rendered + + +def test_production_rc_main_returns_nonzero_for_unapproved_live_mode(capsys) -> None: + exit_code = production_rc_verification.main(["--mode", "live"]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "[failed] live mode guardrail" in captured.out diff --git a/tests/test_release_validation_gate.py b/tests/test_release_validation_gate.py new file mode 100644 index 00000000..8d89c2eb --- /dev/null +++ b/tests/test_release_validation_gate.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPOSITORY_ROOT / "scripts" / "release_validation_gate.py" + +spec = importlib.util.spec_from_file_location("release_validation_gate", SCRIPT_PATH) +assert spec is not None +release_validation_gate = importlib.util.module_from_spec(spec) +assert spec.loader is not None +sys.modules[spec.name] = release_validation_gate +spec.loader.exec_module(release_validation_gate) + + +def test_default_release_gate_commands_are_local_only() -> None: + commands = release_validation_gate.build_default_commands("python") + rendered_commands = [" ".join(command.args) for command in commands] + + assert all(command.local_only for command in commands) + python_test_commands = [ + command for command in commands if command.args[:3] == ("python", "-m", "pytest") + ] + assert len(python_test_commands) == 1 + assert python_test_commands[0].args != ("python", "-m", "pytest") + assert all( + target in python_test_commands[0].args + for target in release_validation_gate.RELEASE_GATE_PYTHON_TEST_TARGETS + ) + assert not any( + "scripts/check_public_safety.py" in command for command in rendered_commands + ) + assert any("carbonfactor_parser.source_acquisition.cli validate" in command for command in rendered_commands) + assert any("carbonfactor_parser.source_acquisition.cli run --dry-run" in command for command in rendered_commands) + assert any("carbonfactor_parser.cli local-dry-run" in command for command in rendered_commands) + dotnet_commands = [ + command for command in commands if command.args[:2] == ("dotnet", "test") + ] + assert len(dotnet_commands) == 1 + dotnet_command = dotnet_commands[0] + assert "src/dotnet/CarbonOps.Parser.sln" not in dotnet_command.args + assert release_validation_gate.RELEASE_GATE_DOTNET_TEST_PROJECT in dotnet_command.args + assert "--filter" in dotnet_command.args + assert ( + release_validation_gate.RELEASE_GATE_DOTNET_TEST_FILTER + in dotnet_command.args + ) + assert "ProductionConfigBoundaryTests" in release_validation_gate.RELEASE_GATE_DOTNET_TEST_FILTER + assert "Phase1OperationalDiagnosticsTests" in release_validation_gate.RELEASE_GATE_DOTNET_TEST_FILTER + assert "PostgreSQLRuntimeConfigGateContractTests" in release_validation_gate.RELEASE_GATE_DOTNET_TEST_FILTER + assert "CarbonOpsParserServiceCommandTests" in release_validation_gate.RELEASE_GATE_DOTNET_TEST_FILTER + assert not any( + "dotnet test src/dotnet/CarbonOps.Parser.sln" in command + for command in rendered_commands + ) + + findings = release_validation_gate.validate_default_commands(commands) + + assert findings == [] + + +def test_release_gate_static_checks_pass_for_repository() -> None: + commands = release_validation_gate.build_default_commands("python") + + findings = release_validation_gate.static_gate_checks(commands) + + assert findings == [] + + +def test_release_validation_workflow_installs_pytest_before_gate() -> None: + workflow_text = ( + REPOSITORY_ROOT / ".github" / "workflows" / "release-validation.yml" + ).read_text(encoding="utf-8") + + pytest_install_index = workflow_text.index("python -m pip install pytest") + release_gate_index = workflow_text.index("python scripts/release_validation_gate.py") + + assert pytest_install_index < release_gate_index + + +def test_integration_validation_is_skipped_without_explicit_opt_in() -> None: + findings = release_validation_gate.validate_integration_environment({}) + + assert findings == [] + assert release_validation_gate.integration_enabled({}) is False + + +def test_integration_validation_requires_runner_controls() -> None: + findings = release_validation_gate.validate_integration_environment( + {"CARBONOPS_RELEASE_GATE_RUN_INTEGRATION": "1"} + ) + + assert [finding.name for finding in findings] == [ + "integration opt-in", + "integration opt-in", + ] + assert "CARBONOPS_RUN_POSTGRESQL_INTEGRATION=1" in findings[0].message + assert "CARBONOPS_POSTGRESQL_TEST_DSN" in findings[1].message + + +def test_sample_config_validation_rejects_raw_connection_strings(tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + (config_dir / "carbonops.config.example.yaml").write_text( + "\n".join( + ( + "database:", + " provider: postgres", + ' host: "${CARBONOPS_PARSER_POSTGRES_HOST}"', + " port: 5432", + ' database: "${CARBONOPS_PARSER_POSTGRES_DATABASE}"', + ' username: "${CARBONOPS_PARSER_POSTGRES_USERNAME}"', + " passwordEnvVar: CARBONOPS_PARSER_POSTGRES_PASSWORD", + " connectionString: postgresql://user:secret@db/carbonops", + ) + ), + encoding="utf-8", + ) + + findings = release_validation_gate.validate_sample_config(tmp_path) + + assert any("postgres(?:ql)?://" in finding.message for finding in findings) + + +def test_release_gate_redacts_sensitive_output() -> None: + output = release_validation_gate.sanitize_output( + "password=super-secret token:abc postgresql://user:secret@localhost/db" + ) + + assert "super-secret" not in output + assert "token:abc" not in output + assert "user:secret@" not in output + assert output.count("[REDACTED]") == 3 diff --git a/tests/test_run_ingestion_summary_cli.py b/tests/test_run_ingestion_summary_cli.py new file mode 100644 index 00000000..0c0178fa --- /dev/null +++ b/tests/test_run_ingestion_summary_cli.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from carbonfactor_parser import cli as parser_cli +from carbonfactor_parser.pipeline.configured_cycle_runner import ( + ConfiguredCycleResult, + ConfiguredCycleRunnerResult, + ConfiguredCycleRunnerStatus, +) +from carbonfactor_parser.pipeline import configured_cycle_runner +from carbonfactor_parser.pipeline.production_e2e_year_orchestrator import ( + ProductionE2EFailureDetail, + ProductionE2EYearFamilyResult, + ProductionE2EYearFamilyStatus, + ProductionE2EYearOrchestratorRequest, + ProductionE2EYearOrchestratorResult, + ProductionE2EYearRunStatus, + ProductionE2EYearRunSummary, + ProductionE2EYearSelectionStatus, + ProductionE2EYearState, +) + + +def test_run_ingestion_summary_output_writes_sanitized_json( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + summary_path = tmp_path / "nested" / "summary.json" + result = _runner_result_with_secret_issue() + + monkeypatch.setattr( + configured_cycle_runner, + "load_configured_cycle_runner_config", + lambda path, *, max_cycles=None: {"path": path, "max_cycles": max_cycles}, + ) + monkeypatch.setattr( + configured_cycle_runner, + "run_configured_cycle_runner", + lambda settings: result, + ) + + exit_code = parser_cli.main( + ["run-ingestion", "--cycles", "1", "--summary-output", str(summary_path)] + ) + + captured = capsys.readouterr() + assert exit_code == 0 + assert captured.out == "" + payload = json.loads(summary_path.read_text(encoding="utf-8")) + assert payload["status"] == "completed" + assert payload["cycles"][0]["run_id"] == "run-123" + assert payload["cycles"][0]["issues"][0]["code"] == "DOWNLOAD_FAILED" + assert payload["cycles"][0]["history_persistence_status"] is None + assert payload["cycles"][0]["history_persistence_issue_count"] == 0 + serialized = json.dumps(payload) + assert "secret" not in serialized + assert "postgresql://user:secret" not in serialized + assert "token=abc" not in serialized + assert "password=***" in serialized + + +def test_run_ingestion_default_behavior_does_not_write_summary( + tmp_path: Path, + monkeypatch, +) -> None: + result = _runner_result_with_secret_issue() + + monkeypatch.setattr( + configured_cycle_runner, + "load_configured_cycle_runner_config", + lambda path, *, max_cycles=None: {"path": path, "max_cycles": max_cycles}, + ) + monkeypatch.setattr( + configured_cycle_runner, + "run_configured_cycle_runner", + lambda settings: result, + ) + + exit_code = parser_cli.main(["run-ingestion", "--cycles", "1"]) + + assert exit_code == 0 + assert not list(tmp_path.glob("*.json")) + + +def _runner_result_with_secret_issue() -> ConfiguredCycleRunnerResult: + failure = ProductionE2EFailureDetail( + source_family="ghg_protocol", + stage="download", + code="DOWNLOAD_FAILED", + message=( + "download failed dsn=postgresql://user:secret@example.invalid/db " + "password=secret token=abc" + ), + ) + family = ProductionE2EYearFamilyResult( + source_family="ghg_protocol", + status=ProductionE2EYearFamilyStatus.FAILED, + year_state=ProductionE2EYearState( + source_family="ghg_protocol", + year_state_key="ghg", + latest_year=None, + target_year=2024, + initial_year=2024, + selection_status=ProductionE2EYearSelectionStatus.INITIAL_YEAR_SELECTED, + ), + failures=(failure,), + ) + cycle = ConfiguredCycleResult( + cycle_number=1, + run_id="run-123", + result=ProductionE2EYearOrchestratorResult( + status=ProductionE2EYearRunStatus.COMPLETED, + request=ProductionE2EYearOrchestratorRequest(run_id="run-123"), + selected_source_families=("ghg_protocol",), + family_results=(family,), + summary=ProductionE2EYearRunSummary( + requested_family_count=1, + completed_family_count=1, + no_available_source_year_count=0, + failed_family_count=0, + parsed_row_count=0, + attempted_insert_count=0, + inserted_count=0, + skipped_duplicate_count=0, + failed_insert_count=0, + failure_count=1, + ), + ), + ) + return ConfiguredCycleRunnerResult( + status=ConfiguredCycleRunnerStatus.COMPLETED, + cycles=(cycle,), + schema_created_table_names=(), + schema_missing_table_names=(), + ) diff --git a/tests/test_source_acquisition_contract_public_api.py b/tests/test_source_acquisition_contract_public_api.py index fd2c86f0..e6a23236 100644 --- a/tests/test_source_acquisition_contract_public_api.py +++ b/tests/test_source_acquisition_contract_public_api.py @@ -61,7 +61,20 @@ "IPCCSourceDiscoveryStatus", "IPCCSourceDiscoveryValidationResult", "IPCCSourceDocumentCandidate", + "IPCCSourceDownloadExecutionIssue", + "IPCCSourceDownloadExecutionRequest", + "IPCCSourceDownloadExecutionResult", + "IPCCSourceDownloadExecutionStatus", + "IPCCSourceDownloadExecutionValidationResult", + "IPCCSourceDownloadTransport", + "IPCCSourceDownloadTransportResponse", + "IPCCSourceDownloadedArtifact", "SourceAcquisitionRunIssue", + "SourceAcquisitionRunRepository", + "SourceAcquisitionRunRepositoryIssue", + "SourceAcquisitionRunRepositoryPersistResult", + "SourceAcquisitionRunRepositoryPersistStatus", + "SourceAcquisitionRunRepositoryValidationResult", "SourceAcquisitionRunRequest", "SourceAcquisitionRunResult", "SourceAcquisitionRunStatus", @@ -98,9 +111,11 @@ "create_ghg_source_download_execution_request", "create_ipcc_source_discovery_request", "create_ipcc_source_discovery_result", + "create_ipcc_source_download_execution_request", "create_source_artifact_parser_input_bridge_entry", "create_source_acquisition_run_request", "create_source_acquisition_run_result", + "create_source_acquisition_run_repository_persist_result", "create_source_download_artifact_from_candidate", "validate_acquisition_to_parser_plan", "validate_acquisition_to_parser_plans", @@ -125,10 +140,14 @@ "validate_ipcc_source_discovery_request", "validate_ipcc_source_discovery_result", "validate_ipcc_source_document_candidate", + "execute_ipcc_source_download", + "validate_ipcc_source_download_execution_request", + "validate_ipcc_source_download_execution_result", "validate_source_artifact_parser_input_bridge_entry", "validate_source_artifact_parser_input_bridge_result", "validate_source_acquisition_run_request", "validate_source_acquisition_run_result", + "validate_source_acquisition_run_repository_inputs", "validate_source_discovery_candidate", "validate_source_discovery_candidate_result", "validate_source_download_artifact", @@ -257,6 +276,19 @@ def test_public_source_acquisition_contract_api_exports_work_together() -> None: assert ipcc_discovery.status is contract_api.IPCCSourceDiscoveryStatus.DECLARED assert ipcc_discovery.candidate_count == 1 assert contract_api.validate_ipcc_source_discovery_result(ipcc_discovery).is_valid + ipcc_download_request = ( + contract_api.create_ipcc_source_download_execution_request( + ipcc_discovery.candidates[0], + target_root="/tmp/carbonops-ipcc", + target_relative_path="ipcc/source.discovery", + ) + ) + assert ( + contract_api.validate_ipcc_source_download_execution_request( + ipcc_download_request, + ).is_valid + is False + ) def test_source_acquisition_package_import_is_lazy_and_runtime_passive() -> None: @@ -331,6 +363,7 @@ def test_source_acquisition_contract_api_does_not_export_internal_module_names() assert "ghg_source_discovery_boundary" not in contract_api.__all__ assert "ghg_source_download_execution_boundary" not in contract_api.__all__ assert "ipcc_source_discovery_boundary" not in contract_api.__all__ + assert "ipcc_source_download_execution_boundary" not in contract_api.__all__ assert all(not name.startswith("_") for name in contract_api.__all__) diff --git a/tests/test_source_acquisition_run_repository_contract.py b/tests/test_source_acquisition_run_repository_contract.py new file mode 100644 index 00000000..c83f4309 --- /dev/null +++ b/tests/test_source_acquisition_run_repository_contract.py @@ -0,0 +1,74 @@ +"""Tests for source acquisition run repository contract metadata helpers.""" + +from __future__ import annotations + +from carbonfactor_parser.source_acquisition.contract_api import ( + SourceAcquisitionRunRepository, + SourceAcquisitionRunRepositoryPersistStatus, + create_source_acquisition_run_repository_persist_result, + create_source_acquisition_run_request, + create_source_acquisition_run_result, + validate_source_acquisition_run_repository_inputs, +) + + +class _InMemoryRepository: + @property + def provider_name(self) -> str: + return "in_memory" + + def persist_runs(self, runs): + return create_source_acquisition_run_repository_persist_result( + provider_name=self.provider_name, + runs=runs, + ) + + +def _sample_run_result(): + request = create_source_acquisition_run_request(source_key="defra_desnz") + return create_source_acquisition_run_result(request) + + +def test_source_acquisition_run_repository_protocol_shape() -> None: + repository: SourceAcquisitionRunRepository = _InMemoryRepository() + + result = repository.persist_runs((_sample_run_result(),)) + + assert isinstance(repository, SourceAcquisitionRunRepository) + assert repository.provider_name == "in_memory" + assert result.status is SourceAcquisitionRunRepositoryPersistStatus.DECLARED + assert result.persisted_count == 1 + assert result.issues == () + + +def test_source_acquisition_run_repository_validation_requires_provider_name() -> None: + validation = validate_source_acquisition_run_repository_inputs( + provider_name="", + runs=(_sample_run_result(),), + ) + + assert validation.is_valid is False + assert validation.issues[0].code == ( + "SOURCE_ACQUISITION_RUN_REPOSITORY_MISSING_PROVIDER_NAME" + ) + + +def test_source_acquisition_run_repository_validation_requires_run_instances() -> None: + validation = validate_source_acquisition_run_repository_inputs( + provider_name="in_memory", + runs=(object(),), + ) + + assert validation.is_valid is False + assert validation.issues[0].code == "SOURCE_ACQUISITION_RUN_REPOSITORY_INVALID_RUN" + + +def test_source_acquisition_run_repository_persist_result_reports_validation_failure() -> None: + result = create_source_acquisition_run_repository_persist_result( + provider_name="", + runs=(object(),), + ) + + assert result.status is SourceAcquisitionRunRepositoryPersistStatus.FAILED_VALIDATION + assert result.persisted_count == 0 + assert len(result.issues) == 2 diff --git a/tests/test_source_document_repository_contract.py b/tests/test_source_document_repository_contract.py new file mode 100644 index 00000000..7b36e1b5 --- /dev/null +++ b/tests/test_source_document_repository_contract.py @@ -0,0 +1,124 @@ +"""Tests for source document repository contract metadata helpers.""" + +from __future__ import annotations + +import inspect + +from carbonfactor_parser.persistence.source_document_mapping import ( + create_source_document_persistence_mapping, +) +from carbonfactor_parser.persistence.source_document_repository import ( + SourceDocumentRepository, + SourceDocumentRepositoryIssue, + SourceDocumentRepositoryPersistStatus, + create_source_document_repository_persist_result, + validate_source_document_repository_inputs, +) +from carbonfactor_parser.persistence import ( + source_document_repository as source_document_repository_module, +) + + +class _InMemorySourceDocumentRepository: + @property + def provider_name(self) -> str: + return "in_memory" + + def persist_source_documents(self, records): + return create_source_document_repository_persist_result( + provider_name=self.provider_name, + records=records, + ) + + +def _sample_records(): + return create_source_document_persistence_mapping().records + + +def test_source_document_repository_protocol_shape() -> None: + repository: SourceDocumentRepository = _InMemorySourceDocumentRepository() + + result = repository.persist_source_documents(_sample_records()) + + assert isinstance(repository, SourceDocumentRepository) + assert repository.provider_name == "in_memory" + assert result.status is SourceDocumentRepositoryPersistStatus.DECLARED + assert result.persisted_count == 3 + assert result.issues == () + + +def test_source_document_repository_validation_requires_provider_name() -> None: + validation = validate_source_document_repository_inputs( + provider_name="", + records=_sample_records(), + ) + + assert validation.is_valid is False + assert validation.issues[0].code == ( + "SOURCE_DOCUMENT_REPOSITORY_MISSING_PROVIDER_NAME" + ) + assert validation.issues[0].field_name == "provider_name" + + +def test_source_document_repository_validation_requires_record_instances() -> None: + validation = validate_source_document_repository_inputs( + provider_name="in_memory", + records=(object(),), + ) + + assert validation.is_valid is False + assert validation.issues[0].code == "SOURCE_DOCUMENT_REPOSITORY_INVALID_RECORD" + assert validation.issues[0].field_name == "records[0]" + + +def test_source_document_repository_persist_result_reports_validation_failure() -> None: + result = create_source_document_repository_persist_result( + provider_name="", + records=(object(),), + ) + + assert result.status is SourceDocumentRepositoryPersistStatus.FAILED_VALIDATION + assert result.persisted_count == 0 + assert len(result.issues) == 2 + + +def test_source_document_repository_persist_result_snapshots_issues() -> None: + issues = [ + SourceDocumentRepositoryIssue( + code="CUSTOM_SOURCE_DOCUMENT_REPOSITORY_WARNING", + message="custom issue", + field_name="records", + severity="warning", + ), + ] + + result = create_source_document_repository_persist_result( + provider_name="in_memory", + records=_sample_records(), + issues=issues, + ) + issues.clear() + + assert result.status is SourceDocumentRepositoryPersistStatus.FAILED_VALIDATION + assert result.persisted_count == 0 + assert len(result.issues) == 1 + assert result.issues[0].code == "CUSTOM_SOURCE_DOCUMENT_REPOSITORY_WARNING" + + +def test_source_document_repository_contract_remains_runtime_passive() -> None: + module_source = inspect.getsource(source_document_repository_module) + + blocked_terms = ( + "connect(", + "execute(", + "open(", + "CREATE TABLE", + "INSERT INTO", + "postgres", + "requests", + "httpx", + "urlopen", + ) + + for term in blocked_terms: + assert term not in module_source diff --git a/tests/test_source_family_repository_contract.py b/tests/test_source_family_repository_contract.py new file mode 100644 index 00000000..9148f610 --- /dev/null +++ b/tests/test_source_family_repository_contract.py @@ -0,0 +1,261 @@ +"""Tests for source-family master/detail repository contract helpers.""" + +from __future__ import annotations + +import inspect +from dataclasses import replace + +from carbonfactor_parser.persistence import source_family_repository as module +from carbonfactor_parser.persistence.postgresql_schema_catalog import SourceFamily +from carbonfactor_parser.persistence.source_family_repository import ( + SourceFamilyDetailRecord, + SourceFamilyMasterRecord, + SourceFamilyRepository, + SourceFamilyRepositoryIssue, + SourceFamilyRepositoryPersistStatus, + create_source_family_repository_persist_result, + source_family_repository_table_names, + validate_source_family_repository_inputs, +) + + +class _InMemorySourceFamilyRepository: + @property + def provider_name(self) -> str: + return "in_memory" + + def persist_source_family_records(self, master_records, detail_records): + return create_source_family_repository_persist_result( + provider_name=self.provider_name, + master_records=master_records, + detail_records=detail_records, + ) + + +def _sample_master_record( + *, + source_family: SourceFamily = SourceFamily.DEFRA, + source_family_master_id: str = "defra_master_001", +) -> SourceFamilyMasterRecord: + return SourceFamilyMasterRecord( + source_family=source_family, + source_family_master_id=source_family_master_id, + source_year=2025, + source_version="conversion-factors-2025", + source_release=None, + source_document_id="source_document_001", + ingestion_run_id=None, + run_id="dry_run_001", + master_external_key="defra_2025_publication", + status="declared", + artifact_reference="artifact://defra/2025", + artifact_checksum_sha256="a" * 64, + archive_reference=None, + archive_checksum_sha256=None, + effective_from=None, + effective_to=None, + record_checksum_sha256="checksum_master_001", + metadata={}, + created_at="dry_run_timestamp_unavailable", + updated_at="dry_run_timestamp_unavailable", + ) + + +def _sample_detail_record( + *, + source_family: SourceFamily = SourceFamily.DEFRA, + source_family_master_id: str = "defra_master_001", +) -> SourceFamilyDetailRecord: + return SourceFamilyDetailRecord( + source_family=source_family, + source_family_detail_id="defra_detail_001", + source_family_master_id=source_family_master_id, + detail_external_key="defra_row_001", + source_row_number=2, + factor_id="DEFRA-2025-ELEC", + factor_name="Electricity", + factor_value="1.25", + factor_unit="kgco2e", + status="declared", + record_checksum_sha256="checksum_detail_001", + raw_fields={}, + normalized_fields={}, + created_at="dry_run_timestamp_unavailable", + updated_at="dry_run_timestamp_unavailable", + ) + + +def test_source_family_repository_protocol_shape() -> None: + repository: SourceFamilyRepository = _InMemorySourceFamilyRepository() + + result = repository.persist_source_family_records( + (_sample_master_record(),), + (_sample_detail_record(),), + ) + + assert isinstance(repository, SourceFamilyRepository) + assert repository.provider_name == "in_memory" + assert result.status is SourceFamilyRepositoryPersistStatus.DECLARED + assert result.persisted_master_count == 1 + assert result.persisted_detail_count == 1 + assert result.issues == () + + +def test_source_family_repository_validation_requires_provider_name() -> None: + validation = validate_source_family_repository_inputs( + provider_name="", + master_records=(_sample_master_record(),), + detail_records=(_sample_detail_record(),), + ) + + assert validation.is_valid is False + assert validation.issues[0].code == ( + "SOURCE_FAMILY_REPOSITORY_MISSING_PROVIDER_NAME" + ) + assert validation.issues[0].field_name == "provider_name" + + +def test_source_family_repository_validation_requires_record_instances() -> None: + validation = validate_source_family_repository_inputs( + provider_name="in_memory", + master_records=(object(),), + detail_records=(object(),), + ) + + assert validation.is_valid is False + assert validation.issues[0].code == ( + "SOURCE_FAMILY_REPOSITORY_INVALID_MASTER_RECORD" + ) + assert validation.issues[1].code == ( + "SOURCE_FAMILY_REPOSITORY_INVALID_DETAIL_RECORD" + ) + + +def test_source_family_repository_validation_rejects_missing_required_fields() -> None: + validation = validate_source_family_repository_inputs( + provider_name="in_memory", + master_records=( + _sample_master_record(source_family_master_id=""), + ), + detail_records=(), + ) + + assert validation.is_valid is False + assert validation.issues[0].code == ( + "SOURCE_FAMILY_REPOSITORY_MISSING_REQUIRED_FIELD" + ) + assert validation.issues[0].field_name == ( + "master_records[0].source_family_master_id" + ) + + +def test_source_family_repository_validation_requires_detail_master_reference() -> None: + validation = validate_source_family_repository_inputs( + provider_name="in_memory", + master_records=(_sample_master_record(),), + detail_records=( + _sample_detail_record(source_family_master_id="missing_master"), + ), + ) + + assert validation.is_valid is False + assert validation.issues[0].code == ( + "SOURCE_FAMILY_REPOSITORY_DETAIL_MASTER_NOT_DECLARED" + ) + assert validation.issues[0].field_name == ( + "detail_records[0].source_family_master_id" + ) + + +def test_source_family_repository_validation_requires_master_source_metadata() -> None: + validation = validate_source_family_repository_inputs( + provider_name="in_memory", + master_records=( + _sample_master_record(source_family_master_id="master_001"), + ), + detail_records=(), + ) + + assert validation.is_valid is True + + invalid_master = replace( + _sample_master_record(source_family_master_id="master_001"), + source_year=True, # type: ignore[arg-type] + ) + invalid = validate_source_family_repository_inputs( + provider_name="in_memory", + master_records=(invalid_master,), + detail_records=(), + ) + + assert invalid.is_valid is False + assert invalid.issues[0].code == "SOURCE_FAMILY_REPOSITORY_INVALID_SOURCE_YEAR" + + +def test_source_family_repository_persist_result_reports_validation_failure() -> None: + result = create_source_family_repository_persist_result( + provider_name="", + master_records=(object(),), + detail_records=(object(),), + ) + + assert result.status is SourceFamilyRepositoryPersistStatus.FAILED_VALIDATION + assert result.persisted_master_count == 0 + assert result.persisted_detail_count == 0 + assert len(result.issues) == 3 + + +def test_source_family_repository_persist_result_snapshots_inputs_and_issues() -> None: + master_records = [_sample_master_record()] + detail_records = [_sample_detail_record()] + issues = [ + SourceFamilyRepositoryIssue( + code="CUSTOM_SOURCE_FAMILY_REPOSITORY_WARNING", + message="custom issue", + field_name="master_records", + severity="warning", + ), + ] + + result = create_source_family_repository_persist_result( + provider_name="in_memory", + master_records=master_records, + detail_records=detail_records, + issues=issues, + ) + master_records.clear() + detail_records.clear() + issues.clear() + + assert result.status is SourceFamilyRepositoryPersistStatus.FAILED_VALIDATION + assert result.persisted_master_count == 0 + assert result.persisted_detail_count == 0 + assert len(result.issues) == 1 + assert result.issues[0].code == "CUSTOM_SOURCE_FAMILY_REPOSITORY_WARNING" + + +def test_source_family_repository_exposes_catalog_table_names() -> None: + assert source_family_repository_table_names(SourceFamily.DEFRA) == ( + "defra_emission_factor_masters", + "defra_emission_factor_details", + ) + + +def test_source_family_repository_contract_remains_runtime_passive() -> None: + module_source = inspect.getsource(module) + + blocked_terms = ( + "connect(", + "execute(", + "open(", + "CREATE TABLE", + "INSERT INTO", + "psycopg", + "sqlalchemy", + "requests", + "httpx", + "urlopen", + ) + + for term in blocked_terms: + assert term not in module_source diff --git a/tests/test_source_onboarding_registry_contract.py b/tests/test_source_onboarding_registry_contract.py new file mode 100644 index 00000000..ba858d92 --- /dev/null +++ b/tests/test_source_onboarding_registry_contract.py @@ -0,0 +1,402 @@ +from __future__ import annotations + +from dataclasses import FrozenInstanceError, replace +import importlib +import json +from pathlib import Path +import sys + +import pytest + +from carbonfactor_parser.source_acquisition.source_onboarding_registry_contract import ( + PHASE2_ONBOARDING_PARSER_KEYS_BY_SOURCE_FAMILY, + PHASE2_ONBOARDING_SOURCE_FAMILIES, + SourceOnboardingDiscoveryStrategy, + SourceOnboardingDocument, + SourceOnboardingParserCapability, + SourceOnboardingRegistry, + SourceOnboardingRegistryEntry, + SourceOnboardingRuntimeSafety, + SourceOnboardingUpdateCadence, + SourceOnboardingValidationExpectations, + create_phase2_source_onboarding_registry, + get_source_onboarding_entry, + list_source_onboarding_entries, + validate_source_onboarding_registry, +) +from carbonfactor_parser.parsers.adapter_registry_contract import ( + create_phase1_parser_adapter_registry, +) + + +EXPECTED_PHASE1_SOURCE_FAMILIES = ( + "ghg_protocol", + "defra_desnz", + "ipcc_efdb", +) + +BANNED_RUNTIME_MODULE_PREFIXES = ( + "requests", + "psycopg", + "sqlalchemy", + "asyncpg", + "dotenv", + "boto3", + "httpx", + "urllib3", +) +PARITY_EXPECTATIONS_PATH = ( + Path(__file__).parent + / "fixtures/parity/source_onboarding_registry_expectations.json" +) + + +def test_phase2_source_onboarding_registry_contains_phase1_source_families() -> None: + registry = create_phase2_source_onboarding_registry() + + assert PHASE2_ONBOARDING_SOURCE_FAMILIES == EXPECTED_PHASE1_SOURCE_FAMILIES + assert tuple(entry.source_id for entry in registry.entries) == ( + EXPECTED_PHASE1_SOURCE_FAMILIES + ) + assert tuple(entry.source_family for entry in registry.entries) == ( + EXPECTED_PHASE1_SOURCE_FAMILIES + ) + + +def test_phase2_source_onboarding_registry_represents_future_metadata() -> None: + registry = create_phase2_source_onboarding_registry() + + for entry in registry.entries: + assert entry.discovery_strategy is ( + SourceOnboardingDiscoveryStrategy.DECLARED_REFERENCE + ) + assert entry.update_cadence is SourceOnboardingUpdateCadence.UNKNOWN + assert entry.parser_capability.parser_key == ( + PHASE2_ONBOARDING_PARSER_KEYS_BY_SOURCE_FAMILY[entry.source_family] + ) + assert entry.parser_capability.parser_source_format == "discovery_reference" + assert entry.validation_expectations.required_document_fields == ( + "document_id", + "display_name", + "source_reference", + "expected_format", + ) + assert entry.documents[0].source_reference.startswith("discovery://") + + +def test_phase2_source_onboarding_registry_matches_shared_parity_expectations() -> None: + expectations = json.loads(PARITY_EXPECTATIONS_PATH.read_text()) + registry = create_phase2_source_onboarding_registry() + + assert _registry_to_wire_snapshot(registry) == { + "phase2_source_families": expectations["phase2_source_families"], + "entries": expectations["entries"], + } + assert expectations["accepted_asymmetries"] == [ + "Python validation raises TypeError or ValueError for invalid registries; " + ".NET validation returns ContractValidationResult errors and lookup helpers " + "throw ArgumentException when an invalid custom registry is supplied." + ] + + +def test_phase2_source_onboarding_parser_keys_align_with_phase1_registry() -> None: + onboarding_registry = create_phase2_source_onboarding_registry() + parser_registry = create_phase1_parser_adapter_registry() + parser_keys_by_source_family = { + descriptor.source_family: descriptor.parser_key + for descriptor in parser_registry.descriptors + } + + assert parser_keys_by_source_family == PHASE2_ONBOARDING_PARSER_KEYS_BY_SOURCE_FAMILY + for entry in onboarding_registry.entries: + assert entry.parser_capability.parser_key == ( + parser_keys_by_source_family[entry.source_family] + ) + + +def test_phase2_source_onboarding_registry_is_runtime_safe_by_default() -> None: + registry = create_phase2_source_onboarding_registry() + + for entry in registry.entries: + assert entry.parser_capability.supports_parser_execution is False + assert entry.validation_expectations.checksum_required is False + assert entry.validation_expectations.schema_validation_required is False + assert entry.runtime_safety.allows_network_calls is False + assert entry.runtime_safety.allows_file_reads is False + assert entry.runtime_safety.allows_database_writes is False + assert entry.runtime_safety.requires_credentials is False + + +def test_source_onboarding_registry_valid_entry_passes_validation() -> None: + registry = SourceOnboardingRegistry(entries=(_valid_entry("new_registry_source"),)) + + assert validate_source_onboarding_registry(registry) == registry + + +def test_source_onboarding_registry_invalid_entry_type_raises() -> None: + registry = SourceOnboardingRegistry( + entries=("not an entry",), # type: ignore[arg-type] + ) + + with pytest.raises(TypeError, match="SourceOnboardingRegistryEntry"): + validate_source_onboarding_registry(registry) + + +def test_source_onboarding_registry_duplicate_source_ids_raise() -> None: + entry = _valid_entry("duplicate_registry_source") + registry = SourceOnboardingRegistry( + entries=( + entry, + replace(entry, source_family="duplicate_registry_source_two"), + ), + ) + + with pytest.raises(ValueError, match="Duplicate source_id found"): + validate_source_onboarding_registry(registry) + + +def test_source_onboarding_registry_duplicate_source_families_raise() -> None: + entry = _valid_entry("duplicate_registry_family") + registry = SourceOnboardingRegistry( + entries=( + entry, + replace(entry, source_id="duplicate_registry_family_two"), + ), + ) + + with pytest.raises(ValueError, match="Duplicate source_family found"): + validate_source_onboarding_registry(registry) + + +def test_source_onboarding_registry_duplicate_document_ids_raise() -> None: + entry = _valid_entry( + "new_registry_source", + documents=( + _valid_document("duplicate_document"), + _valid_document("duplicate_document"), + ), + ) + registry = SourceOnboardingRegistry(entries=(entry,)) + + with pytest.raises(ValueError, match="Duplicate document_id found"): + validate_source_onboarding_registry(registry) + + +def test_source_onboarding_registry_missing_required_fields_raise() -> None: + entry = replace(_valid_entry("new_registry_source"), source_family=" ") + registry = SourceOnboardingRegistry(entries=(entry,)) + + with pytest.raises(ValueError, match="source_family must be a non-empty string"): + validate_source_onboarding_registry(registry) + + +def test_source_onboarding_registry_missing_documents_raise() -> None: + entry = replace(_valid_entry("new_registry_source"), documents=()) + registry = SourceOnboardingRegistry(entries=(entry,)) + + with pytest.raises(ValueError, match="documents must include at least one"): + validate_source_onboarding_registry(registry) + + +def test_source_onboarding_registry_deterministic_ordering_is_enforced() -> None: + registry = create_phase2_source_onboarding_registry() + reordered = SourceOnboardingRegistry( + entries=(registry.entries[1], registry.entries[0], registry.entries[2]), + ) + mixed_source_ids = SourceOnboardingRegistry( + entries=( + _valid_entry("z_custom_source_id", source_family="ghg_protocol"), + _valid_entry("a_custom_source_id", source_family="custom_source_family"), + ), + ) + mixed_source_ids_reordered = SourceOnboardingRegistry( + entries=tuple(reversed(mixed_source_ids.entries)), + ) + + assert create_phase2_source_onboarding_registry() == registry + assert list_source_onboarding_entries(registry) == registry.entries + assert validate_source_onboarding_registry(mixed_source_ids) == mixed_source_ids + with pytest.raises(ValueError, match="Phase 1 source order"): + validate_source_onboarding_registry(reordered) + with pytest.raises(ValueError, match="Phase 1 source order"): + validate_source_onboarding_registry(mixed_source_ids_reordered) + + +def test_source_onboarding_registry_lookup_is_deterministic() -> None: + registry = create_phase2_source_onboarding_registry() + + for source_family in EXPECTED_PHASE1_SOURCE_FAMILIES: + entry = get_source_onboarding_entry(source_family, registry) + + assert entry is not None + assert entry == get_source_onboarding_entry(source_family) + assert entry.source_family == source_family + + assert get_source_onboarding_entry("unknown_source", registry) is None + + +def test_source_onboarding_registry_lookup_rejects_invalid_custom_registry() -> None: + entry = _valid_entry("duplicate_registry_source") + registry = SourceOnboardingRegistry( + entries=( + entry, + replace(entry, source_family="duplicate_registry_source_two"), + ), + ) + + with pytest.raises(ValueError, match="Duplicate source_id found"): + get_source_onboarding_entry("duplicate_registry_source", registry) + + +def test_source_onboarding_registry_records_are_immutable() -> None: + registry = create_phase2_source_onboarding_registry() + + with pytest.raises(FrozenInstanceError): + registry.entries = () # type: ignore[misc] + with pytest.raises(FrozenInstanceError): + registry.entries[0].source_id = "mutated" # type: ignore[misc] + + +def test_source_onboarding_registry_import_is_runtime_passive( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import builtins + import os + + module_name = ( + "carbonfactor_parser.source_acquisition.source_onboarding_registry_contract" + ) + sys.modules.pop(module_name, None) + + open_calls: list[tuple[object, ...]] = [] + getenv_calls: list[tuple[object, ...]] = [] + + def guard_open(*args: object, **kwargs: object) -> object: + open_calls.append(args) + raise AssertionError("source onboarding registry import read a file") + + def guard_getenv(*args: object, **kwargs: object) -> object: + getenv_calls.append(args) + raise AssertionError("source onboarding registry import read environment") + + monkeypatch.setattr(builtins, "open", guard_open) + monkeypatch.setattr(os, "getenv", guard_getenv) + monkeypatch.setattr(os, "environ", {}) + + imported_modules_before = set(sys.modules) + module = importlib.import_module(module_name) + imported_modules_after = set(sys.modules) + + assert hasattr(module, "create_phase2_source_onboarding_registry") + assert open_calls == [] + assert getenv_calls == [] + + newly_imported = imported_modules_after - imported_modules_before + assert not any( + module_name == prefix or module_name.startswith(f"{prefix}.") + for module_name in newly_imported + for prefix in BANNED_RUNTIME_MODULE_PREFIXES + ) + + +def _valid_entry( + source_id: str, + *, + source_family: str | None = None, + documents: tuple[SourceOnboardingDocument, ...] | None = None, +) -> SourceOnboardingRegistryEntry: + return SourceOnboardingRegistryEntry( + source_id=source_id, + source_family=source_family or source_id, + display_name="New Registry Source", + documents=documents or (_valid_document(f"{source_id}_document"),), + discovery_strategy=SourceOnboardingDiscoveryStrategy.SOURCE_SPECIFIC_DISCOVERY, + parser_capability=SourceOnboardingParserCapability( + parser_key=f"{source_id}_parser", + parser_source_format="discovery_reference", + supports_parser_execution=False, + capability_notes="metadata only", + ), + validation_expectations=SourceOnboardingValidationExpectations( + required_document_fields=("document_id", "source_reference"), + checksum_required=True, + schema_validation_required=True, + validation_notes="shape validation only", + ), + update_cadence=SourceOnboardingUpdateCadence.PERIODIC, + runtime_safety=SourceOnboardingRuntimeSafety( + allows_network_calls=False, + allows_file_reads=False, + allows_database_writes=False, + requires_credentials=False, + safety_notes="contract metadata only", + ), + ) + + +def _registry_to_wire_snapshot(registry: SourceOnboardingRegistry) -> dict[str, object]: + return { + "phase2_source_families": list(PHASE2_ONBOARDING_SOURCE_FAMILIES), + "entries": [ + { + "source_id": entry.source_id, + "source_family": entry.source_family, + "display_name": entry.display_name, + "documents": [ + { + "document_id": document.document_id, + "display_name": document.display_name, + "source_reference": document.source_reference, + "expected_format": document.expected_format, + "required": document.required, + } + for document in entry.documents + ], + "discovery_strategy": entry.discovery_strategy.value, + "parser_capability": { + "parser_key": entry.parser_capability.parser_key, + "parser_source_format": entry.parser_capability.parser_source_format, + "supports_parser_execution": ( + entry.parser_capability.supports_parser_execution + ), + "capability_notes": entry.parser_capability.capability_notes, + }, + "validation_expectations": { + "required_document_fields": list( + entry.validation_expectations.required_document_fields + ), + "checksum_required": ( + entry.validation_expectations.checksum_required + ), + "schema_validation_required": ( + entry.validation_expectations.schema_validation_required + ), + "validation_notes": ( + entry.validation_expectations.validation_notes + ), + }, + "update_cadence": entry.update_cadence.value, + "runtime_safety": { + "allows_network_calls": entry.runtime_safety.allows_network_calls, + "allows_file_reads": entry.runtime_safety.allows_file_reads, + "allows_database_writes": ( + entry.runtime_safety.allows_database_writes + ), + "requires_credentials": entry.runtime_safety.requires_credentials, + "safety_notes": entry.runtime_safety.safety_notes, + }, + "enabled": entry.enabled, + } + for entry in registry.entries + ], + } + + +def _valid_document(document_id: str) -> SourceOnboardingDocument: + return SourceOnboardingDocument( + document_id=document_id, + display_name="Declared document", + source_reference=f"discovery://{document_id}", + expected_format="discovery", + required=True, + )