diff --git a/.gitignore b/.gitignore index 8fad24a..6b90528 100644 --- a/.gitignore +++ b/.gitignore @@ -200,3 +200,4 @@ docs/reports/generated/*.md # Allow all files in sample_data to be tracked, even if normally ignored !/sample_data/ !/sample_data/** +.gstack/ diff --git a/README.md b/README.md index d362c23..8aec5c7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![CI](https://github.com///actions/workflows/ci.yml/badge.svg)](https://github.com///actions/workflows/ci.yml) -# Meshic Geospatial Data Pipeline +# Suhail Geospatial Data Pipeline A sophisticated **two-stage geospatial data processing pipeline** for the Saudi real estate market. The pipeline is now fully database-driven: all tiles to be processed are stored in the `tile_urls` table, supporting province-wide and all-Saudi scrapes with resumable, robust processing. @@ -102,29 +102,29 @@ cp .env.example .env #### Stage 1: Geometric Processing (DB-driven) ```bash # Seed tiles (optional; per province or all provinces) -meshic-pipeline seed-tiles --province riyadh --limit 1000 --stride 2 +suhail-pipeline seed-tiles --province riyadh --limit 1000 --stride 2 # Process tiles using the DB queue (recommended) -meshic-pipeline db-geometric --batch-size 1000 --concurrency 5 --adaptive +suhail-pipeline db-geometric --batch-size 1000 --concurrency 5 --adaptive # Or traditional bbox mode -meshic-pipeline geometric --bbox 46.428223 24.367114 47.010498 24.896402 +suhail-pipeline geometric --bbox 46.428223 24.367114 47.010498 24.896402 ``` #### Stage 2: Enrichment ```bash # Fast enrichment (new parcels that need data) -meshic-pipeline fast-enrich --batch-size 200 +suhail-pipeline fast-enrich --batch-size 200 # Incremental enrichment (stale by days) -meshic-pipeline incremental-enrich --days-old 30 --batch-size 100 +suhail-pipeline incremental-enrich --days-old 30 --batch-size 100 # Delta enrichment (only parcels with price changes) -meshic-pipeline delta-enrich --auto-geometric +suhail-pipeline delta-enrich --auto-geometric # Monitoring -meshic-pipeline monitor status -meshic-pipeline monitor recommend +suhail-pipeline monitor status +suhail-pipeline monitor recommend ``` ## 🔄 Enrichment Strategies @@ -136,7 +136,7 @@ The pipeline provides **multiple enrichment modes** to leverage this insight: ### **🎯 TRIGGER-BASED** (Maximum Efficiency) ```bash -meshic-pipeline enrich fast-enrich --batch-size 400 +suhail-pipeline enrich fast-enrich --batch-size 400 ``` - **🚀 Leverages your insight**: Only processes parcels with `transaction_price > 0` - **93.3% efficiency gain**: Skips 962,796 parcels that don't need enrichment @@ -145,7 +145,7 @@ meshic-pipeline enrich fast-enrich --batch-size 400 ### **🆕 NEW PARCELS** (Standard Approach) ```bash -meshic-pipeline enrich fast-enrich --batch-size 200 +suhail-pipeline enrich fast-enrich --batch-size 200 ``` - Processes parcels never enriched before (same as trigger-based but different implementation) - Perfect for initial runs or capturing new parcels @@ -154,10 +154,10 @@ meshic-pipeline enrich fast-enrich --batch-size 200 ### **🔄 INCREMENTAL UPDATES** (Weekly/Monthly) ```bash # Weekly updates (recommended) -meshic-pipeline enrich incremental-enrich --days-old 7 --batch-size 100 +suhail-pipeline enrich incremental-enrich --days-old 7 --batch-size 100 # Monthly updates -meshic-pipeline enrich incremental-enrich --days-old 30 --batch-size 100 +suhail-pipeline enrich incremental-enrich --days-old 30 --batch-size 100 ``` - **🎯 Captures new transactions on existing parcels** - Re-processes parcels not enriched recently @@ -165,7 +165,7 @@ meshic-pipeline enrich incremental-enrich --days-old 30 --batch-size 100 ### **🔥 FULL REFRESH** (Quarterly) ```bash -meshic-pipeline enrich full-refresh --batch-size 50 +suhail-pipeline enrich full-refresh --batch-size 50 ``` - Re-processes ALL enrichable parcels - Guarantees 100% data completeness @@ -174,13 +174,13 @@ meshic-pipeline enrich full-refresh --batch-size 50 ### **🎯 DELTA ENRICHMENT** (Revolutionary Precision) ```bash # Automatic workflow (recommended) -meshic-pipeline enrich delta-enrich --auto-geometric +suhail-pipeline enrich delta-enrich --auto-geometric # Manual workflow (if fresh MVT data already exists) -meshic-pipeline enrich delta-enrich +suhail-pipeline enrich delta-enrich # Testing with limits -meshic-pipeline enrich delta-enrich --limit 100 --auto-geometric +suhail-pipeline enrich delta-enrich --limit 100 --auto-geometric ``` - **🚀 MVT-based change detection**: Only enriches parcels with actual transaction price changes - **Perfect precision**: No false positives from time-based approaches @@ -197,19 +197,19 @@ meshic-pipeline enrich delta-enrich --limit 100 --auto-geometric ### Status Monitoring (CLI) ```bash # Queue status, enrichment coverage, failures -meshic-pipeline monitor status +suhail-pipeline monitor status # Automated recommendations -meshic-pipeline monitor recommend +suhail-pipeline monitor recommend # Scheduling guidance -meshic-pipeline monitor schedule-info +suhail-pipeline monitor schedule-info # Reset stale in_progress tiles (for cron) -meshic-pipeline monitor reset-stale -- --stale-minutes 60 +suhail-pipeline monitor reset-stale -- --stale-minutes 60 # Sample performance measurements (writes docs/reports) -meshic-pipeline monitor perf -- --label baseline --iterations 5 +suhail-pipeline monitor perf -- --label baseline --iterations 5 # Repair missing province metadata (tile URL/bbox) python scripts/util/backfill_province_metadata.py --province-id 21012 @@ -256,12 +256,12 @@ layers: [parcels, transactions, neighborhoods, ...] ### **Performance Tuning** ```bash # High-performance settings -meshic-pipeline enrich incremental-enrich \ +suhail-pipeline enrich incremental-enrich \ --batch-size 500 \ --days-old 7 # Memory-optimized settings -meshic-pipeline enrich incremental-enrich \ +suhail-pipeline enrich incremental-enrich \ --batch-size 100 \ --days-old 7 ``` @@ -294,16 +294,16 @@ The pipeline **guarantees** capture of new transactions through: ## 🚨 Important Notes ### **For New Deployments** -1. Seed provinces (optional): `meshic-pipeline seed-tiles --province riyadh` -2. Run geometric pipeline: `meshic-pipeline db-geometric` -3. Run initial enrichment: `meshic-pipeline fast-enrich` -4. Setup monitoring: `meshic-pipeline monitor status` +1. Seed provinces (optional): `suhail-pipeline seed-tiles --province riyadh` +2. Run geometric pipeline: `suhail-pipeline db-geometric` +3. Run initial enrichment: `suhail-pipeline fast-enrich` +4. Setup monitoring: `suhail-pipeline monitor status` ### **For Ongoing Operations** - **💡 LEVERAGE THE INSIGHT**: Use `fast-enrich` after geometric pipeline for maximum efficiency - **Never use only `fast-enrich`** for ongoing operations - it misses new transactions on existing parcels - **Use `incremental-enrich`** weekly to capture new transaction data -- **Monitor regularly** with `uv run meshic-pipeline monitor recommend` +- **Monitor regularly** with `uv run suhail-pipeline monitor recommend` ### **🚀 EFFICIENCY BREAKTHROUGH** Your insight reveals a **93.3% efficiency gain**: @@ -335,10 +335,10 @@ uv run pytest tests/integration check_db # Memory issues during processing -meshic-pipeline incremental-enrich --batch-size 50 +suhail-pipeline incremental-enrich --batch-size 50 # Check enrichment status -meshic-pipeline monitor status +suhail-pipeline monitor status ``` ### **Performance Optimization** @@ -377,7 +377,7 @@ Welcome! If you're picking up the pipeline/data debugging and remediation, here If you see `ModuleNotFoundError: No module named 'src'` when running enrichment via the CLI, use this workaround: ```bash -PYTHONPATH=$(pwd) python src/meshic_pipeline/run_enrichment_pipeline.py fast-enrich --limit 100 +PYTHONPATH=$(pwd) python src/suhail_pipeline/run_enrichment_pipeline.py fast-enrich --limit 100 ``` This is required because the CLI currently invokes the enrichment script as a subprocess, which does not set up the Python path correctly. This will be fixed in a future release. @@ -419,33 +419,33 @@ This does not affect current functionality but should be addressed in the future ### Core Commands -- `meshic-pipeline geometric [--bbox ...] [--recreate-db] [--save-as-temp ...]` +- `suhail-pipeline geometric [--bbox ...] [--recreate-db] [--save-as-temp ...]` - Run geometric pipeline (Stage 1) - Options: - `--bbox min_lon min_lat max_lon max_lat` — Bounding box for processing - `--recreate-db` — Drop and recreate the database schema - `--save-as-temp ` — Save parcels to a temporary table -- `meshic-pipeline fast-enrich [--batch-size ...] [--limit ...]` +- `suhail-pipeline fast-enrich [--batch-size ...] [--limit ...]` - Enrich new parcels with transaction prices - Options: - `--batch-size ` — Number of parcels per batch (default: 200) - `--limit ` — Limit parcels for testing -- `meshic-pipeline incremental-enrich [--batch-size ...] [--days-old ...] [--limit ...]` +- `suhail-pipeline incremental-enrich [--batch-size ...] [--days-old ...] [--limit ...]` - Enrich parcels not updated in X days - Options: - `--batch-size ` — Number of parcels per batch (default: 100) - `--days-old ` — Days old threshold (default: 30) - `--limit ` — Limit parcels for testing -- `meshic-pipeline full-refresh [--batch-size ...] [--limit ...]` +- `suhail-pipeline full-refresh [--batch-size ...] [--limit ...]` - Enrich ALL parcels (complete refresh) - Options: - `--batch-size ` — Number of parcels per batch (default: 50) - `--limit ` — Limit parcels for testing -- `meshic-pipeline delta-enrich [--batch-size ...] [--limit ...] [--fresh-table ...] [--auto-geometric] [--show-details/--no-details]` +- `suhail-pipeline delta-enrich [--batch-size ...] [--limit ...] [--fresh-table ...] [--auto-geometric] [--show-details/--no-details]` - Only process parcels with actual transaction price changes - Options: - `--batch-size ` — Number of parcels per batch (default: 200) @@ -456,13 +456,13 @@ This does not affect current functionality but should be addressed in the future ### Advanced/Composite Commands -- `meshic-pipeline smart-pipeline [--geometric-first] [--batch-size ...] [--bbox ...]` +- `suhail-pipeline smart-pipeline [--geometric-first] [--batch-size ...] [--bbox ...]` - Complete geometric + enrichment workflow (recommended for full runs) -- `meshic-pipeline monitor ` +- `suhail-pipeline monitor ` - Run enrichment monitoring commands -- `meshic-pipeline province-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` +- `suhail-pipeline province-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` - Geometric pipeline for a specific province - Options: - `province` — Province name (al_qassim, riyadh, madinah, asir, eastern, makkah) @@ -470,53 +470,53 @@ This does not affect current functionality but should be addressed in the future - `--recreate-db` — Drop and recreate the database schema - `--save-as-temp
` — Save parcels to a temporary table -- `meshic-pipeline saudi-arabia-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` +- `suhail-pipeline saudi-arabia-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` - Geometric pipeline for ALL Saudi provinces -- `meshic-pipeline discovery-summary` +- `suhail-pipeline discovery-summary` - Show province discovery capabilities/statistics -- `meshic-pipeline province-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` +- `suhail-pipeline province-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` - Complete province pipeline: geometric + enrichment for specific province -- `meshic-pipeline saudi-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` +- `suhail-pipeline saudi-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` - Complete Saudi Arabia pipeline: ALL provinces geometric + enrichment ### Usage Examples ```bash # Run geometric pipeline for a bounding box -meshic-pipeline geometric --bbox 46.428223 24.367114 47.010498 24.896402 +suhail-pipeline geometric --bbox 46.428223 24.367114 47.010498 24.896402 # Enrich new parcels (fast) -meshic-pipeline fast-enrich --batch-size 400 +suhail-pipeline fast-enrich --batch-size 400 # Incremental enrichment (parcels not updated in 7 days) -meshic-pipeline incremental-enrich --days-old 7 --batch-size 100 +suhail-pipeline incremental-enrich --days-old 7 --batch-size 100 # Full refresh (all enrichable parcels) -meshic-pipeline full-refresh --batch-size 50 +suhail-pipeline full-refresh --batch-size 50 # Delta enrichment (only parcels with price changes, auto-run geometric) -meshic-pipeline delta-enrich --auto-geometric +suhail-pipeline delta-enrich --auto-geometric # Province-wide geometric processing -meshic-pipeline province-geometric riyadh --strategy optimal +suhail-pipeline province-geometric riyadh --strategy optimal # All-province geometric processing -meshic-pipeline saudi-arabia-geometric --strategy efficient +suhail-pipeline saudi-arabia-geometric --strategy efficient # Complete province pipeline (geometric + enrichment) -meshic-pipeline province-pipeline riyadh --strategy optimal --batch-size 300 +suhail-pipeline province-pipeline riyadh --strategy optimal --batch-size 300 # Complete Saudi pipeline (all provinces) -meshic-pipeline saudi-pipeline --strategy efficient --batch-size 500 +suhail-pipeline saudi-pipeline --strategy efficient --batch-size 500 # Show discovery summary -meshic-pipeline discovery-summary +suhail-pipeline discovery-summary # Monitor enrichment status -meshic-pipeline monitor status +suhail-pipeline monitor status ``` ## 🧪 Running Tests diff --git a/REPOSITORY_INDEX.md b/REPOSITORY_INDEX.md index cf239f0..40c64e1 100644 --- a/REPOSITORY_INDEX.md +++ b/REPOSITORY_INDEX.md @@ -6,7 +6,7 @@ High-level map of the whole tree for navigation and onboarding. Pair with [docs/ | Path | Role | |------|------| -| [src/meshic_pipeline/](src/meshic_pipeline/) | Installable package: CLI (`cli.py`), geometric/enrichment runners, persistence, enrichment, geometry, decoder, discovery. | +| [src/suhail_pipeline/](src/suhail_pipeline/) | Installable package: CLI (`cli.py`), geometric/enrichment runners, persistence, enrichment, geometry, decoder, discovery. | | [alembic/](alembic/) | Database migrations (`env.py`, `versions/`, `versions_backup/`). | | [tests/](tests/) | `pytest` suites: `unit/`, `integration/`. | | [scripts/](scripts/) | Operational and utility scripts (`util/`, `db/`, reports). | @@ -15,7 +15,7 @@ High-level map of the whole tree for navigation and onboarding. Pair with [docs/ | Path | Role | |------|------| -| [pyproject.toml](pyproject.toml) | Project `meshic-pipeline`, dependencies, `[project.scripts]`, pytest config, `[dependency-groups].dev`. | +| [pyproject.toml](pyproject.toml) | Project `suhail-pipeline`, dependencies, `[project.scripts]`, pytest config, `[dependency-groups].dev`. | | [uv.lock](uv.lock) | Locked dependency graph for reproducible installs (`uv sync --frozen`). | | [.github/workflows/ci.yml](.github/workflows/ci.yml) | CI: `astral-sh/setup-uv`, `uv sync --all-groups --frozen`, `uv run pytest`. | @@ -57,5 +57,5 @@ High-level map of the whole tree for navigation and onboarding. Pair with [docs/ ## Entrypoints (from `pyproject.toml`) -- `meshic-pipeline` → `meshic_pipeline.cli:app` -- `check_db` → `meshic_pipeline.utils.db_checker:app` +- `suhail-pipeline` → `suhail_pipeline.cli:app` +- `check_db` → `suhail_pipeline.utils.db_checker:app` diff --git a/_bmad-output/project-context.md b/_bmad-output/project-context.md index 5a7a767..a12a3c2 100644 --- a/_bmad-output/project-context.md +++ b/_bmad-output/project-context.md @@ -9,12 +9,12 @@ sections_completed: # Project context for AI agents -Lean rules for implementing code in **meshic-pipeline**. Prefer existing patterns in `src/meshic_pipeline/`. Full narrative lives in `docs/` and `docs/docs-distillate/`. +Lean rules for implementing code in **suhail-pipeline**. Prefer existing patterns in `src/suhail_pipeline/`. Full narrative lives in `docs/` and `docs/docs-distillate/`. ## Technology stack and versions - **Language:** Python 3.9+ (`pyproject.toml`); **CI uses Python 3.11** — match CI when running tests locally. -- **Package:** `meshic-pipeline` 0.1.0, `src` layout, entrypoints `meshic-pipeline` and `check_db`. +- **Package:** `suhail-pipeline` 0.1.0, `src` layout, entrypoints `suhail-pipeline` and `check_db`. - **DB:** PostgreSQL + PostGIS; SQLAlchemy 2.x, GeoAlchemy2, Alembic; async via `asyncpg` where used. - **Geo / ETL:** GeoPandas, Shapely, h3/h3pandas, `mapbox-vector-tile`, mercantile, aiohttp. - **CLI / config:** Typer, Pydantic / pydantic-settings, YAML + `.env` for secrets. @@ -31,7 +31,7 @@ Lean rules for implementing code in **meshic-pipeline**. Prefer existing pattern ### Code organization -- **Package root:** `src/meshic_pipeline/` — new modules follow existing folders: `persistence/`, `enrichment/`, `geometry/`, `decoder/`, `downloader/`, `discovery/`. +- **Package root:** `src/suhail_pipeline/` — new modules follow existing folders: `persistence/`, `enrichment/`, `geometry/`, `decoder/`, `downloader/`, `discovery/`. - **CLI:** New commands go through `cli.py` with Typer; keep help strings and safe defaults consistent with documented commands in `docs/CLI_COMMAND_AUDIT.md`. - **Migrations:** Schema changes require Alembic revisions under `alembic/versions/`; do not hand-edit production without a migration. diff --git a/alembic/env.py b/alembic/env.py index 6511393..641b4d6 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -5,7 +5,7 @@ from sqlalchemy import engine_from_config from sqlalchemy import pool from alembic import context -from src.meshic_pipeline.persistence.models import Base +from src.suhail_pipeline.persistence.models import Base # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/alembic/versions/19c587b33197_add_critical_performance_indexes.py b/alembic/versions/19c587b33197_add_critical_performance_indexes.py index 2bfc940..84f7c83 100644 --- a/alembic/versions/19c587b33197_add_critical_performance_indexes.py +++ b/alembic/versions/19c587b33197_add_critical_performance_indexes.py @@ -1,6 +1,6 @@ """add_critical_performance_indexes -Critical database performance optimizations for the Meshic pipeline. +Critical database performance optimizations for the Suhail pipeline. Based on docs/archive/legacy-root-md/DATABASE_ARCHITECTURE_ANALYSIS.md recommendations. Provides 60-80% performance improvement through safe index additions. diff --git a/alembic/versions/c7f4a9d21b60_suhail_2026_source_expansion.py b/alembic/versions/c7f4a9d21b60_suhail_2026_source_expansion.py new file mode 100644 index 0000000..bce8307 --- /dev/null +++ b/alembic/versions/c7f4a9d21b60_suhail_2026_source_expansion.py @@ -0,0 +1,146 @@ +"""Suhail 2026 source expansion: inline market time-series, reshaped layers, new tables + +Adds the data the redesigned Suhail tiles / API now expose (see +docs/SUHAIL_SOURCE_AUDIT_2026-07.md): + +* parcels / parcels_centroids / neighborhoods / subdivisions: inline market + time-series (1w/1m/6m/12m transaction price, price-of-meter, transaction count + and date) plus zoning_group and a few identifiers. +* metro_stations / riyadh_bus_stations: station_long / station_lat. +* qi_population_metrics: reshaped metric fields (colour-coded) + region_id. +* qi_stripes: centroid_longitude / centroid_latitude. +* bus_lines: native tile fields (busroute/color/type/origin/originar). +* non_saudi_ownership_zones: new table (unique id PK). +* transactions: richer queryable columns (type, property/land-use, selling type, + source, areas, subdivision/neighborhood ids). + +All column adds are IF NOT EXISTS and additive; downgrade drops the added columns +and the new table. Existing data is preserved. + +Revision ID: c7f4a9d21b60 +Revises: b3e8a1c0d4f2 +Create Date: 2026-07-15 +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c7f4a9d21b60" +down_revision = "b3e8a1c0d4f2" +branch_labels = None +depends_on = None + + +# (window suffix) time-series columns shared by parcels/neighborhoods/subdivisions/centroids +_TS = [] +for _w in ("1w", "1m", "6m", "12m"): + _TS.append((f"transaction_price_{_w}", "DOUBLE PRECISION")) + _TS.append((f"price_of_meter_{_w}", "DOUBLE PRECISION")) + _TS.append((f"transactions_count_{_w}", "BIGINT")) + _TS.append((f"transaction_date_{_w}", "TIMESTAMP")) + + +def _add(table: str, columns): + for name, ddl_type in columns: + op.execute( + f'ALTER TABLE public."{table}" ADD COLUMN IF NOT EXISTS "{name}" {ddl_type}' + ) + + +def _drop(table: str, columns): + for name, _ in columns: + op.execute( + f'ALTER TABLE public."{table}" DROP COLUMN IF EXISTS "{name}"' + ) + + +_PARCELS_COLS = [("zoning_group", "TEXT")] + _TS +_CENTROIDS_COLS = [("transactions_count", "BIGINT")] + _TS +_NEIGHBORHOODS_COLS = [ + ("zoning_group", "TEXT"), + ("neighborhood_name", "TEXT"), +] + _TS +_SUBDIVISIONS_COLS = [ + ("subdivision_name_ar", "TEXT"), + ("neighborhood_id", "BIGINT"), + ("region_id", "BIGINT"), +] + _TS +_METRO_STATION_COLS = [("station_long", "DOUBLE PRECISION"), ("station_lat", "DOUBLE PRECISION")] +_BUS_STATION_COLS = [("station_long", "DOUBLE PRECISION"), ("station_lat", "DOUBLE PRECISION")] +_QI_POP_COLS = [ + ("region_id", "BIGINT"), + ("population_density", "TEXT"), + ("rent_apartment", "TEXT"), + ("rent_villa", "TEXT"), + ("rent_shop", "TEXT"), + ("rent_office", "TEXT"), + ("purchasing_power", "TEXT"), + ("weighted_median_income_monthly", "TEXT"), + ("poi_count", "TEXT"), +] +_QI_STRIPE_COLS = [("centroid_longitude", "DOUBLE PRECISION"), ("centroid_latitude", "DOUBLE PRECISION")] +_BUS_LINE_COLS = [ + ("busroute", "TEXT"), + ("color", "TEXT"), + ("type", "TEXT"), + ("origin", "TEXT"), + ("originar", "TEXT"), +] +_TRANSACTION_COLS = [ + ("transaction_type", "TEXT"), + ("property_type", "TEXT"), + ("metrics_type", "TEXT"), + ("land_use_group", "TEXT"), + ("land_use_detailed", "TEXT"), + ("selling_type", "TEXT"), + ("transaction_source", "TEXT"), + ("total_area", "DOUBLE PRECISION"), + ("subdivision_id", "BIGINT"), + ("neighborhood_id", "BIGINT"), + ("is_low_value_transaction", "BOOLEAN"), +] + + +def upgrade() -> None: + _add("parcels", _PARCELS_COLS) + _add("parcels_centroids", _CENTROIDS_COLS) + _add("neighborhoods", _NEIGHBORHOODS_COLS) + _add("subdivisions", _SUBDIVISIONS_COLS) + _add("metro_stations", _METRO_STATION_COLS) + _add("riyadh_bus_stations", _BUS_STATION_COLS) + _add("qi_population_metrics", _QI_POP_COLS) + _add("qi_stripes", _QI_STRIPE_COLS) + _add("bus_lines", _BUS_LINE_COLS) + _add("transactions", _TRANSACTION_COLS) + + # New layer: non_saudi_ownership_zones (unique id PK, clean upsert). + op.execute( + """ + CREATE TABLE IF NOT EXISTS public.non_saudi_ownership_zones ( + id BIGINT PRIMARY KEY, + geometry geometry(GEOMETRY, 4326), + name_ar TEXT, + name_en TEXT, + is_show BOOLEAN, + region_id BIGINT, + province_id BIGINT + ) + """ + ) + op.execute( + "CREATE INDEX IF NOT EXISTS idx_non_saudi_ownership_zones_geom " + "ON public.non_saudi_ownership_zones USING GIST (geometry)" + ) + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS public.non_saudi_ownership_zones CASCADE") + _drop("transactions", _TRANSACTION_COLS) + _drop("bus_lines", _BUS_LINE_COLS) + _drop("qi_stripes", _QI_STRIPE_COLS) + _drop("qi_population_metrics", _QI_POP_COLS) + _drop("riyadh_bus_stations", _BUS_STATION_COLS) + _drop("metro_stations", _METRO_STATION_COLS) + _drop("subdivisions", _SUBDIVISIONS_COLS) + _drop("neighborhoods", _NEIGHBORHOODS_COLS) + _drop("parcels_centroids", _CENTROIDS_COLS) + _drop("parcels", _PARCELS_COLS) diff --git a/alembic/versions/d8b1e6f42a90_add_dimensions_building_detection.py b/alembic/versions/d8b1e6f42a90_add_dimensions_building_detection.py new file mode 100644 index 0000000..dbba013 --- /dev/null +++ b/alembic/versions/d8b1e6f42a90_add_dimensions_building_detection.py @@ -0,0 +1,70 @@ +"""Add dimensions and building_detection tile layers + +- `dimensions`: per-parcel edge measurements (length_m, azimuth). Many rows per + parcel, no natural key -> surrogate row_id + `source_tile` for the tile-scoped + delete+append write. Indexed on source_tile so per-tile deletes stay cheap. +- `building_detection`: AI-detected building footprints/classification, year-stamped. + Keyless in the source -> deterministic synthetic bd_id primary key for upsert. + +Both are additive (new tables). See docs/SUHAIL_SOURCE_AUDIT_2026-07.md §5/§6. + +Revision ID: d8b1e6f42a90 +Revises: c7f4a9d21b60 +Create Date: 2026-07-15 +""" +from alembic import op + +revision = "d8b1e6f42a90" +down_revision = "c7f4a9d21b60" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + """ + CREATE TABLE IF NOT EXISTS public.dimensions ( + row_id BIGSERIAL PRIMARY KEY, + parcel_objectid BIGINT, + geometry geometry(POINT, 4326), + length_m DOUBLE PRECISION, + azimuth DOUBLE PRECISION, + province_id BIGINT, + source_tile TEXT + ) + """ + ) + op.execute( + "CREATE INDEX IF NOT EXISTS idx_dimensions_source_tile " + "ON public.dimensions (source_tile)" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS idx_dimensions_parcel " + "ON public.dimensions (parcel_objectid)" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS idx_dimensions_geom " + "ON public.dimensions USING GIST (geometry)" + ) + + op.execute( + """ + CREATE TABLE IF NOT EXISTS public.building_detection ( + bd_id BIGINT PRIMARY KEY, + geometry geometry(GEOMETRY, 4326), + class_pred TEXT, + prediction_year BIGINT, + region_id BIGINT, + source_tile TEXT + ) + """ + ) + op.execute( + "CREATE INDEX IF NOT EXISTS idx_building_detection_geom " + "ON public.building_detection USING GIST (geometry)" + ) + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS public.building_detection CASCADE") + op.execute("DROP TABLE IF EXISTS public.dimensions CASCADE") diff --git a/auto_restart_enrichment.sh b/auto_restart_enrichment.sh index ea2b1fc..a978d7d 100755 --- a/auto_restart_enrichment.sh +++ b/auto_restart_enrichment.sh @@ -18,7 +18,7 @@ while true; do if [[ "$PROCESS_STATUS" == "STOPPED" ]] && [[ "$REMAINING" -gt 0 ]]; then echo "[$TIMESTAMP] 🚨 Process stopped with $REMAINING parcels remaining - RESTARTING!" - nohup bash -lc "cd '$ROOT' && uv run meshic-pipeline full-refresh --batch-size 200" >> logs/enrichment-full.log 2>&1 & echo $! > logs/enrichment-full.pid + nohup bash -lc "cd '$ROOT' && uv run suhail-pipeline full-refresh --batch-size 200" >> logs/enrichment-full.log 2>&1 & echo $! > logs/enrichment-full.pid echo "[$TIMESTAMP] 🔄 Enrichment restarted with PID $(cat logs/enrichment-full.pid)" >> logs/auto_restart.log diff --git a/check_enrichment_status.sh b/check_enrichment_status.sh index 383cd0d..f113cb7 100755 --- a/check_enrichment_status.sh +++ b/check_enrichment_status.sh @@ -20,7 +20,7 @@ fi uv run python - <<'EOF' from sqlalchemy import create_engine, text -from meshic_pipeline.config import settings +from suhail_pipeline.config import settings import time engine = create_engine(str(settings.database_url)) diff --git a/docs/ACCEPTANCE_CRITERIA.md b/docs/ACCEPTANCE_CRITERIA.md index fc2dfd7..733129d 100644 --- a/docs/ACCEPTANCE_CRITERIA.md +++ b/docs/ACCEPTANCE_CRITERIA.md @@ -1,4 +1,4 @@ -# Acceptance Criteria: Meshic Geospatial Data Pipeline +# Acceptance Criteria: Suhail Geospatial Data Pipeline Date: 2025-10-16 Author: Mary (Business Analyst) @@ -29,9 +29,9 @@ Author: Mary (Business Analyst) ## Epic 2: Monitoring & Alerting - Monitoring CLI - - `meshic-pipeline monitor status` outputs tile queue counts by status, top errors, and age of oldest `in_progress`. - - `meshic-pipeline monitor recommend` outputs actionable scheduling guidance (which enrichment strategy to run next, with batch sizes). - - `meshic-pipeline monitor schedule-info` displays recommended cadence (daily/weekly/monthly) based on data freshness. + - `suhail-pipeline monitor status` outputs tile queue counts by status, top errors, and age of oldest `in_progress`. + - `suhail-pipeline monitor recommend` outputs actionable scheduling guidance (which enrichment strategy to run next, with batch sizes). + - `suhail-pipeline monitor schedule-info` displays recommended cadence (daily/weekly/monthly) based on data freshness. - Stale Reset - Scheduled job resets stale `in_progress` tiles after configurable threshold (e.g., 60 minutes) using built-in method. - Reset operations are logged with count of tiles affected. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e53d88c..68ac0fd 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,4 +1,4 @@ -# Architecture: Meshic Geospatial Data Pipeline (Lean) +# Architecture: Suhail Geospatial Data Pipeline (Lean) Date: 2025-10-16 Author: Mary (Business Analyst) @@ -8,7 +8,7 @@ Status: v0.1 (Lean Architecture for Planning) ## 1. System Context -Meshic is a two-stage, production-grade data pipeline for Saudi Arabian parcel intelligence: +Suhail is a two-stage, production-grade data pipeline for Saudi Arabian parcel intelligence: - Inputs - Province-specific Mapbox Vector Tiles (MVT) — geometry + attributes @@ -26,16 +26,16 @@ Meshic is a two-stage, production-grade data pipeline for Saudi Arabian parcel i ## 2. Core Components - CLI (Typer) - - `meshic-pipeline db-geometric`, `geometric`, `province-geometric`, `saudi-arabia-geometric` + - `suhail-pipeline db-geometric`, `geometric`, `province-geometric`, `saudi-arabia-geometric` - `fast-enrich`, `incremental-enrich`, `full-refresh`, `universal-metrics`, `delta-enrich` - `seed-tiles`, `discovery-summary`, `monitor ` - - File: `src/meshic_pipeline/cli.py` + - File: `src/suhail_pipeline/cli.py` - Discovery & Queue - `tile_urls` table stores z/x/y tile work items with statuses - Seeding via province bbox metadata (`provinces` table) - Model: `TileURL` with `claim_tiles_for_processing(..., SKIP LOCKED)` and `reset_stale_in_progress(...)` - - Files: `src/meshic_pipeline/persistence/models.py`, `src/meshic_pipeline/cli.py` + - Files: `src/suhail_pipeline/persistence/models.py`, `src/suhail_pipeline/cli.py` - Geometric Workers (Stage 1) - Downloader: `aiohttp` concurrent fetch with retry/backoff @@ -43,27 +43,27 @@ Meshic is a two-stage, production-grade data pipeline for Saudi Arabian parcel i - Validation: geometry sanity checks - Stitcher: PostGIS-based dissolve across tiles using temporary tables - Persister: Schema-driven writes, upsert support, temp table utilities - - Files: `src/meshic_pipeline/run_db_geometric.py`, `.../decoder/mvt_decoder.py`, `.../geometry/stitcher.py`, `.../persistence/postgis_persister.py`, `.../persistence/table_management.py` + - Files: `src/suhail_pipeline/run_db_geometric.py`, `.../decoder/mvt_decoder.py`, `.../geometry/stitcher.py`, `.../persistence/postgis_persister.py`, `.../persistence/table_management.py` - Enrichment Workers (Stage 2) - Strategies: fast, incremental, full, universal metrics, delta - API client: Suhail endpoints; batch processing; async persistence - - Files: `src/meshic_pipeline/run_enrichment_pipeline.py`, `.../enrichment/strategies.py`, `.../enrichment/api_client.py`, `.../persistence/enrichment_persister.py` + - Files: `src/suhail_pipeline/run_enrichment_pipeline.py`, `.../enrichment/strategies.py`, `.../enrichment/api_client.py`, `.../persistence/enrichment_persister.py` - Monitoring & Ops - CLI `monitor` subcommands; queue and enrichment visibility - Stale-tile reset via `TileURL.reset_stale_in_progress` - - Files: `src/meshic_pipeline/run_monitoring.py`, `src/meshic_pipeline/persistence/models.py` + - Files: `src/suhail_pipeline/run_monitoring.py`, `src/suhail_pipeline/persistence/models.py` - Configuration - Pydantic settings; DB URL, API endpoints, layers, batch sizes - Province metadata loader (from DB) and tile server templates - - File: `src/meshic_pipeline/config.py` + - File: `src/suhail_pipeline/config.py` - Schema & Migrations - SQLAlchemy models (15+ tables) - Alembic migrations incl. critical performance indexes and temp-table cleanup - - Files: `src/meshic_pipeline/persistence/models.py`, `alembic/versions/` + - Files: `src/suhail_pipeline/persistence/models.py`, `alembic/versions/` --- @@ -142,10 +142,10 @@ Indexes (via Alembic `19c587b33197...`): ## 7. Operations & Runbooks -- Seeding: `meshic-pipeline seed-tiles [--province|--region-slugs] [--limit|--stride]` -- Geometric run: `meshic-pipeline db-geometric --batch-size --concurrency --adaptive` +- Seeding: `suhail-pipeline seed-tiles [--province|--region-slugs] [--limit|--stride]` +- Geometric run: `suhail-pipeline db-geometric --batch-size --concurrency --adaptive` - Enrichment runs: `fast-enrich`, `incremental-enrich`, `full-refresh`, `universal-metrics`, `delta-enrich` -- Monitoring: `meshic-pipeline monitor status|recommend|schedule-info` +- Monitoring: `suhail-pipeline monitor status|recommend|schedule-info` - Stale reset: schedule `TileURL.reset_stale_in_progress(..., stale_minutes=60)` - Migrations: apply Alembic (indexes + hygiene) during low-traffic windows @@ -198,6 +198,6 @@ Temp Table Policy - docs/ACCEPTANCE_CRITERIA.md - docs/BROWNFIELD_PROJECT_DOCUMENTATION.md - alembic/versions/19c587b33197_add_critical_performance_indexes.py -- src/meshic_pipeline/cli.py -- src/meshic_pipeline/persistence/models.py +- src/suhail_pipeline/cli.py +- src/suhail_pipeline/persistence/models.py diff --git a/docs/BMAD_PROJECT_SCAN.md b/docs/BMAD_PROJECT_SCAN.md index 127280e..a1b628a 100644 --- a/docs/BMAD_PROJECT_SCAN.md +++ b/docs/BMAD_PROJECT_SCAN.md @@ -8,7 +8,7 @@ - **Type:** Python data platform — geospatial ETL + API enrichment (PostGIS, MVT, Typer CLI). - **Maturity:** Production-scale dataset documented in `BROWNFIELD_PROJECT_DOCUMENTATION.md` (millions of parcels and metrics). -- **Package:** `meshic-pipeline` (`pyproject.toml`), source layout under `src/meshic_pipeline/`. +- **Package:** `suhail-pipeline` (`pyproject.toml`), source layout under `src/suhail_pipeline/`. ## Authoritative documentation diff --git a/docs/BROWNFIELD_PROJECT_DOCUMENTATION.md b/docs/BROWNFIELD_PROJECT_DOCUMENTATION.md index 8709519..48619d5 100644 --- a/docs/BROWNFIELD_PROJECT_DOCUMENTATION.md +++ b/docs/BROWNFIELD_PROJECT_DOCUMENTATION.md @@ -1,7 +1,7 @@ -# Brownfield Project Documentation: Meshic Geospatial Data Pipeline +# Brownfield Project Documentation: Suhail Geospatial Data Pipeline **Document Type**: Comprehensive Brownfield Analysis -**Project**: Meshic Real Estate Data Processing Pipeline +**Project**: Suhail Real Estate Data Processing Pipeline **Date**: October 16, 2025 **Analyst**: Mary, Business Analyst **Status**: Production System with 2.16M+ Parcels @@ -10,7 +10,7 @@ ## Executive Summary -The Meshic pipeline is a **mature, production-scale geospatial data processing system** for Saudi Arabian real estate data. The system successfully processes 2.16M+ land parcels across Saudi Arabia with comprehensive enrichment data, demonstrating proven operational capability at commercial scale. +The Suhail pipeline is a **mature, production-scale geospatial data processing system** for Saudi Arabian real estate data. The system successfully processes 2.16M+ land parcels across Saudi Arabia with comprehensive enrichment data, demonstrating proven operational capability at commercial scale. ### Key Findings @@ -27,8 +27,8 @@ The Meshic pipeline is a **mature, production-scale geospatial data processing s ### 1.1 Database Reality Check -**Database Name**: `meshic` (NOT `meshic_pipeline` as documented) -**Owner**: `postgres` (NOT `meshic_user` as might be expected) +**Database Name**: `suhail` (NOT `suhail_pipeline` as documented) +**Owner**: `postgres` (NOT `suhail_user` as might be expected) #### Production Data Volumes (Actual Counts) @@ -58,7 +58,7 @@ in_progress: 788 tiles (2.3%) ```python # Actual Implementation from pyproject.toml Language: Python 3.9+ -Package Name: "meshic-pipeline" (v0.1.0) +Package Name: "suhail-pipeline" (v0.1.0) Database: PostgreSQL 14+ with PostGIS ORM: SQLAlchemy 2.0+ (async) HTTP Client: aiohttp (async) @@ -69,7 +69,7 @@ Spatial Processing: GeoPandas, Shapely, h3 #### Module Structure (Actual Source Code) ``` -src/meshic_pipeline/ +src/suhail_pipeline/ ├── cli.py # 18 commands (NOT 6-7 as simplified docs suggest) ├── config.py # Pydantic settings with province loading ├── persistence/ @@ -184,36 +184,36 @@ The system provides **18 commands** across 5 categories (not the simplified 6-7 #### Geometric Processing Commands ```bash -meshic-pipeline geometric [--bbox ...] [--recreate-db] [--save-as-temp
] -meshic-pipeline province-geometric [--strategy ...] [--recreate-db] -meshic-pipeline saudi-arabia-geometric [--strategy ...] [--recreate-db] -meshic-pipeline db-geometric [--batch-size] [--concurrency] [--adaptive] +suhail-pipeline geometric [--bbox ...] [--recreate-db] [--save-as-temp
] +suhail-pipeline province-geometric [--strategy ...] [--recreate-db] +suhail-pipeline saudi-arabia-geometric [--strategy ...] [--recreate-db] +suhail-pipeline db-geometric [--batch-size] [--concurrency] [--adaptive] ``` #### Enrichment Strategy Commands ```bash -meshic-pipeline fast-enrich [--batch-size 200] [--limit] -meshic-pipeline incremental-enrich [--batch-size 100] [--days-old 30] -meshic-pipeline full-refresh [--batch-size 50] [--limit] -meshic-pipeline universal-metrics [--batch-size 200] [--limit] -meshic-pipeline delta-enrich [--auto-geometric] [--fresh-table] [--show-details] +suhail-pipeline fast-enrich [--batch-size 200] [--limit] +suhail-pipeline incremental-enrich [--batch-size 100] [--days-old 30] +suhail-pipeline full-refresh [--batch-size 50] [--limit] +suhail-pipeline universal-metrics [--batch-size 200] [--limit] +suhail-pipeline delta-enrich [--auto-geometric] [--fresh-table] [--show-details] ``` #### Composite Workflow Commands ```bash -meshic-pipeline smart-pipeline [--geometric-first] [--batch-size] [--bbox] -meshic-pipeline province-pipeline [--strategy] [--batch-size] -meshic-pipeline saudi-pipeline [--strategy] [--batch-size] +suhail-pipeline smart-pipeline [--geometric-first] [--batch-size] [--bbox] +suhail-pipeline province-pipeline [--strategy] [--batch-size] +suhail-pipeline saudi-pipeline [--strategy] [--batch-size] ``` #### Orchestration & Monitoring ```bash -meshic-pipeline seed-tiles [--province] [--provinces] [--region-slugs] [--stride] [--limit] -meshic-pipeline discovery-summary -meshic-pipeline monitor +suhail-pipeline seed-tiles [--province] [--provinces] [--region-slugs] [--stride] [--limit] +suhail-pipeline discovery-summary +suhail-pipeline monitor ``` **Critical Observation**: The CLI is far more sophisticated than documented, with comprehensive province-wide and all-Saudi processing capabilities. @@ -479,7 +479,7 @@ DROP TABLE IF EXISTS temp_parcels, temp_neighborhoods, temp_subdivisions; **Priority 2: README Accuracy** - Update data coverage numbers (currently shows 1M parcels, should be 2.16M) -- Clarify which database name is actually used (`meshic` vs `meshic_pipeline`) +- Clarify which database name is actually used (`suhail` vs `suhail_pipeline`) - Document the 788 "in_progress" tiles situation ### 5.3 Code Quality Improvements @@ -677,7 +677,7 @@ DROP TABLE IF EXISTS temp_parcels, temp_neighborhoods, temp_subdivisions; ### The Bottom Line -The Meshic pipeline is a **production-grade geospatial data processing system** that has successfully achieved commercial scale with **2.16 million Saudi Arabian land parcels**. The system demonstrates: +The Suhail pipeline is a **production-grade geospatial data processing system** that has successfully achieved commercial scale with **2.16 million Saudi Arabian land parcels**. The system demonstrates: ✅ **Proven Architecture**: DB-driven tile orchestration, async processing, comprehensive enrichment ✅ **Production Data**: 76M+ price metrics, 130K+ building rules, 70K+ transactions diff --git a/docs/CLEAN_SLATE_PROTOCOL.md b/docs/CLEAN_SLATE_PROTOCOL.md index 5fcf966..cf2da6e 100644 --- a/docs/CLEAN_SLATE_PROTOCOL.md +++ b/docs/CLEAN_SLATE_PROTOCOL.md @@ -47,11 +47,11 @@ ``` - [ ] **8. Create the Project Database:** **Crucially, use `template0`** to ensure a pristine, extension-free starting point. ```sh - createdb meshic -T template0 + createdb suhail -T template0 ``` - [ ] **9. Enable PostGIS Extension:** Activate PostGIS within your newly created database. ```sh - psql -d meshic -c "CREATE EXTENSION postgis;" + psql -d suhail -c "CREATE EXTENSION postgis;" ``` --- @@ -87,16 +87,16 @@ uv run alembic upgrade head ``` - [ ] **16. Verify the Schema in PSQL:** - - [ ] Check for tables: `psql -d meshic -c "\dt"` - - [ ] Inspect a spatial table's structure: `psql -d meshic -c "\d+ your_spatial_table"` - - [ ] Confirm PostGIS tracking: `psql -d meshic -c "SELECT * FROM geometry_columns;"` + - [ ] Check for tables: `psql -d suhail -c "\dt"` + - [ ] Inspect a spatial table's structure: `psql -d suhail -c "\d+ your_spatial_table"` + - [ ] Confirm PostGIS tracking: `psql -d suhail -c "SELECT * FROM geometry_columns;"` - [ ] **17. Run Application Tests:** Execute your project's test suite to confirm that the database connection, schema, and spatial queries work as expected. --- **Reference DB URL:** ``` -postgresql+psycopg2://raedmundjennings@localhost:5432/meshic +postgresql+psycopg2://raedmundjennings@localhost:5432/suhail ``` --- diff --git a/docs/PRD.md b/docs/PRD.md index d4fd9ac..27bc733 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -1,4 +1,4 @@ -# Suhail Product Requirements Document (PRD): Meshic Geospatial Data Pipeline +# Suhail Product Requirements Document (PRD): Suhail Geospatial Data Pipeline Author: Mary (Business Analyst) Date: 2025-10-16 diff --git a/docs/PROJECT_BRIEF.md b/docs/PROJECT_BRIEF.md index 84c91f2..f8cffe1 100644 --- a/docs/PROJECT_BRIEF.md +++ b/docs/PROJECT_BRIEF.md @@ -1,4 +1,4 @@ -# Product Brief: Meshic Geospatial Data Pipeline +# Product Brief: Suhail Geospatial Data Pipeline Date: 2025-10-16 Author: Mary (Business Analyst) @@ -8,7 +8,7 @@ Status: Draft for PM Review ## Executive Summary -Meshic is a production-grade, DB-driven geospatial data pipeline that extracts, processes, and enriches Saudi Arabian land parcel data at national scale. The pipeline has processed 2.16M+ parcels across 12 provinces, generated 76M+ price metrics, and captured 130K+ building rules. A database-orchestrated tile queue (34,726 tiles) enables distributed, resumable geometric processing, followed by API-based enrichment (transactions, rules, price metrics). The system is ready for optimization and productization (monitoring, SLAs, client delivery interfaces). +Suhail is a production-grade, DB-driven geospatial data pipeline that extracts, processes, and enriches Saudi Arabian land parcel data at national scale. The pipeline has processed 2.16M+ parcels across 12 provinces, generated 76M+ price metrics, and captured 130K+ building rules. A database-orchestrated tile queue (34,726 tiles) enables distributed, resumable geometric processing, followed by API-based enrichment (transactions, rules, price metrics). The system is ready for optimization and productization (monitoring, SLAs, client delivery interfaces). --- @@ -146,7 +146,7 @@ Organizations require comprehensive, up-to-date, and spatially accurate parcel i - docs/BROWNFIELD_PROJECT_DOCUMENTATION.md - `docs/archive/legacy-root-md/DATABASE_ARCHITECTURE_ANALYSIS.md` - alembic/versions/19c587b33197_add_critical_performance_indexes.py - - src/meshic_pipeline/cli.py, src/meshic_pipeline/persistence/models.py, src/meshic_pipeline/run_db_geometric.py + - src/suhail_pipeline/cli.py, src/suhail_pipeline/persistence/models.py, src/suhail_pipeline/run_db_geometric.py --- diff --git a/docs/REPO_HYGIENE.md b/docs/REPO_HYGIENE.md index 52163e5..7a98814 100644 --- a/docs/REPO_HYGIENE.md +++ b/docs/REPO_HYGIENE.md @@ -15,7 +15,7 @@ Prioritized for **disk win** and **git index sanity**. Derived from a 2026-03-20 ## Done in this hygiene pass - Removed **symlink skill trees** and empty `.cursor/`, `.claude/`, `.agent/` directories. -- **`git rm`** tracked junk: `.cursor/rules` (missing on disk), `src/meshic_pipeline/.DS_Store`, `logs/*.pid`. +- **`git rm`** tracked junk: `.cursor/rules` (missing on disk), `src/suhail_pipeline/.DS_Store`, `logs/*.pid`. - **`.gitignore`:** `logs/*.pid` (plus existing `.DS_Store`, `.venv_py39_backup/`). ## Recommended next steps (highest impact first) diff --git a/docs/SUHAIL_SOURCE_AUDIT_2026-07.md b/docs/SUHAIL_SOURCE_AUDIT_2026-07.md new file mode 100644 index 0000000..9c4b12a --- /dev/null +++ b/docs/SUHAIL_SOURCE_AUDIT_2026-07.md @@ -0,0 +1,391 @@ +# Suhail source audit & integration gap analysis (2026-07-15) + +Forensic, evidence-led audit of the **current live Suhail site** against this +repository's scraper, decoder, parser, schema and ingestion pipeline. Every claim +below is backed by a captured payload or a source line. Captured evidence lives in +[`tests/fixtures/suhail_live_2026_07/`](../tests/fixtures/suhail_live_2026_07) and the +working capture set under the session scratchpad. + +> TL;DR — The site was rebuilt on a new React/Vite front end backed by `api2.suhail.ai`. +> The tile schema is dramatically richer than when the scraper was written: the +> `parcels` MVT layer now carries **inline time-series market data** (1w/1m/6m/12m +> transaction price, price-of-meter, transaction count and date), and there are +> **five new tile layers** (`dimensions`, `streets`, `provinces`, `building_detection`, +> `non_saudi_ownership_zones`) plus a duplicate `parcels-base`. Several existing layers +> (`bus_lines`, `metro_stations`, `riyadh_bus_stations`, `qi_population_metrics`, +> `qi_stripes`) changed field names and now capture **nothing or only their key**. +> All three enrichment REST endpoints still work and are unauthenticated; a new +> consolidated `parcel/{id}` endpoint and a province-wide neighbourhood-metrics list +> endpoint are now available. + +--- + +## 1. Method & evidence + +Reconnaissance was done against transport, not the rendered UI: + +- Loaded `https://www.suhail.ai` + `/Riyadh/metrics` in a headless browser and captured + all network activity. +- Downloaded and decoded a live MVT tile (`tiles.suhail.ai/maps/riyadh/15/20636/14069.vector.pbf`, + and a residential tile `.../20640/14060`) with `mapbox_vector_tile`. +- Pulled the authoritative Mapbox GL style (`tiles.suhail.ai/gl-styles/ksa.json`) — it + declares every tile source, source-layer and tile URL template. +- Pulled `api2.suhail.ai/{regions,settings/app}` and `tiles.suhail.ai/modes/`. +- Mined the front-end JS bundles (`assets/index-*.js` etc.) for the complete axios API + surface (base `https://api2.suhail.ai/`). +- Probed each REST endpoint directly with real parcel/neighbourhood/region IDs taken + from the decoded tile. + +Capture manifest: [`CAPTURE_MANIFEST.md`](../tests/fixtures/suhail_live_2026_07) equivalent +saved in scratchpad; representative payloads committed under `tests/fixtures/suhail_live_2026_07/`. + +### Transport facts that matter + +- **Tiles are always gzipped.** `tiles.suhail.ai` returns `Content-Encoding: gzip` + even for `Accept-Encoding: identity`. `aiohttp` auto-decompresses this, so the + downloader is unaffected — but any code that reads tile bytes *without* HTTP-layer + decompression (raw `requests` without `--compressed`, reading a hand-saved `.pbf`) will + see gzip magic `1f 8b` and fail to decode. Evidence: `curl -H "Accept-Encoding: identity" -D-` + still returns gzip; `mapbox_vector_tile.decode` raises `DecodeError` on the raw bytes, + succeeds after `gzip.decompress`. +- **No auth on data endpoints.** Every `api2.suhail.ai` data GET returned `200` with no + `Authorization` header. `settings/app` advertises `AnonUserRestrictionsEnabled: true`, + `AnonUserParcelClicks: 15`, `AnonUserSessionTimeSec: 900` — these are **UI-side** limits, + not server-enforced on the REST endpoints we probed. +- **Tile server = Tegola** (`Tegola-Cache: HIT`), `maxzoom: 15`, CORS `Vary: Origin`. + +--- + +## 2. Live source capability inventory + +### 2.1 Hosts + +| Host | Role | +|------|------| +| `www.suhail.ai` | React/Vite SPA. Routes: `/:Region/metrics`, `/:Region/parcel/:parcelObjectId`, `/parcel/:id`, `/map`, `/mobile-metrics`, `/offer/:offerCode` | +| `api2.suhail.ai` | Primary JSON API (axios base `https://api2.suhail.ai/`) | +| `tiles.suhail.ai` | MVT vector tiles, `gl-styles/ksa.json`, `modes/`, raster overlays | +| `reports.suhail.ai` | PDF reports: `/api/report/Parcel`, `/api/report/Order` | +| `beacon.suhail.ai` | Strapi CMS (news feed, mega-project content, GPTV form) | + +### 2.2 REST API surface (from JS bundle + live probes) + +Method is GET unless noted. Status column = observed against live probe. + +| Endpoint (relative to `api2.suhail.ai/`) | Params | Purpose | Status | Captured by repo? | +|---|---|---|---|---| +| `regions` | – | 13 regions, nested provinces, centroids, bbox, `mapStyleUrl`, `mapKey`, `metricsUrl` | 200 | Partial — `scripts/util/sync_provinces.py` reads a subset | +| `settings/app` | – | anon limits, transaction filter types, map bounds, unit-status colours, report URLs, `realEstateTaxPercentage`, Nafath flag | 200 | **No** | +| `parcel/{parcelObjectId}` | path | **Consolidated** parcel detail incl. geometry + centroid + dimensionsCount | 200 | **No** (new) | +| `transactions?parcelObjectId=` | parcel | Per-parcel transaction records (39 fields) | 200 | Yes (5 fields + raw) | +| `consolidatedTransactions` | `parcelObjectId, regionId, LookbackValue, LookbackType, fromPrice, toPrice, type` | Filtered per-parcel transactions | 200 | **No** (new) | +| `transactionsAsMapboxGeojson` | `RegionId, LookbackValue, LookbackType, NewDaysThreshold, fromPrice, toPrice, type` | Region-wide transaction points as GeoJSON (2728 features for Riyadh/1mo) | 200 | **No** (new) | +| `transactions/neighbourhood?neighborhoodId=` | nbhd | Paginated neighbourhood transactions | 200 | **No** (new) | +| `parcel/buildingRules?parcelObjectId=` | parcel | Zoning/building rules (16 fields incl. setbacks) | 200 | Yes (fully) | +| `api/parcel/metrics/priceOfMeter?parcelObjsIds=&groupingType=Monthly` | parcels | Per-parcel monthly price metrics (neighbourhood-derived) | 200 | Yes (partial — drops `neighborhoodId`) | +| `api/mapMetrics/landMetrics?neighborhoodId=&growthRateType=` | nbhd | Neighbourhood market metrics + growth indicators, per land-use group | 200 | **No** (util-only, not persisted) | +| `api/mapMetrics/landMetrics/list?regionId=&offset=&limit=` | region, page | **Province-wide** paginated neighbourhood metrics (totals, medians, growth, last-execution price per land-use group) | 200 | **No** (new) | +| `api/parcel/landZoningGroups` | – | Zoning-group lookup with national usage counts | 200 | **No** (new) | +| `api/parcel/search` | query | Parcel search | 400 w/o valid query | **No** (new) | +| `api/parcel/shareLink?parcelObjectId=` | parcel | Share link | – | **No** | +| `api/bookmark`, `Bookmark`, `Badges/*`, `oauth/*`, `accounts/*`, `complaints`, `events/log`, `api/attachments`, `api/StrapiContent/*`, `api/MegaProjects/project/content` | – | User/account/CMS features | – | N/A (out of scope for ingestion) | + +### 2.3 Tile map style (`gl-styles/ksa.json`) + +- 13 per-region vector tile sources: `tiles.suhail.ai/maps/{riyadh,madinah,qassim,asir,eastern,makkah,bahah,hail,jawf,jazan,najran,northern_borders,tabuk}/{z}/{x}/{y}.vector.pbf` plus an all-KSA `maps/ksa/...` source, each `maxzoom: 15` with a `bounds` array. +- Suhail-specific **source-layers** referenced by the style (excludes Mapbox base layers): + `parcels`, `parcels-centroids`, `neighborhoods`, `neighborhoods-centroids`, `subdivisions`, + `provinces`, `provinces-centroids`, `dimensions`, `streets`, `building_detection`, + `non_saudi_ownership_zones`, `mega_projects`, `sb_shape`, `sb_area`, `qi_population_metrics`, + `metro_lines`, `bus_lines`, `metro_stations`, `riyadh_bus_stations`, `regions`. +- New raster overlays: per-region base rasters on Huawei OBS (`qeye-tiles-prod.obs.me-east-1.myhuaweicloud.com`) + and named development-project rasters (`tiles.suhail.ai/tiles/{watheer,rakiz,osus,...}`). +- Map coloring modes (`tiles/modes/`): `zoning_color` (by `zoning_id`), `shape_area`, + `price_of_meter`, `transaction_price`, `plain` — confirms these are the live-styled parcel fields. + +### 2.4 Live MVT tile — decoded layers & fields (ground truth) + +Decoded from `riyadh/15/20636/14069` (downtown) and `.../20640/14060` (residential). 17 layers present: + +**`parcels`** (3083 feats, Polygon) — the headline change. Fields: +`parcel_id, parcel_objectid, neighborhaname, neighborhood_id, province_id, municipality_aname, +block_no, subdivision_id, subdivision_no, shape_area, zoning_id, zoning_color, ruleid, +zoning_group, transaction_price, price_of_meter, +transaction_price_1w/1m/6m/12m, price_of_meter_1w/1m/6m/12m, +transactions_count_1w/1m/6m/12m, transaction_date_1w/1m/6m/12m, +landuseadetailed, landuseagroup, parcel_no`. + +**`parcels-base`** (3083, Polygon) — identical field set to `parcels` (base/unstyled copy). + +**`parcels-centroids`** (3081, Point) — now carries the same time-series block plus +`transactions_count`, `transaction_date`. + +**`neighborhoods`** (Polygon/MultiPolygon) — adds `neighborh_aname, zoning_group` and the +full 1w/1m/6m/12m time-series (`transactions_count_*` are strings here). + +**`subdivisions`** — adds `subdivision_name_ar, neighborhood_id, region_id` and time-series. + +**`neighborhoods-centroids`** (Point) — now only `neighborh_aname, province_id`. + +**`provinces`** (Polygon) — NEW: `province_aname, province_enname, region_id, province_code`. + +**`dimensions`** (13773, Point) — NEW: `parcel_objectid, length_m, province_id, azimuth` +(per-edge parcel dimensions / setbacks). + +**`streets`** (290, LineString) — NEW: `width, name_ar, name_en`. + +**`metro_lines`** (LineString) — `track_name, track_color, track_length`. + +**`bus_lines`** (LineString) — `color, type, busroute, origin, originar`. + +**`metro_stations`** (Point) — `station_code, station_name, station_long, station_lat`. + +**`riyadh_bus_stations`** (Point) — `station_name, station_code, station_long, station_lat`. + +**`qi_population_metrics`** (Polygon) — now `grid_id, population_density, rent_apartment, +rent_villa, rent_shop, rent_office, purchasing_power, weighted_median_income_monthly, +poi_count, region_id` — **values are hex colour strings**, not raw numbers. + +**`qi_stripes`** (Polygon) — now `strip_id, centroid_longitude, centroid_latitude`. + +**`building_detection`** (3082, Polygon) — NEW: `class_pred, prediction_year, region_id` +(AI-detected building footprints). + +**`non_saudi_ownership_zones`** (Polygon) — NEW: `id, region_id, province_id, is_show, +name_ar, name_en` (foreign-ownership / special-zone polygons). + +### 2.5 Key REST payload shapes (captured) + +- **`transactions` record (39 fields)**: `transactionNumber, transactionPrice, priceOfMeter, + _priceOfMeter, transactionDate, type, propertyType, metricsType, subdivisionNo, subdivisionId, + neighborhood, neighborhoodId, region, regionId, provinceId, provinceName, parcelId, + parcelObjectId, parcelNo, blockNo, area, totalArea, noOfProperties, centroidX, centroidY, + centroid, polygonData (WKT-ish GeoJSON string), geometry, landUsageGroup, landUseGroup, + landUseaDetailed, sellingType, transactionSource, isLowValueTransaction, orignalTransactionNum, + propertyNumber, projectName, parcelImageURL, details`. The top-level `data` object also + carries `lastExecutionDate`, `lastExecutionPrice`. +- **`buildingRules` (16 fields)**: `id, zoningId, zoningColor, zoningGroup, landuse, description, + name, coloring, coloringDescription, maxBuildingCoefficient, maxBuildingHeight, + maxParcelCoverage, maxRuleDepth, mainStreetsSetback, secondaryStreetsSetback, sideRearSetback`. +- **`priceOfMeter`**: `data[].{parcelObjId, neighborhoodId, from, to, groupingType, + neighborhoodMetrics[], parcelMetrics[]}` where each metric = `{neighborhoodId, month, year, + metricsType, avaragePriceOfMeter}` (API misspells "average"). +- **`landMetrics/list`**: `data[].{neighborhoodId, neighborhoodName, provinceId, provinceName, + totalMetricData{totalCount, totalPrice, median, *GrowthIndicator, *GrowthValue}, landUseGroup[] + {landUseGroup, totalCount, totalPrice, median, lastExecutionPrice{transactionDate, priceOfMeter}}}`. + +--- + +## 3. Forensic audit of the existing implementation + +Pipeline traced end-to-end: discovery (`discovery/tile_discovery.py`, `tile_urls` queue) → +request (`downloader/async_tile_downloader.py`, `run_db_geometric.fetch_many`) → decode +(`decoder/mvt_decoder.py`) → validate/stitch (`geometry/`) → schema-filter +(`SCHEMA_MAP` in `persistence/postgis_persister.py`) → persist (`PostGISPersister`, +`models.py`, Alembic) → enrichment (`enrichment/api_client.py`, `enrichment_persister.py`, +`strategies.py`). + +Classification per live capability: + +| Capability | Status | Evidence | +|---|---|---| +| Tile discovery / `tile_urls` queue | **Fully captured** — still valid; URL shape `maps/{region}/{z}/{x}/{y}.vector.pbf` unchanged | `gl-styles/ksa.json` sources match `config.tile_base_url`; live tiles 200 | +| Parcels geometry + core attributes | **Fully captured** | 16 core parcel fields in `SCHEMA_MAP['parcels']` match tile | +| Parcels **inline market time-series** (16 fields) + `zoning_group` | **Available from source but not captured** | tile has `transaction_price_1w/1m/6m/12m`, `price_of_meter_*`, `transactions_count_*`, `transaction_date_*`, `zoning_group`; none in `SCHEMA_MAP` → filtered out at `pipeline_orchestrator.py:298` / `run_db_geometric.py:90` | +| Parcels `neighborhood_ar`, `municipality_ar` | **Captured but discarded / mis-mapped** | tile field `neighborhaname`→`neighborhood_ar` exists but `neighborhood_ar` not in `SCHEMA_MAP['parcels']`; `municipality_aname` has no `ARABIC_COLUMN_MAP` entry so `municipality_ar` stays NULL | +| parcels-centroids market data | **Partially captured** | only `transaction_date/price/price_of_meter` kept; time-series + `transactions_count` dropped | +| neighborhoods / subdivisions market data | **Partially captured** | time-series, `zoning_group`, `subdivision_name_ar`, `neighborhood_id`, `region_id` dropped | +| `bus_lines` | **No longer compatible — ZERO fields captured** | tile fields `color,type,busroute,origin,originar`; `SCHEMA_MAP['bus_lines']` expects `route_name,route_color,route_length,route_type,route_id`; `models.BusLines` has yet a third set (`busroute,route_name,route_type`). Nothing matches → only geometry+null id stored | +| `metro_stations` / `riyadh_bus_stations` | **Partially captured / mis-mapped** | tile has `station_long,station_lat`; schema expects `location,line_id/route_id` → those columns NULL | +| `qi_population_metrics` | **Captured but incorrectly interpreted** | schema expects a numeric `population`; tile now emits 9 colour-coded metric fields (`population_density`, `rent_*`, `purchasing_power`, `weighted_median_income_monthly`, `poi_count`). Only `grid_id` survives; `population` always NULL | +| `qi_stripes` | **No longer compatible / likely obsolete** | schema expects `stripe_value,stripe_type`; tile has `centroid_longitude,centroid_latitude`; layer absent from `gl-styles` (not rendered) | +| `dimensions` layer | **Available from source but not captured** | present in `id_column_per_layer` but absent from `layers_to_process`, `table_name_mapping`, `SCHEMA_MAP`, `models` — 13773 features/tile dropped | +| `streets`, `provinces`(tile), `building_detection`, `non_saudi_ownership_zones`, `parcels-base` | **Available from source but not captured** | new tile layers, no schema/model/table | +| Enrichment: transactions | **Partially captured** | only `transaction_id, transaction_price, price_of_meter, transaction_date, area` + `raw_data`; drops `type/propertyType, metricsType, landUseGroup, sellingType, transactionSource, totalArea, subdivisionId, neighborhoodId, polygonData(geometry)` from queryable columns | +| Enrichment: building rules | **Captured but silently reduced** | `enrichment_persister.py:53` dedupes to **one rule per parcel** (`{parcel_objectid: rule}`) despite composite PK `(parcel_objectid, building_rule_id)` — extra rules dropped | +| Enrichment: price metrics `neighborhood_id` | **Captured but discarded** | `api_client.py:244-266` never sets `neighborhood_id` though every metric object contains `neighborhoodId`; column left NULL | +| Enrichment upserts | **Captured but never refreshed** | all conflict handling is `ON CONFLICT DO NOTHING` (`enrichment_persister.py`), so changed transaction/rule/metric values never update on re-run | +| `regions` endpoint richness | **Partially captured** | `sync_provinces.py` reads a subset; `mapKey`, `metricsUrl`, `defaultTransactionsDateRange`, per-region bounds not persisted | +| Provenance (source tile z/x/y, fetch time, per-feature) | **Missing** | no per-row source coordinates or fetch timestamp; `geometry_hash` column defined but never computed in Stage-1 | +| `landMetrics` neighbourhood market intelligence | **Available from source but not captured** | `enhanced_province_discovery.py` fetches `landMetrics` but no persister consumes it | +| Auth / API-key / adaptive rate-limit (docs claim) | **Redundant / aspirational** | `docs/brownfield-architecture.md` claims API-key auth + adaptive rate limiting; code sends no key and has no enrichment rate limiter | + +--- + +## 4. Gap analysis (evidence-indexed) + +Precise per-layer field comparison (live tile field, snake_cased + Arabic-mapped, vs +`SCHEMA_MAP`) is reproduced in §2.4. The material gaps, ranked: + +1. **Parcels inline market time-series (16 fields) + `zoning_group` are dropped.** This is + the single biggest miss: the tile now delivers, for free, the same signal the enrichment + pipeline pays per-parcel API calls to approximate. Evidence: decoded tile vs `SCHEMA_MAP['parcels']`. +2. **`bus_lines` captures zero attributes** — three-way name mismatch (tile vs `SCHEMA_MAP` vs `models`). +3. **`qi_population_metrics` semantics changed** — `population` column stays NULL; 9 new metric fields dropped. +4. **Five new tile layers unmodelled**: `dimensions` (13773/tile), `building_detection` (3082/tile), + `streets`, `provinces`, `non_saudi_ownership_zones`; plus `parcels-base` duplicate. +5. **Transaction enrichment discards queryable market attributes** (type, land-use, selling + type, source, areas, subdivision/neighbourhood ids, transaction geometry). +6. **Building-rules dedup bug** collapses multiple rules per parcel to one. +7. **Price-metrics `neighborhood_id` never persisted.** +8. **Mis-mapped Arabic fields** (`municipality_ar`, parcels `neighborhood_ar`). +9. **No per-feature provenance** (source tile, fetch time) and `geometry_hash` never computed. +10. **New high-value endpoints unused**: consolidated `parcel/{id}`, `landMetrics/list` + (province-wide neighbourhood market data), `landZoningGroups`, `transactionsAsMapboxGeojson`. +11. **`ON CONFLICT DO NOTHING`** everywhere in enrichment → no refresh of changed values. +12. **gzip fragility** in any non-aiohttp read path. + +--- + +## 5. Architectural & platform improvements (recommended) + +Prefer general ingestion strengthening over more one-off Suhail hacks: + +- **Single source-of-truth layer schema.** `SCHEMA_MAP` (persister), `models.py`, + `id_column_per_layer`/`aggregation_rules`/`table_name_mapping` (config) and the migrations + have drifted (e.g. `bus_lines` differs in all three). Collapse to one declarative + per-layer spec (fields, types, pk, arabic map, geometry type) that generates the schema + map, the aggregation columns and the DDL, so drift is impossible. +- **Schema-drift detection.** A CI/ops check that decodes a live sample tile and diffs its + field set against the declared spec; fail/alert on unknown or missing fields. Would have + caught all of §4 automatically. +- **Raw capture / replay + contract tests.** Persist a small set of real gzipped tiles and + REST payloads as fixtures (done — `tests/fixtures/suhail_live_2026_07/`) and run the + decoders/parsers against them in CI. There were previously **no** fixture-replay tests for + the API client or decoder field mapping. +- **Provenance & lineage.** Add per-feature `source_tile_z/x/y`, `source_region`, `fetched_at`, + and compute `geometry_hash`, so a row can be traced to the tile and run that produced it and + change-detection becomes deterministic. +- **Direct structured requests over UI assumptions.** The consolidated `parcel/{id}` and the + `landMetrics/list` pagination let us pull neighbourhood market intelligence province-wide + without per-parcel fan-out; prefer these to the sparse per-parcel `transactions`/`buildingRules`. +- **Idempotent upserts.** Replace enrichment `DO NOTHING` with `DO UPDATE` (or a + content-hash guard) so re-runs refresh changed values; today they cannot. +- **Consistent transport policy.** Unify retry/backoff/timeout across the three enrichment + fetchers (transactions uses decorator backoff+jitter; rules/metrics use a hand-rolled fixed + 1s loop with no timeout), add explicit 429 handling, and make gzip handling explicit. +- **Dead code / obsolete paths.** `run_db_geometric_fixed.py` duplicates `run_db_geometric.py`; + `qi_stripes` is no longer in the live style; `parcels-base` is a duplicate of `parcels`. + Decide keep/drop deliberately. + +--- + +## 6. Implemented in this pass + +Scope chosen by **evidence strength × value ÷ risk**. All changes are additive or +correctness fixes; nothing removes existing behaviour. + +### Stage 1 (geometric) +- **Parcels inline market time-series captured** — `SCHEMA_MAP['parcels']`, aggregation + rules, `Parcel` model and migration now include `transaction_price_{1w,1m,6m,12m}`, + `price_of_meter_{…}`, `transactions_count_{…}`, `transaction_date_{…}`, plus `zoning_group` + and `neighborhood_ar`. Same time-series added to `parcels-centroids`, `neighborhoods`, + `subdivisions` (+ `subdivision_name_ar`, `neighborhood_id`, `region_id`). +- **`municipality_ar` mapping fixed** — added `municipality_aname → municipality_ar` to + `ARABIC_COLUMN_MAP` (was always NULL). +- **Broken layers remapped** — `bus_lines` (native `busroute/color/type/origin/originar`, + was capturing zero attributes), `metro_stations` / `riyadh_bus_stations` + (`station_long/station_lat`), `qi_population_metrics` (reshaped colour-coded metric + fields + `region_id`), `qi_stripes` (`centroid_longitude/latitude`). +- **New layer wired in** — `non_saudi_ownership_zones` (unique `id` PK → clean upsert): + `layers_to_process`, `table_name_mapping`, `id_column_per_layer`, `SCHEMA_MAP`, + `NonSaudiOwnershipZones` model, new table in the migration. +- **New keyless layers wired in with two accumulate write modes** (added after the first + pass — these previously could not be persisted across a multi-tile province run because the + keyless branch used per-batch `if_exists="replace"`, keeping only the last batch): + - **`dimensions`** (per-parcel edge measurements: `length_m`, `azimuth`; ~4–6 rows/parcel) → + **tile-scoped delete+append** (`TILE_SCOPED_LAYERS`, `PostGISPersister.write_tile_scoped`). + A `source_tile` provenance column records the originating tile; each batch deletes exactly + its own tiles' rows then appends, so multi-tile runs accumulate and reruns are idempotent. + Deliberately **not** keyed on `parcel_objectid` (that would collapse every parcel to one edge). + - **`building_detection`** (AI footprints, year-stamped) → **deterministic synthetic key** + (`SYNTHETIC_PK_CONFIG`, `compute_synthetic_pk`: SHA1 of `region_id|class_pred|prediction_year` + + geometry → `bd_id`), reusing the existing upsert path. Idempotent, versions by prediction + year, and dedupes tile-overlap duplicates for free. + - Both stamped in the shared `decode_and_validate_tile` choke point, so both persist paths + inherit them. Migration `d8b1e6f42a90` creates the two tables (indexed on `source_tile`). +- **Still deferred** — `streets` (keyless lines, same pattern available if wanted). Tile + `provinces` intentionally omitted (collides with the metadata table). + +### Stage 2 (enrichment) +- **Transactions enriched** — promoted `type, propertyType, metricsType, landUseGroup, + landUseaDetailed, sellingType, transactionSource, totalArea, subdivisionId, neighborhoodId, + isLowValueTransaction` from `raw_data` into queryable columns (model + migration + parser + + persister). +- **Building-rules dedup bug fixed** — persister now keys on `(parcel_objectid, + building_rule_id)`, not `parcel_objectid` alone (previously discarded all but one rule). +- **Price-metrics `neighborhood_id` now captured** (was always NULL), with neighborhood + **stub insertion** added to `fast_store_batch_data` so populating the FK column cannot abort + a batch when the neighborhood row is absent. +- **Transport/parse separation** — the three parsers are now pure functions + (`parse_transactions_payload`, `parse_building_rules_payload`, `parse_price_metrics_payload`) + callable without HTTP, enabling fixture-backed contract tests. + +### Tests, fixtures, migration +- Live payloads committed as regression fixtures under + `tests/fixtures/suhail_live_2026_07/` (tiles + API JSON). +- `tests/unit/test_live_tile_schema.py` (decode real tiles → assert new fields survive the + schema filter) and `tests/unit/test_enrichment_parsers_contract.py` (parse real API + payloads → assert promoted fields, multi-rule survival, neighborhood_id). 11 new tests; + full unit suite 88 passed. +- Alembic migration `c7f4a9d21b60_suhail_2026_source_expansion` (idempotent `ADD COLUMN IF + NOT EXISTS`, new table, reversible — verified downgrade/upgrade round-trip). + +### Deliberately not done (recommended next, with rationale) +- New endpoints as ingest sources (`landMetrics/list` province-wide neighbourhood market + intelligence; consolidated `parcel/{id}`; `landZoningGroups` lookup) — high value, but new + persisters/tables beyond this pass's tested scope. +- `streets` — keyless line layer, still deferred (the tile-scoped write mode built for + `dimensions` applies directly if wanted). `dimensions` and `building_detection` are now done + (see §6 above). +- Replace enrichment `ON CONFLICT DO NOTHING` with `DO UPDATE` for true refresh-on-rerun. +- Per-feature provenance (`source_tile_z/x/y`, `fetched_at`) and `geometry_hash` computation. +- Collapse the drifting layer-schema definitions (`SCHEMA_MAP` / models / config) into one + declarative spec + a schema-drift CI check. +- **Quarantine of dropped features** — a `quarantined_features` table capturing malformed + geometries / unmappable rows the pipeline currently drops silently (decoder feature-loop + `except: continue`, empty-geometry validation, null-PK filtering), plus a `province_id_mapping` + lookup. Prototyped on the abandoned `autotune-error-reporting` branch (evaluated and dropped: + ~1yr stale, mostly superseded, no consumer). **Not built here on purpose** — it's only worth it + once real, quantified data-loss is observed and a consumer (report/alert) exists; building an + unmonitored quarantine table now would be speculative. Build fresh against current code if a + need is demonstrated. + +## 7. End-to-end validation results (live, 2026-07-15) + +Validated against the **live** `tiles.suhail.ai` / `api2.suhail.ai` and the local +`suhail_pipeline` PostgreSQL 18 database, through the real pipeline modules. + +**Stage 1** (`fetch_many → decode_and_validate_tile → _apply_arabic_and_columns → +PostGISPersister.write` upsert into schema-identical tables): +- 4/4 live tiles fetched and decoded (7,542 parcels). +- New columns populated: `zoning_group` 7,438; `neighborhood_ar` 7,541; `municipality_ar` + 7,542; `transaction_price_12m` 7,542; `transactions_count_12m>0` 71; `transaction_date_12m` 71. +- **Idempotent**: rerun left row count and values unchanged (7,542 → 7,542). +- **Source-vs-persisted cross-check**: parcel `9941681` persisted + `price_of_meter=6393.18`, `transaction_date_12m=2026-05-19`, `transactions_count_12m=6`, + `zoning_group=سكني`, `neighborhood_ar=الروضة` — matches the live `/transactions` API + (`lastExecutionPrice=6393.0`, 6 transactions). +- Fixed layers captured live: `bus_lines` 24 rows with `busroute` (was zero), + `metro_stations` 3 with coordinates, `qi_population_metrics` 87 with metric fields, + `non_saudi_ownership_zones` 1 with `name_en`. + +**Stage 2** (live `SuhailAPIClient` → real `fast_store_batch_data`): +- Live fetch for parcel `9941681`: 6 transactions, 1 building rule, 83 metrics. +- Persisted transaction row carried every promoted column + (`transaction_type=مبنى سكني, metrics_type=فلل, land_use_group=سكني, selling_type=فردي, + transaction_source=RER, total_area=187.7, subdivision_id=1023186, neighborhood_id=1000417, + is_low_value_transaction=False`). +- All 80 persisted metrics carried `neighborhood_id` (previously always NULL); neighborhood + stub insertion prevented an FK-violation batch abort. + +**New keyless layers** (live tiles → real persister write modes → dedicated tables): +- `dimensions` (tile-scoped): batch A 18,447 rows → batch B **accumulated** to 30,869 + (= A + B; the old `replace` would have kept only B's 12,422). Reprocessing batch A left the + count unchanged (idempotent). A parcel retained all 23 of its edges — not collapsed. +- `building_detection` (synthetic key): batch A 4,286 → A∪B 7,541 distinct predictions, + idempotent on rerun; rows carry `bd_id`, `class_pred`, `prediction_year`, `source_tile`. + +All temporary validation tables/rows were cleaned up; the database was left in its prior +state (enrichment tables empty, no seeded rows). Migrations `c7f4a9d21b60` and `d8b1e6f42a90` +both round-trip (downgrade/upgrade verified). Full unit suite: **93 passed**. diff --git a/docs/archive/WHAT_TO_DO_NEXT_PIPELINE_TABLES.md b/docs/archive/WHAT_TO_DO_NEXT_PIPELINE_TABLES.md index 0365a38..e32c504 100644 --- a/docs/archive/WHAT_TO_DO_NEXT_PIPELINE_TABLES.md +++ b/docs/archive/WHAT_TO_DO_NEXT_PIPELINE_TABLES.md @@ -78,7 +78,7 @@ def downgrade(): ## 6. **Update Version Control** - Commit all config, migration, and documentation changes to git with a clear message: - - `git add src/meshic_pipeline/config.py alembic/versions/ pipeline_table_uniqueness_review.md WHAT_TO_DO_NEXT_PIPELINE_TABLES.md` + - `git add src/suhail_pipeline/config.py alembic/versions/ pipeline_table_uniqueness_review.md WHAT_TO_DO_NEXT_PIPELINE_TABLES.md` - `git commit -m "Robustify pipeline table upsert/replace logic and document uniqueness strategy"` --- diff --git a/docs/archive/generated-txt/all_test_results.txt b/docs/archive/generated-txt/all_test_results.txt index d17aa7e..682dc3a 100644 --- a/docs/archive/generated-txt/all_test_results.txt +++ b/docs/archive/generated-txt/all_test_results.txt @@ -1,260 +1,260 @@ -2025-07-06 14:37:17,380 - meshic_pipeline.persistence.db - INFO - Setting up database tables... -2025-07-06 14:37:17,412 - meshic_pipeline.persistence.db - INFO - Tables checked/created successfully. -2025-07-06 14:37:17,415 - meshic_pipeline.enrichment.strategies - INFO - Found 126 enrichable parcels (including previously processed). +2025-07-06 14:37:17,380 - suhail_pipeline.persistence.db - INFO - Setting up database tables... +2025-07-06 14:37:17,412 - suhail_pipeline.persistence.db - INFO - Tables checked/created successfully. +2025-07-06 14:37:17,415 - suhail_pipeline.enrichment.strategies - INFO - Found 126 enrichable parcels (including previously processed). 2025-07-06 14:37:17,415 - __main__ - INFO - Starting get_all_enrichable_parcel_ids for 126 parcels with batch size 50 -2025-07-06 14:37:17,415 - meshic_pipeline.enrichment.processor - INFO - 🚀 Processing batch 1/3 (50 parcels) -2025-07-06 14:37:17,416 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,416 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,416 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,416 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,416 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,416 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,416 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,416 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,417 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,418 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,419 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,420 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,420 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,420 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,420 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,420 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,420 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,420 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:17,906 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,909 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,909 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,926 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,927 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,927 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,927 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,929 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,929 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,929 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,929 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,929 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,930 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,930 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,930 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,930 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,930 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,931 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,931 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,931 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,931 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,931 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,931 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,932 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,933 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,933 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,933 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,934 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,934 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,934 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,934 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,935 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,935 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:17,935 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,002 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,050 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,052 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,057 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,058 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,058 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,063 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,064 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,064 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,064 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,064 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,064 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,065 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,065 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,065 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,081 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,507 - meshic_pipeline.enrichment.processor - INFO - 🚀 Processing batch 2/3 (50 parcels) -2025-07-06 14:37:18,508 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,508 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,508 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,508 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,509 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,509 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,509 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,509 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,509 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,509 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,509 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,509 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,509 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,509 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,510 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,510 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,510 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,510 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,510 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,510 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,510 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,510 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,510 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,510 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,511 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,511 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,511 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,511 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,511 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,511 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,511 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,511 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,511 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,511 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,512 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,512 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,512 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,512 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,512 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,512 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,512 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,512 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,512 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,512 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,513 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,513 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,513 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,513 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,513 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,513 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:18,632 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,633 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,633 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,633 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,634 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,635 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,635 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,636 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,637 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,637 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,638 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,638 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,638 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,639 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,640 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,640 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,640 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,640 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,641 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,641 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,641 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,642 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,642 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,643 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,643 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,643 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,643 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,644 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,644 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,644 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,644 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,644 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,644 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,644 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,645 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,645 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,645 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,646 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,646 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,646 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,647 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,647 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,647 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,647 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,647 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,656 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,677 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,939 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,940 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:18,970 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,100 - meshic_pipeline.enrichment.processor - INFO - 🚀 Processing batch 3/3 (26 parcels) -2025-07-06 14:37:19,100 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,101 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,101 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,101 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,101 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,101 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,101 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,101 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,101 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,102 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,102 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,102 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,102 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,102 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,102 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,102 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,102 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,102 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,102 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,103 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,103 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,103 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,103 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,103 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,103 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,103 - meshic_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions -2025-07-06 14:37:19,226 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,227 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,227 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,227 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,227 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,228 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,228 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,229 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,229 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,229 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,229 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,229 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,229 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,229 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,229 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,231 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,231 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,231 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,232 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,233 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,234 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,234 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,234 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,234 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,234 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions -2025-07-06 14:37:19,258 - meshic_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,415 - suhail_pipeline.enrichment.processor - INFO - 🚀 Processing batch 1/3 (50 parcels) +2025-07-06 14:37:17,416 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,416 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,416 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,416 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,416 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,416 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,416 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,416 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,417 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,418 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,419 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,420 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,420 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,420 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,420 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,420 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,420 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,420 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:17,906 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,909 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,909 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,926 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,927 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,927 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,927 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,929 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,929 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,929 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,929 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,929 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,930 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,930 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,930 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,930 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,930 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,931 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,931 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,931 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,931 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,931 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,931 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,932 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,933 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,933 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,933 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,934 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,934 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,934 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,934 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,935 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,935 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:17,935 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,002 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,050 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,052 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,057 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,058 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,058 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,063 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,064 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,064 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,064 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,064 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,064 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,065 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,065 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,065 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,081 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,507 - suhail_pipeline.enrichment.processor - INFO - 🚀 Processing batch 2/3 (50 parcels) +2025-07-06 14:37:18,508 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,508 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,508 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,508 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,509 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,509 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,509 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,509 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,509 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,509 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,509 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,509 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,509 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,509 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,510 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,510 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,510 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,510 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,510 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,510 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,510 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,510 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,510 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,510 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,511 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,511 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,511 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,511 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,511 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,511 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,511 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,511 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,511 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,511 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,512 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,512 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,512 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,512 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,512 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,512 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,512 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,512 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,512 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,512 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,513 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,513 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,513 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,513 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,513 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,513 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:18,632 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,633 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,633 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,633 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,634 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,635 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,635 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,636 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,637 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,637 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,638 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,638 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,638 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,639 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,640 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,640 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,640 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,640 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,641 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,641 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,641 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,642 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,642 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,643 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,643 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,643 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,643 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,644 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,644 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,644 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,644 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,644 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,644 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,644 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,645 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,645 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,645 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,646 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,646 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,646 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,647 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,647 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,647 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,647 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,647 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,656 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,677 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,939 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,940 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:18,970 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,100 - suhail_pipeline.enrichment.processor - INFO - 🚀 Processing batch 3/3 (26 parcels) +2025-07-06 14:37:19,100 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,101 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,101 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,101 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,101 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,101 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,101 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,101 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,101 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,102 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,102 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,102 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,102 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,102 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,102 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,102 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,102 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,102 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,102 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,103 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,103 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,103 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,103 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,103 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,103 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,103 - suhail_pipeline.enrichment.api_client - INFO - Starting operation: fetch_transactions +2025-07-06 14:37:19,226 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,227 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,227 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,227 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,227 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,228 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,228 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,229 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,229 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,229 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,229 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,229 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,229 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,229 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,229 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,231 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,231 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,231 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,232 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,233 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,234 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,234 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,234 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,234 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,234 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions +2025-07-06 14:37:19,258 - suhail_pipeline.enrichment.api_client - INFO - Completed operation: fetch_transactions 2025-07-06 14:37:19,396 - __main__ - INFO - Finished get_all_enrichable_parcel_ids. Added 76 transactions, 124 building rules, 876 price metrics. diff --git a/docs/archive/generated-txt/flake8_report.txt b/docs/archive/generated-txt/flake8_report.txt index b12a442..d3b3101 100644 --- a/docs/archive/generated-txt/flake8_report.txt +++ b/docs/archive/generated-txt/flake8_report.txt @@ -1,895 +1,895 @@ src/__init__.py:1:1: W293 blank line contains whitespace -src/meshic_pipeline/__init__.py:1:62: W291 trailing whitespace -src/meshic_pipeline/cli.py:13:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:43:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:45:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/cli.py:46:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/cli.py:48:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/cli.py:51:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:53:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/cli.py:54:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/cli.py:55:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/cli.py:57:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/cli.py:58:80: E501 line too long (101 > 79 characters) -src/meshic_pipeline/cli.py:60:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:62:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/cli.py:63:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/cli.py:68:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:70:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/cli.py:71:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/cli.py:72:80: E501 line too long (103 > 79 characters) -src/meshic_pipeline/cli.py:73:80: E501 line too long (109 > 79 characters) -src/meshic_pipeline/cli.py:74:80: E501 line too long (104 > 79 characters) -src/meshic_pipeline/cli.py:76:80: E501 line too long (106 > 79 characters) -src/meshic_pipeline/cli.py:77:80: E501 line too long (167 > 79 characters) -src/meshic_pipeline/cli.py:79:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:81:80: E501 line too long (105 > 79 characters) -src/meshic_pipeline/cli.py:82:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/cli.py:83:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/cli.py:85:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/cli.py:87:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/cli.py:94:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:105:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:107:80: E501 line too long (113 > 79 characters) -src/meshic_pipeline/cli.py:108:80: E501 line too long (126 > 79 characters) -src/meshic_pipeline/cli.py:109:80: E501 line too long (107 > 79 characters) -src/meshic_pipeline/cli.py:110:80: E501 line too long (113 > 79 characters) -src/meshic_pipeline/cli.py:112:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/cli.py:114:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/cli.py:121:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:123:80: E501 line too long (126 > 79 characters) -src/meshic_pipeline/cli.py:124:80: E501 line too long (107 > 79 characters) -src/meshic_pipeline/cli.py:125:80: E501 line too long (113 > 79 characters) -src/meshic_pipeline/cli.py:127:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/cli.py:129:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/cli.py:136:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:143:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:145:80: E501 line too long (113 > 79 characters) -src/meshic_pipeline/cli.py:146:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/cli.py:147:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/cli.py:148:80: E501 line too long (105 > 79 characters) -src/meshic_pipeline/cli.py:150:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/cli.py:152:61: W291 trailing whitespace -src/meshic_pipeline/cli.py:153:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/cli.py:158:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/cli.py:160:80: E501 line too long (134 > 79 characters) -src/meshic_pipeline/cli.py:161:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/cli.py:162:80: E501 line too long (105 > 79 characters) -src/meshic_pipeline/cli.py:164:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/cli.py:166:58: W291 trailing whitespace -src/meshic_pipeline/cli.py:172:1: E305 expected 2 blank lines after class or function definition, found 1 -src/meshic_pipeline/config.py:3:1: F401 'os' imported but unused -src/meshic_pipeline/config.py:5:1: F401 'typing.Tuple' imported but unused -src/meshic_pipeline/config.py:23:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/config.py:36:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/config.py:43:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/config.py:47:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/config.py:52:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/config.py:57:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/config.py:60:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/config.py:62:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/config.py:63:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/config.py:64:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/config.py:66:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/config.py:68:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/config.py:69:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/config.py:70:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/config.py:71:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/config.py:72:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/config.py:73:80: E501 line too long (95 > 79 characters) -src/meshic_pipeline/config.py:76:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/config.py:90:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/config.py:96:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/config.py:97:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/config.py:98:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/config.py:102:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/config.py:108:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/config.py:111:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/config.py:113:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/config.py:134:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/config.py:141:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/config.py:148:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/config.py:150:80: E501 line too long (108 > 79 characters) -src/meshic_pipeline/config.py:151:80: E501 line too long (100 > 79 characters) -src/meshic_pipeline/config.py:152:80: E501 line too long (100 > 79 characters) -src/meshic_pipeline/config.py:153:80: E501 line too long (108 > 79 characters) -src/meshic_pipeline/config.py:154:80: E501 line too long (109 > 79 characters) -src/meshic_pipeline/config.py:167:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/config.py:170:80: E501 line too long (99 > 79 characters) -src/meshic_pipeline/config.py:174:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/config.py:176:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/config.py:193:80: E501 line too long (150 > 79 characters) -src/meshic_pipeline/config.py:201:43: W291 trailing whitespace -src/meshic_pipeline/config.py:212:46: W291 trailing whitespace -src/meshic_pipeline/config.py:218:40: W291 trailing whitespace -src/meshic_pipeline/config.py:234:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/config.py:251:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/config.py:256:1: W293 blank line contains whitespace -src/meshic_pipeline/config.py:271:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/config.py:281:80: E501 line too long (101 > 79 characters) -src/meshic_pipeline/config.py:326:22: W291 trailing whitespace -src/meshic_pipeline/decoder/mvt_decoder.py:4:1: F401 'typing.Sequence' imported but unused -src/meshic_pipeline/decoder/mvt_decoder.py:6:1: F401 'json' imported but unused -src/meshic_pipeline/decoder/mvt_decoder.py:7:1: F401 'os' imported but unused -src/meshic_pipeline/decoder/mvt_decoder.py:22:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:26:71: W291 trailing whitespace -src/meshic_pipeline/decoder/mvt_decoder.py:29:1: W293 blank line contains whitespace -src/meshic_pipeline/decoder/mvt_decoder.py:30:80: E501 line too long (102 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:38:1: W293 blank line contains whitespace -src/meshic_pipeline/decoder/mvt_decoder.py:50:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:55:80: E501 line too long (112 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:76:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:84:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:92:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:138:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:146:1: W293 blank line contains whitespace -src/meshic_pipeline/decoder/mvt_decoder.py:150:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:154:1: W293 blank line contains whitespace -src/meshic_pipeline/decoder/mvt_decoder.py:161:1: W293 blank line contains whitespace -src/meshic_pipeline/decoder/mvt_decoder.py:167:1: W293 blank line contains whitespace -src/meshic_pipeline/decoder/mvt_decoder.py:171:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:177:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:178:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:183:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:187:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:189:1: W293 blank line contains whitespace -src/meshic_pipeline/decoder/mvt_decoder.py:192:1: W293 blank line contains whitespace -src/meshic_pipeline/decoder/mvt_decoder.py:196:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:210:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/decoder/mvt_decoder.py:216:1: W293 blank line contains whitespace -src/meshic_pipeline/discovery/tile_discovery.py:3:80: E501 line too long (172 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:19:1: F401 'shapely.wkb' imported but unused -src/meshic_pipeline/discovery/tile_discovery.py:30:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/discovery/tile_discovery.py:34:80: E501 line too long (96 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:37:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/discovery/tile_discovery.py:43:80: E501 line too long (96 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:49:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/discovery/tile_discovery.py:61:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/discovery/tile_discovery.py:73:80: E501 line too long (99 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:76:80: E501 line too long (105 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:77:80: E501 line too long (105 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:79:80: E501 line too long (112 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:85:80: E501 line too long (140 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:86:80: E501 line too long (113 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:87:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:88:80: E501 line too long (89 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:90:80: E501 line too long (98 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:91:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:100:80: E501 line too long (111 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:104:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/discovery/tile_discovery.py:111:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:114:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:121:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:136:80: E501 line too long (96 > 79 characters) -src/meshic_pipeline/discovery/tile_discovery.py:143:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/discovery/tile_discovery.py:148:24: W291 trailing whitespace -src/meshic_pipeline/discovery/tile_discovery.py:148:25: W292 no newline at end of file -src/meshic_pipeline/downloader/async_tile_downloader.py:37:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/downloader/async_tile_downloader.py:41:80: E501 line too long (89 > 79 characters) -src/meshic_pipeline/downloader/async_tile_downloader.py:74:1: W293 blank line contains whitespace -src/meshic_pipeline/downloader/async_tile_downloader.py:80:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/downloader/async_tile_downloader.py:83:1: W293 blank line contains whitespace -src/meshic_pipeline/downloader/async_tile_downloader.py:97:52: E261 at least two spaces before inline comment -src/meshic_pipeline/downloader/async_tile_downloader.py:98:1: W293 blank line contains whitespace -src/meshic_pipeline/downloader/async_tile_downloader.py:99:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/downloader/async_tile_downloader.py:107:1: W293 blank line contains whitespace -src/meshic_pipeline/downloader/async_tile_downloader.py:111:1: W293 blank line contains whitespace -src/meshic_pipeline/downloader/async_tile_downloader.py:114:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/downloader/async_tile_downloader.py:116:1: W293 blank line contains whitespace -src/meshic_pipeline/downloader/async_tile_downloader.py:120:1: W293 blank line contains whitespace -src/meshic_pipeline/downloader/async_tile_downloader.py:121:32: W291 trailing whitespace -src/meshic_pipeline/enrichment/__init__.py:1:1: W391 blank line at end of file -src/meshic_pipeline/enrichment/api_client.py:3:1: F401 'typing.Any' imported but unused -src/meshic_pipeline/enrichment/api_client.py:28:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:39:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:43:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:44:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:45:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:53:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:55:80: E501 line too long (95 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:68:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:86:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:95:80: E501 line too long (89 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:99:80: E501 line too long (106 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:118:80: E501 line too long (101 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:128:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:134:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:135:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:139:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:140:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:141:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:152:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:185:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:186:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:222:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:228:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:235:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:241:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/enrichment/api_client.py:264:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/enrichment/processor.py:2:1: F401 'aiohttp' imported but unused -src/meshic_pipeline/enrichment/processor.py:12:1: F401 'meshic_pipeline.persistence.db.get_async_db_engine' imported but unused -src/meshic_pipeline/enrichment/processor.py:13:1: F401 'sqlalchemy.ext.asyncio.async_sessionmaker' imported but unused -src/meshic_pipeline/enrichment/processor.py:20:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/enrichment/processor.py:26:1: W293 blank line contains whitespace -src/meshic_pipeline/enrichment/processor.py:28:33: E203 whitespace before ':' -src/meshic_pipeline/enrichment/processor.py:30:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/enrichment/processor.py:33:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/enrichment/processor.py:34:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/enrichment/processor.py:45:1: W293 blank line contains whitespace -src/meshic_pipeline/enrichment/processor.py:47:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/enrichment/processor.py:48:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/enrichment/processor.py:49:80: E501 line too long (89 > 79 characters) -src/meshic_pipeline/enrichment/processor.py:56:80: E501 line too long (110 > 79 characters) -src/meshic_pipeline/enrichment/processor.py:57:80: E501 line too long (106 > 79 characters) -src/meshic_pipeline/enrichment/processor.py:60:65: W291 trailing whitespace -src/meshic_pipeline/enrichment/strategies.py:14:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/enrichment/strategies.py:30:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/enrichment/strategies.py:39:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/enrichment/strategies.py:43:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:48:1: W293 blank line contains whitespace -src/meshic_pipeline/enrichment/strategies.py:50:80: E501 line too long (99 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:51:80: E501 line too long (109 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:71:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/enrichment/strategies.py:88:80: E501 line too long (89 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:92:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/enrichment/strategies.py:99:1: W293 blank line contains whitespace -src/meshic_pipeline/enrichment/strategies.py:101:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:121:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:124:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/enrichment/strategies.py:129:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:133:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:134:80: E501 line too long (104 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:135:80: E501 line too long (100 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:136:80: E501 line too long (144 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:137:80: E501 line too long (115 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:144:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:145:80: E501 line too long (119 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:146:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:151:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/enrichment/strategies.py:152:25: W291 trailing whitespace -src/meshic_pipeline/enrichment/strategies.py:159:1: W293 blank line contains whitespace -src/meshic_pipeline/enrichment/strategies.py:164:1: W293 blank line contains whitespace -src/meshic_pipeline/enrichment/strategies.py:166:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:181:80: E501 line too long (101 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:186:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:189:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/enrichment/strategies.py:196:1: W293 blank line contains whitespace -src/meshic_pipeline/enrichment/strategies.py:221:21: F541 f-string is missing placeholders -src/meshic_pipeline/enrichment/strategies.py:222:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/enrichment/strategies.py:228:22: W291 trailing whitespace -src/meshic_pipeline/exceptions.py:19:1: F401 'typing.List' imported but unused -src/meshic_pipeline/exceptions.py:19:1: F401 'typing.Union' imported but unused -src/meshic_pipeline/exceptions.py:74:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:110:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:122:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:134:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:146:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:159:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:171:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:183:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:206:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:219:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:254:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/exceptions.py:255:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/exceptions.py:262:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/exceptions.py:286:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:304:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:316:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:321:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:329:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/exceptions.py:335:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:337:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/exceptions.py:338:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/exceptions.py:349:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:352:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:356:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:364:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/exceptions.py:370:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:372:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/exceptions.py:373:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/exceptions.py:385:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:388:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:394:1: W293 blank line contains whitespace -src/meshic_pipeline/exceptions.py:411:21: W291 trailing whitespace -src/meshic_pipeline/geometry/stitcher.py:9:1: F401 'shapely.geometry.Polygon' imported but unused -src/meshic_pipeline/geometry/stitcher.py:10:1: F401 'shapely.errors.GEOSException' imported but unused -src/meshic_pipeline/geometry/stitcher.py:11:1: F401 'shapely.ops.snap' imported but unused -src/meshic_pipeline/geometry/stitcher.py:12:1: F401 'tqdm.tqdm' imported but unused -src/meshic_pipeline/geometry/stitcher.py:13:1: F401 'mercantile' imported but unused -src/meshic_pipeline/geometry/stitcher.py:14:1: F401 'sqlalchemy.text' imported but unused -src/meshic_pipeline/geometry/stitcher.py:55:80: E501 line too long (108 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:68:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:92:80: E501 line too long (103 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:132:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:148:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:150:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:157:80: E501 line too long (117 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:163:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:169:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:174:80: E501 line too long (101 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:176:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:181:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:186:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:214:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:218:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/geometry/stitcher.py:223:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/geometry/validator.py:20:38: W291 trailing whitespace -src/meshic_pipeline/logging_utils.py:18:1: F401 'asyncio' imported but unused -src/meshic_pipeline/logging_utils.py:24:1: F401 'typing.Union' imported but unused -src/meshic_pipeline/logging_utils.py:27:1: F401 'weakref' imported but unused -src/meshic_pipeline/logging_utils.py:38:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:50:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:54:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:57:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:60:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:63:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:67:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:73:80: E501 line too long (122 > 79 characters) -src/meshic_pipeline/logging_utils.py:75:80: E501 line too long (111 > 79 characters) -src/meshic_pipeline/logging_utils.py:77:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/logging_utils.py:83:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:84:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/logging_utils.py:91:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:96:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:105:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:109:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:115:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:121:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/logging_utils.py:123:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:128:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:136:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:143:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:147:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:152:47: W291 trailing whitespace -src/meshic_pipeline/logging_utils.py:155:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:163:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/logging_utils.py:164:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:167:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/logging_utils.py:169:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:174:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:181:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:185:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:194:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:208:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:218:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:233:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:234:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/logging_utils.py:246:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:252:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:256:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/logging_utils.py:258:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:267:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/logging_utils.py:268:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/logging_utils.py:274:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:280:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:284:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:290:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:293:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/logging_utils.py:294:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:297:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:301:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:305:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:309:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:313:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:317:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:322:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:325:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/logging_utils.py:332:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:351:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:358:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:361:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:367:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:369:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/logging_utils.py:372:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:376:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:379:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:381:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/logging_utils.py:383:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:392:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/logging_utils.py:393:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/logging_utils.py:394:80: E501 line too long (101 > 79 characters) -src/meshic_pipeline/logging_utils.py:406:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:408:29: W291 trailing whitespace -src/meshic_pipeline/logging_utils.py:409:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/logging_utils.py:411:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:416:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:427:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:430:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/logging_utils.py:432:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:441:80: E501 line too long (99 > 79 characters) -src/meshic_pipeline/logging_utils.py:442:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:448:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:449:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/logging_utils.py:451:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:466:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:471:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:474:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:476:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/logging_utils.py:478:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:489:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/logging_utils.py:491:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:495:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/logging_utils.py:497:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:499:9: F811 redefinition of unused 'asyncio' from line 18 -src/meshic_pipeline/logging_utils.py:504:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:508:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/logging_utils.py:514:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:519:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:520:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/logging_utils.py:521:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:527:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/logging_utils.py:537:1: W293 blank line contains whitespace -src/meshic_pipeline/logging_utils.py:543:16: W291 trailing whitespace -src/meshic_pipeline/memory_utils.py:47:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:56:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:62:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/memory_utils.py:64:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/memory_utils.py:66:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:69:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:75:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/memory_utils.py:79:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:85:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:87:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:92:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:97:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:101:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:106:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:109:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:114:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:117:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:120:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:130:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:132:80: E501 line too long (107 > 79 characters) -src/meshic_pipeline/memory_utils.py:135:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:137:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:143:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:144:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/memory_utils.py:150:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:154:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:160:56: W291 trailing whitespace -src/meshic_pipeline/memory_utils.py:163:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:169:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:171:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:175:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:181:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:182:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/memory_utils.py:184:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:186:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:205:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/memory_utils.py:208:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:210:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:213:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:218:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/memory_utils.py:223:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:224:5: F841 local variable 'e' is assigned to but never used -src/meshic_pipeline/memory_utils.py:229:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:233:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:236:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/memory_utils.py:245:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/memory_utils.py:251:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:255:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:258:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:262:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:266:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:269:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:276:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:280:80: E501 line too long (108 > 79 characters) -src/meshic_pipeline/memory_utils.py:284:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:286:80: E501 line too long (95 > 79 characters) -src/meshic_pipeline/memory_utils.py:288:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:291:80: E501 line too long (99 > 79 characters) -src/meshic_pipeline/memory_utils.py:292:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:300:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:303:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:306:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/memory_utils.py:307:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/memory_utils.py:309:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:314:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/memory_utils.py:316:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:329:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:332:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:339:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/memory_utils.py:342:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:344:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/memory_utils.py:345:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:350:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:353:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:358:1: W293 blank line contains whitespace -src/meshic_pipeline/memory_utils.py:362:14: W291 trailing whitespace -src/meshic_pipeline/persistence/db.py:16:21: F821 undefined name 'settings' -src/meshic_pipeline/persistence/db.py:16:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/persistence/db.py:19:19: F821 undefined name 'settings' -src/meshic_pipeline/persistence/db.py:20:22: F821 undefined name 'settings' -src/meshic_pipeline/persistence/db.py:20:50: F821 undefined name 'settings' -src/meshic_pipeline/persistence/db.py:21:23: F821 undefined name 'settings' -src/meshic_pipeline/persistence/db.py:22:22: F821 undefined name 'settings' -src/meshic_pipeline/persistence/db.py:23:14: F821 undefined name 'settings' -src/meshic_pipeline/persistence/db.py:25:22: F821 undefined name 'settings' -src/meshic_pipeline/persistence/db.py:27:32: F821 undefined name 'settings' -src/meshic_pipeline/persistence/db.py:52:80: E501 line too long (114 > 79 characters) -src/meshic_pipeline/persistence/db.py:57:80: E501 line too long (109 > 79 characters) -src/meshic_pipeline/persistence/db.py:72:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/persistence/db.py:73:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/persistence/db.py:78:25: W291 trailing whitespace -src/meshic_pipeline/persistence/db.py:78:26: W292 no newline at end of file -src/meshic_pipeline/persistence/enrichment_persister.py:5:1: F401 'math.ceil' imported but unused -src/meshic_pipeline/persistence/enrichment_persister.py:19:80: E501 line too long (112 > 79 characters) -src/meshic_pipeline/persistence/enrichment_persister.py:46:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/persistence/enrichment_persister.py:82:80: E501 line too long (102 > 79 characters) -src/meshic_pipeline/persistence/enrichment_persister.py:105:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/persistence/enrichment_persister.py:120:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/persistence/enrichment_persister.py:136:48: W291 trailing whitespace -src/meshic_pipeline/persistence/models.py:2:1: F401 'sqlalchemy.create_engine' imported but unused -src/meshic_pipeline/persistence/models.py:2:1: F401 'sqlalchemy.select' imported but unused -src/meshic_pipeline/persistence/models.py:2:1: F401 'sqlalchemy.update' imported but unused -src/meshic_pipeline/persistence/models.py:18:1: F401 'sqlalchemy.orm.load_only' imported but unused -src/meshic_pipeline/persistence/models.py:35:80: E501 line too long (100 > 79 characters) -src/meshic_pipeline/persistence/models.py:43:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/persistence/models.py:50:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/persistence/models.py:67:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/persistence/models.py:73:1: W293 blank line contains whitespace -src/meshic_pipeline/persistence/models.py:77:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/persistence/models.py:84:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/persistence/models.py:89:80: E501 line too long (100 > 79 characters) -src/meshic_pipeline/persistence/models.py:90:1: W293 blank line contains whitespace -src/meshic_pipeline/persistence/models.py:96:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/persistence/models.py:103:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/persistence/models.py:121:1: W293 blank line contains whitespace -src/meshic_pipeline/persistence/models.py:125:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/persistence/models.py:127:1: W293 blank line contains whitespace -src/meshic_pipeline/persistence/models.py:128:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:135:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/persistence/models.py:144:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/persistence/models.py:147:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:179:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:189:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/persistence/models.py:193:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:198:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:202:1: W293 blank line contains whitespace -src/meshic_pipeline/persistence/models.py:205:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:210:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:217:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:225:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:233:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:241:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:248:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:255:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:261:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:267:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/models.py:280:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/persistence/models.py:301:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/persistence/models.py:303:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/persistence/models.py:306:80: E501 line too long (102 > 79 characters) -src/meshic_pipeline/persistence/models.py:312:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/persistence/models.py:313:80: E501 line too long (194 > 79 characters) -src/meshic_pipeline/persistence/models.py:324:33: E711 comparison to None should be 'if cond is not None:' -src/meshic_pipeline/persistence/models.py:326:80: E501 line too long (113 > 79 characters) -src/meshic_pipeline/persistence/models.py:328:23: W291 trailing whitespace -src/meshic_pipeline/persistence/models.py:328:24: W292 no newline at end of file -src/meshic_pipeline/persistence/postgis_persister.py:26:1: E305 expected 2 blank lines after class or function definition, found 1 -src/meshic_pipeline/persistence/postgis_persister.py:131:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/persistence/postgis_persister.py:134:71: W291 trailing whitespace -src/meshic_pipeline/persistence/postgis_persister.py:137:1: W293 blank line contains whitespace -src/meshic_pipeline/persistence/postgis_persister.py:142:1: W293 blank line contains whitespace -src/meshic_pipeline/persistence/postgis_persister.py:143:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:151:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:160:80: E501 line too long (99 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:162:80: E501 line too long (114 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:163:80: E501 line too long (101 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:172:80: E501 line too long (120 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:179:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:183:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:205:80: E501 line too long (100 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:207:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:209:80: E501 line too long (100 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:211:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:217:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:219:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:221:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:248:80: E501 line too long (99 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:250:80: E501 line too long (96 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:258:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:259:80: E501 line too long (96 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:265:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:268:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:275:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:276:1: W293 blank line contains whitespace -src/meshic_pipeline/persistence/postgis_persister.py:279:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:282:80: E501 line too long (98 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:284:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:291:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:295:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:309:80: E501 line too long (115 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:310:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:312:80: E501 line too long (101 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:314:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:326:80: E501 line too long (126 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:340:80: E501 line too long (107 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:360:80: E501 line too long (111 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:362:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:363:80: E501 line too long (104 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:366:80: E501 line too long (121 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:368:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:375:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:377:80: E501 line too long (96 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:402:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:415:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:424:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/persistence/postgis_persister.py:429:36: W291 trailing whitespace -src/meshic_pipeline/persistence/table_management.py:7:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/persistence/table_management.py:16:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/persistence/table_management.py:18:75: W291 trailing whitespace -src/meshic_pipeline/persistence/table_management.py:18:76: W292 no newline at end of file -src/meshic_pipeline/pipeline_orchestrator.py:4:1: F401 'typing.Dict' imported but unused -src/meshic_pipeline/pipeline_orchestrator.py:6:1: F401 'concurrent.futures' imported but unused -src/meshic_pipeline/pipeline_orchestrator.py:12:1: F401 'sqlalchemy.create_engine' imported but unused -src/meshic_pipeline/pipeline_orchestrator.py:22:1: F401 'sqlalchemy.text' imported but unused -src/meshic_pipeline/pipeline_orchestrator.py:77:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:82:80: E501 line too long (107 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:85:80: E501 line too long (110 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:109:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:114:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:171:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:172:80: E501 line too long (89 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:210:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:211:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:229:80: E501 line too long (98 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:231:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:252:5: F841 local variable 'stitcher' is assigned to but never used -src/meshic_pipeline/pipeline_orchestrator.py:252:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:264:80: E501 line too long (104 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:271:80: E501 line too long (101 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:274:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:295:80: E501 line too long (141 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:296:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:299:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:304:80: E501 line too long (96 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:305:80: E501 line too long (108 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:310:80: E501 line too long (111 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:315:80: E501 line too long (109 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:316:80: E501 line too long (126 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:333:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:340:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:354:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:359:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:365:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:369:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:378:80: E501 line too long (112 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:381:80: E501 line too long (119 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:388:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:394:80: E501 line too long (114 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:399:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/pipeline_orchestrator.py:409:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:10:1: F401 'typing.Optional' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:11:1: F401 'sqlalchemy.create_engine' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:12:1: F401 'sqlalchemy.ext.asyncio.create_async_engine' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:12:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:13:1: F401 'sqlalchemy.dialects.postgresql.insert as pg_insert' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:14:1: F401 'datetime.timedelta' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:23:1: F401 'meshic_pipeline.persistence.models.Base' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:23:1: F401 'meshic_pipeline.persistence.models.Transaction' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:23:1: F401 'meshic_pipeline.persistence.models.ParcelPriceMetric' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:23:1: F401 'meshic_pipeline.persistence.models.BuildingRule' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:23:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:29:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:30:1: F401 'logging' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:30:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.BigInteger' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.Column' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.DateTime' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.Float' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.Integer' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.String' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.JSON' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:31:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:31:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:32:1: F401 'sqlalchemy.orm.sessionmaker' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:32:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:33:1: F401 'sqlalchemy.ForeignKey' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:33:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:34:1: F401 'meshic_pipeline.exceptions.ValidationException' imported but unused -src/meshic_pipeline/run_enrichment_pipeline.py:34:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:35:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:36:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:43:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:44:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:48:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:51:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:52:1: E402 module level import not at top of file -src/meshic_pipeline/run_enrichment_pipeline.py:72:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_enrichment_pipeline.py:75:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:81:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:85:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:91:1: W293 blank line contains whitespace -src/meshic_pipeline/run_enrichment_pipeline.py:99:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:107:80: E501 line too long (127 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:111:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_enrichment_pipeline.py:115:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:117:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_enrichment_pipeline.py:122:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:126:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:128:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_enrichment_pipeline.py:134:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:144:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_enrichment_pipeline.py:152:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:155:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_enrichment_pipeline.py:160:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:162:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:165:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:168:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:171:80: E501 line too long (117 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:173:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:176:1: W293 blank line contains whitespace -src/meshic_pipeline/run_enrichment_pipeline.py:187:80: E501 line too long (89 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:193:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:208:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:209:80: E501 line too long (89 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:213:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:223:80: E501 line too long (104 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:225:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:226:80: E501 line too long (106 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:230:80: E501 line too long (89 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:231:80: E501 line too long (101 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:234:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:239:32: F541 f-string is missing placeholders -src/meshic_pipeline/run_enrichment_pipeline.py:240:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:243:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:244:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:245:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:251:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:263:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:264:80: E501 line too long (112 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:294:80: E501 line too long (99 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:300:80: E501 line too long (98 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:303:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:305:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:306:1: W293 blank line contains whitespace -src/meshic_pipeline/run_enrichment_pipeline.py:309:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_enrichment_pipeline.py:311:80: E501 line too long (105 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:312:80: E501 line too long (108 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:313:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:314:80: E501 line too long (82 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:317:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:318:1: W293 blank line contains whitespace -src/meshic_pipeline/run_enrichment_pipeline.py:323:1: W293 blank line contains whitespace -src/meshic_pipeline/run_enrichment_pipeline.py:327:1: W293 blank line contains whitespace -src/meshic_pipeline/run_enrichment_pipeline.py:330:80: E501 line too long (89 > 79 characters) -src/meshic_pipeline/run_enrichment_pipeline.py:332:1: W293 blank line contains whitespace -src/meshic_pipeline/run_enrichment_pipeline.py:337:1: W293 blank line contains whitespace -src/meshic_pipeline/run_enrichment_pipeline.py:341:1: W293 blank line contains whitespace -src/meshic_pipeline/run_enrichment_pipeline.py:347:1: W293 blank line contains whitespace -src/meshic_pipeline/run_enrichment_pipeline.py:355:1: E305 expected 2 blank lines after class or function definition, found 1 -src/meshic_pipeline/run_enrichment_pipeline.py:356:10: W291 trailing whitespace -src/meshic_pipeline/run_geometric_pipeline.py:3:1: F401 'subprocess' imported but unused -src/meshic_pipeline/run_geometric_pipeline.py:4:1: F401 'sys' imported but unused -src/meshic_pipeline/run_geometric_pipeline.py:5:1: F401 'typing.Tuple' imported but unused -src/meshic_pipeline/run_geometric_pipeline.py:17:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_geometric_pipeline.py:23:80: E501 line too long (100 > 79 characters) -src/meshic_pipeline/run_geometric_pipeline.py:29:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/run_geometric_pipeline.py:48:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/run_geometric_pipeline.py:63:1: W293 blank line contains whitespace -src/meshic_pipeline/run_geometric_pipeline.py:65:51: W291 trailing whitespace -src/meshic_pipeline/run_geometric_pipeline.py:77:80: E501 line too long (86 > 79 characters) -src/meshic_pipeline/run_geometric_pipeline.py:79:1: W293 blank line contains whitespace -src/meshic_pipeline/run_geometric_pipeline.py:83:1: W293 blank line contains whitespace -src/meshic_pipeline/run_geometric_pipeline.py:105:1: E305 expected 2 blank lines after class or function definition, found 1 -src/meshic_pipeline/run_geometric_pipeline.py:106:10: W291 trailing whitespace -src/meshic_pipeline/run_monitoring.py:4:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/run_monitoring.py:17:1: E402 module level import not at top of file -src/meshic_pipeline/run_monitoring.py:21:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_monitoring.py:25:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:28:80: E501 line too long (118 > 79 characters) -src/meshic_pipeline/run_monitoring.py:29:80: E501 line too long (123 > 79 characters) -src/meshic_pipeline/run_monitoring.py:30:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/run_monitoring.py:31:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:34:19: W291 trailing whitespace -src/meshic_pipeline/run_monitoring.py:35:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/run_monitoring.py:36:80: E501 line too long (92 > 79 characters) -src/meshic_pipeline/run_monitoring.py:37:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/run_monitoring.py:38:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/run_monitoring.py:41:32: W291 trailing whitespace -src/meshic_pipeline/run_monitoring.py:44:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:47:15: F541 f-string is missing placeholders -src/meshic_pipeline/run_monitoring.py:53:15: F541 f-string is missing placeholders -src/meshic_pipeline/run_monitoring.py:55:80: E501 line too long (99 > 79 characters) -src/meshic_pipeline/run_monitoring.py:57:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:59:19: F541 f-string is missing placeholders -src/meshic_pipeline/run_monitoring.py:61:65: W291 trailing whitespace -src/meshic_pipeline/run_monitoring.py:66:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:68:80: E501 line too long (107 > 79 characters) -src/meshic_pipeline/run_monitoring.py:70:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_monitoring.py:74:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:76:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/run_monitoring.py:77:9: F841 local variable 'trigger_parcels' is assigned to but never used -src/meshic_pipeline/run_monitoring.py:79:80: E501 line too long (84 > 79 characters) -src/meshic_pipeline/run_monitoring.py:82:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:84:80: E501 line too long (121 > 79 characters) -src/meshic_pipeline/run_monitoring.py:85:80: E501 line too long (121 > 79 characters) -src/meshic_pipeline/run_monitoring.py:87:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:92:40: W291 trailing whitespace -src/meshic_pipeline/run_monitoring.py:95:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:96:48: W291 trailing whitespace -src/meshic_pipeline/run_monitoring.py:100:40: W291 trailing whitespace -src/meshic_pipeline/run_monitoring.py:103:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:106:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:108:80: E501 line too long (104 > 79 characters) -src/meshic_pipeline/run_monitoring.py:109:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:110:15: F541 f-string is missing placeholders -src/meshic_pipeline/run_monitoring.py:112:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/run_monitoring.py:115:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:118:80: E501 line too long (98 > 79 characters) -src/meshic_pipeline/run_monitoring.py:119:80: E501 line too long (114 > 79 characters) -src/meshic_pipeline/run_monitoring.py:120:19: F541 f-string is missing placeholders -src/meshic_pipeline/run_monitoring.py:120:80: E501 line too long (108 > 79 characters) -src/meshic_pipeline/run_monitoring.py:121:80: E501 line too long (97 > 79 characters) -src/meshic_pipeline/run_monitoring.py:125:19: F541 f-string is missing placeholders -src/meshic_pipeline/run_monitoring.py:125:80: E501 line too long (108 > 79 characters) -src/meshic_pipeline/run_monitoring.py:127:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:131:19: F541 f-string is missing placeholders -src/meshic_pipeline/run_monitoring.py:132:19: F541 f-string is missing placeholders -src/meshic_pipeline/run_monitoring.py:132:80: E501 line too long (111 > 79 characters) -src/meshic_pipeline/run_monitoring.py:135:77: W291 trailing whitespace -src/meshic_pipeline/run_monitoring.py:136:19: F541 f-string is missing placeholders -src/meshic_pipeline/run_monitoring.py:136:80: E501 line too long (112 > 79 characters) -src/meshic_pipeline/run_monitoring.py:138:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:143:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/run_monitoring.py:149:1: W293 blank line contains whitespace -src/meshic_pipeline/run_monitoring.py:152:80: E501 line too long (118 > 79 characters) -src/meshic_pipeline/run_monitoring.py:154:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_monitoring.py:155:80: E501 line too long (94 > 79 characters) -src/meshic_pipeline/run_monitoring.py:156:80: E501 line too long (98 > 79 characters) -src/meshic_pipeline/run_monitoring.py:158:80: E501 line too long (88 > 79 characters) -src/meshic_pipeline/run_monitoring.py:166:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_monitoring.py:166:15: W291 trailing whitespace -src/meshic_pipeline/run_monitoring.py:175:80: E501 line too long (87 > 79 characters) -src/meshic_pipeline/run_monitoring.py:176:80: E501 line too long (91 > 79 characters) -src/meshic_pipeline/run_monitoring.py:180:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/run_monitoring.py:181:80: E501 line too long (85 > 79 characters) -src/meshic_pipeline/run_monitoring.py:189:80: E501 line too long (81 > 79 characters) -src/meshic_pipeline/run_monitoring.py:191:1: E305 expected 2 blank lines after class or function definition, found 1 -src/meshic_pipeline/run_monitoring.py:192:10: W291 trailing whitespace -src/meshic_pipeline/run_tile_pipeline.py:14:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_tile_pipeline.py:28:80: E501 line too long (105 > 79 characters) -src/meshic_pipeline/run_tile_pipeline.py:30:80: E501 line too long (95 > 79 characters) -src/meshic_pipeline/run_tile_pipeline.py:37:80: E501 line too long (93 > 79 characters) -src/meshic_pipeline/run_tile_pipeline.py:39:80: E501 line too long (83 > 79 characters) -src/meshic_pipeline/run_tile_pipeline.py:42:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/run_tile_pipeline.py:51:80: E501 line too long (107 > 79 characters) -src/meshic_pipeline/run_tile_pipeline.py:59:1: E305 expected 2 blank lines after class or function definition, found 1 -src/meshic_pipeline/run_tile_pipeline.py:60:11: W291 trailing whitespace -src/meshic_pipeline/run_tile_pipeline.py:60:12: W292 no newline at end of file -src/meshic_pipeline/show_discovery_summary.py:4:80: E501 line too long (80 > 79 characters) -src/meshic_pipeline/show_discovery_summary.py:12:1: E402 module level import not at top of file -src/meshic_pipeline/show_discovery_summary.py:13:1: E402 module level import not at top of file -src/meshic_pipeline/show_discovery_summary.py:15:1: E302 expected 2 blank lines, found 1 -src/meshic_pipeline/show_discovery_summary.py:19:1: W293 blank line contains whitespace -src/meshic_pipeline/show_discovery_summary.py:21:80: E501 line too long (107 > 79 characters) -src/meshic_pipeline/show_discovery_summary.py:27:1: W293 blank line contains whitespace -src/meshic_pipeline/show_discovery_summary.py:28:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:31:1: W293 blank line contains whitespace -src/meshic_pipeline/show_discovery_summary.py:32:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:35:1: W293 blank line contains whitespace -src/meshic_pipeline/show_discovery_summary.py:41:1: W293 blank line contains whitespace -src/meshic_pipeline/show_discovery_summary.py:42:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:43:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:44:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:45:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:46:1: W293 blank line contains whitespace -src/meshic_pipeline/show_discovery_summary.py:47:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:48:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:49:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:50:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:51:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:52:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:53:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:54:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:55:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:55:80: E501 line too long (90 > 79 characters) -src/meshic_pipeline/show_discovery_summary.py:56:1: W293 blank line contains whitespace -src/meshic_pipeline/show_discovery_summary.py:57:11: F541 f-string is missing placeholders -src/meshic_pipeline/show_discovery_summary.py:59:1: E305 expected 2 blank lines after class or function definition, found 1 -src/meshic_pipeline/show_discovery_summary.py:60:11: W291 trailing whitespace -src/meshic_pipeline/utils/tile_list_generator.py:10:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/__init__.py:1:62: W291 trailing whitespace +src/suhail_pipeline/cli.py:13:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:43:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:45:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/cli.py:46:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/cli.py:48:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/cli.py:51:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:53:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/cli.py:54:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/cli.py:55:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/cli.py:57:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/cli.py:58:80: E501 line too long (101 > 79 characters) +src/suhail_pipeline/cli.py:60:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:62:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/cli.py:63:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/cli.py:68:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:70:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/cli.py:71:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/cli.py:72:80: E501 line too long (103 > 79 characters) +src/suhail_pipeline/cli.py:73:80: E501 line too long (109 > 79 characters) +src/suhail_pipeline/cli.py:74:80: E501 line too long (104 > 79 characters) +src/suhail_pipeline/cli.py:76:80: E501 line too long (106 > 79 characters) +src/suhail_pipeline/cli.py:77:80: E501 line too long (167 > 79 characters) +src/suhail_pipeline/cli.py:79:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:81:80: E501 line too long (105 > 79 characters) +src/suhail_pipeline/cli.py:82:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/cli.py:83:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/cli.py:85:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/cli.py:87:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/cli.py:94:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:105:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:107:80: E501 line too long (113 > 79 characters) +src/suhail_pipeline/cli.py:108:80: E501 line too long (126 > 79 characters) +src/suhail_pipeline/cli.py:109:80: E501 line too long (107 > 79 characters) +src/suhail_pipeline/cli.py:110:80: E501 line too long (113 > 79 characters) +src/suhail_pipeline/cli.py:112:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/cli.py:114:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/cli.py:121:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:123:80: E501 line too long (126 > 79 characters) +src/suhail_pipeline/cli.py:124:80: E501 line too long (107 > 79 characters) +src/suhail_pipeline/cli.py:125:80: E501 line too long (113 > 79 characters) +src/suhail_pipeline/cli.py:127:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/cli.py:129:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/cli.py:136:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:143:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:145:80: E501 line too long (113 > 79 characters) +src/suhail_pipeline/cli.py:146:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/cli.py:147:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/cli.py:148:80: E501 line too long (105 > 79 characters) +src/suhail_pipeline/cli.py:150:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/cli.py:152:61: W291 trailing whitespace +src/suhail_pipeline/cli.py:153:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/cli.py:158:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/cli.py:160:80: E501 line too long (134 > 79 characters) +src/suhail_pipeline/cli.py:161:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/cli.py:162:80: E501 line too long (105 > 79 characters) +src/suhail_pipeline/cli.py:164:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/cli.py:166:58: W291 trailing whitespace +src/suhail_pipeline/cli.py:172:1: E305 expected 2 blank lines after class or function definition, found 1 +src/suhail_pipeline/config.py:3:1: F401 'os' imported but unused +src/suhail_pipeline/config.py:5:1: F401 'typing.Tuple' imported but unused +src/suhail_pipeline/config.py:23:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/config.py:36:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/config.py:43:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/config.py:47:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/config.py:52:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/config.py:57:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/config.py:60:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/config.py:62:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/config.py:63:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/config.py:64:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/config.py:66:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/config.py:68:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/config.py:69:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/config.py:70:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/config.py:71:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/config.py:72:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/config.py:73:80: E501 line too long (95 > 79 characters) +src/suhail_pipeline/config.py:76:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/config.py:90:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/config.py:96:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/config.py:97:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/config.py:98:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/config.py:102:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/config.py:108:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/config.py:111:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/config.py:113:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/config.py:134:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/config.py:141:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/config.py:148:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/config.py:150:80: E501 line too long (108 > 79 characters) +src/suhail_pipeline/config.py:151:80: E501 line too long (100 > 79 characters) +src/suhail_pipeline/config.py:152:80: E501 line too long (100 > 79 characters) +src/suhail_pipeline/config.py:153:80: E501 line too long (108 > 79 characters) +src/suhail_pipeline/config.py:154:80: E501 line too long (109 > 79 characters) +src/suhail_pipeline/config.py:167:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/config.py:170:80: E501 line too long (99 > 79 characters) +src/suhail_pipeline/config.py:174:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/config.py:176:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/config.py:193:80: E501 line too long (150 > 79 characters) +src/suhail_pipeline/config.py:201:43: W291 trailing whitespace +src/suhail_pipeline/config.py:212:46: W291 trailing whitespace +src/suhail_pipeline/config.py:218:40: W291 trailing whitespace +src/suhail_pipeline/config.py:234:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/config.py:251:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/config.py:256:1: W293 blank line contains whitespace +src/suhail_pipeline/config.py:271:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/config.py:281:80: E501 line too long (101 > 79 characters) +src/suhail_pipeline/config.py:326:22: W291 trailing whitespace +src/suhail_pipeline/decoder/mvt_decoder.py:4:1: F401 'typing.Sequence' imported but unused +src/suhail_pipeline/decoder/mvt_decoder.py:6:1: F401 'json' imported but unused +src/suhail_pipeline/decoder/mvt_decoder.py:7:1: F401 'os' imported but unused +src/suhail_pipeline/decoder/mvt_decoder.py:22:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:26:71: W291 trailing whitespace +src/suhail_pipeline/decoder/mvt_decoder.py:29:1: W293 blank line contains whitespace +src/suhail_pipeline/decoder/mvt_decoder.py:30:80: E501 line too long (102 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:38:1: W293 blank line contains whitespace +src/suhail_pipeline/decoder/mvt_decoder.py:50:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:55:80: E501 line too long (112 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:76:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:84:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:92:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:138:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:146:1: W293 blank line contains whitespace +src/suhail_pipeline/decoder/mvt_decoder.py:150:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:154:1: W293 blank line contains whitespace +src/suhail_pipeline/decoder/mvt_decoder.py:161:1: W293 blank line contains whitespace +src/suhail_pipeline/decoder/mvt_decoder.py:167:1: W293 blank line contains whitespace +src/suhail_pipeline/decoder/mvt_decoder.py:171:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:177:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:178:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:183:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:187:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:189:1: W293 blank line contains whitespace +src/suhail_pipeline/decoder/mvt_decoder.py:192:1: W293 blank line contains whitespace +src/suhail_pipeline/decoder/mvt_decoder.py:196:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:210:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/decoder/mvt_decoder.py:216:1: W293 blank line contains whitespace +src/suhail_pipeline/discovery/tile_discovery.py:3:80: E501 line too long (172 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:19:1: F401 'shapely.wkb' imported but unused +src/suhail_pipeline/discovery/tile_discovery.py:30:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/discovery/tile_discovery.py:34:80: E501 line too long (96 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:37:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/discovery/tile_discovery.py:43:80: E501 line too long (96 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:49:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/discovery/tile_discovery.py:61:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/discovery/tile_discovery.py:73:80: E501 line too long (99 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:76:80: E501 line too long (105 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:77:80: E501 line too long (105 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:79:80: E501 line too long (112 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:85:80: E501 line too long (140 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:86:80: E501 line too long (113 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:87:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:88:80: E501 line too long (89 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:90:80: E501 line too long (98 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:91:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:100:80: E501 line too long (111 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:104:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/discovery/tile_discovery.py:111:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:114:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:121:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:136:80: E501 line too long (96 > 79 characters) +src/suhail_pipeline/discovery/tile_discovery.py:143:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/discovery/tile_discovery.py:148:24: W291 trailing whitespace +src/suhail_pipeline/discovery/tile_discovery.py:148:25: W292 no newline at end of file +src/suhail_pipeline/downloader/async_tile_downloader.py:37:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/downloader/async_tile_downloader.py:41:80: E501 line too long (89 > 79 characters) +src/suhail_pipeline/downloader/async_tile_downloader.py:74:1: W293 blank line contains whitespace +src/suhail_pipeline/downloader/async_tile_downloader.py:80:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/downloader/async_tile_downloader.py:83:1: W293 blank line contains whitespace +src/suhail_pipeline/downloader/async_tile_downloader.py:97:52: E261 at least two spaces before inline comment +src/suhail_pipeline/downloader/async_tile_downloader.py:98:1: W293 blank line contains whitespace +src/suhail_pipeline/downloader/async_tile_downloader.py:99:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/downloader/async_tile_downloader.py:107:1: W293 blank line contains whitespace +src/suhail_pipeline/downloader/async_tile_downloader.py:111:1: W293 blank line contains whitespace +src/suhail_pipeline/downloader/async_tile_downloader.py:114:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/downloader/async_tile_downloader.py:116:1: W293 blank line contains whitespace +src/suhail_pipeline/downloader/async_tile_downloader.py:120:1: W293 blank line contains whitespace +src/suhail_pipeline/downloader/async_tile_downloader.py:121:32: W291 trailing whitespace +src/suhail_pipeline/enrichment/__init__.py:1:1: W391 blank line at end of file +src/suhail_pipeline/enrichment/api_client.py:3:1: F401 'typing.Any' imported but unused +src/suhail_pipeline/enrichment/api_client.py:28:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:39:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:43:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:44:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:45:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:53:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:55:80: E501 line too long (95 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:68:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:86:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:95:80: E501 line too long (89 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:99:80: E501 line too long (106 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:118:80: E501 line too long (101 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:128:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:134:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:135:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:139:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:140:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:141:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:152:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:185:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:186:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:222:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:228:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:235:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:241:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/enrichment/api_client.py:264:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/enrichment/processor.py:2:1: F401 'aiohttp' imported but unused +src/suhail_pipeline/enrichment/processor.py:12:1: F401 'suhail_pipeline.persistence.db.get_async_db_engine' imported but unused +src/suhail_pipeline/enrichment/processor.py:13:1: F401 'sqlalchemy.ext.asyncio.async_sessionmaker' imported but unused +src/suhail_pipeline/enrichment/processor.py:20:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/enrichment/processor.py:26:1: W293 blank line contains whitespace +src/suhail_pipeline/enrichment/processor.py:28:33: E203 whitespace before ':' +src/suhail_pipeline/enrichment/processor.py:30:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/enrichment/processor.py:33:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/enrichment/processor.py:34:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/enrichment/processor.py:45:1: W293 blank line contains whitespace +src/suhail_pipeline/enrichment/processor.py:47:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/enrichment/processor.py:48:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/enrichment/processor.py:49:80: E501 line too long (89 > 79 characters) +src/suhail_pipeline/enrichment/processor.py:56:80: E501 line too long (110 > 79 characters) +src/suhail_pipeline/enrichment/processor.py:57:80: E501 line too long (106 > 79 characters) +src/suhail_pipeline/enrichment/processor.py:60:65: W291 trailing whitespace +src/suhail_pipeline/enrichment/strategies.py:14:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/enrichment/strategies.py:30:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/enrichment/strategies.py:39:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/enrichment/strategies.py:43:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:48:1: W293 blank line contains whitespace +src/suhail_pipeline/enrichment/strategies.py:50:80: E501 line too long (99 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:51:80: E501 line too long (109 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:71:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/enrichment/strategies.py:88:80: E501 line too long (89 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:92:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/enrichment/strategies.py:99:1: W293 blank line contains whitespace +src/suhail_pipeline/enrichment/strategies.py:101:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:121:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:124:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/enrichment/strategies.py:129:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:133:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:134:80: E501 line too long (104 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:135:80: E501 line too long (100 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:136:80: E501 line too long (144 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:137:80: E501 line too long (115 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:144:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:145:80: E501 line too long (119 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:146:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:151:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/enrichment/strategies.py:152:25: W291 trailing whitespace +src/suhail_pipeline/enrichment/strategies.py:159:1: W293 blank line contains whitespace +src/suhail_pipeline/enrichment/strategies.py:164:1: W293 blank line contains whitespace +src/suhail_pipeline/enrichment/strategies.py:166:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:181:80: E501 line too long (101 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:186:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:189:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/enrichment/strategies.py:196:1: W293 blank line contains whitespace +src/suhail_pipeline/enrichment/strategies.py:221:21: F541 f-string is missing placeholders +src/suhail_pipeline/enrichment/strategies.py:222:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/enrichment/strategies.py:228:22: W291 trailing whitespace +src/suhail_pipeline/exceptions.py:19:1: F401 'typing.List' imported but unused +src/suhail_pipeline/exceptions.py:19:1: F401 'typing.Union' imported but unused +src/suhail_pipeline/exceptions.py:74:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:110:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:122:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:134:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:146:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:159:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:171:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:183:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:206:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:219:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:254:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/exceptions.py:255:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/exceptions.py:262:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/exceptions.py:286:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:304:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:316:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:321:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:329:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/exceptions.py:335:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:337:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/exceptions.py:338:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/exceptions.py:349:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:352:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:356:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:364:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/exceptions.py:370:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:372:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/exceptions.py:373:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/exceptions.py:385:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:388:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:394:1: W293 blank line contains whitespace +src/suhail_pipeline/exceptions.py:411:21: W291 trailing whitespace +src/suhail_pipeline/geometry/stitcher.py:9:1: F401 'shapely.geometry.Polygon' imported but unused +src/suhail_pipeline/geometry/stitcher.py:10:1: F401 'shapely.errors.GEOSException' imported but unused +src/suhail_pipeline/geometry/stitcher.py:11:1: F401 'shapely.ops.snap' imported but unused +src/suhail_pipeline/geometry/stitcher.py:12:1: F401 'tqdm.tqdm' imported but unused +src/suhail_pipeline/geometry/stitcher.py:13:1: F401 'mercantile' imported but unused +src/suhail_pipeline/geometry/stitcher.py:14:1: F401 'sqlalchemy.text' imported but unused +src/suhail_pipeline/geometry/stitcher.py:55:80: E501 line too long (108 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:68:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:92:80: E501 line too long (103 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:132:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:148:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:150:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:157:80: E501 line too long (117 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:163:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:169:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:174:80: E501 line too long (101 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:176:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:181:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:186:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:214:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:218:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/geometry/stitcher.py:223:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/geometry/validator.py:20:38: W291 trailing whitespace +src/suhail_pipeline/logging_utils.py:18:1: F401 'asyncio' imported but unused +src/suhail_pipeline/logging_utils.py:24:1: F401 'typing.Union' imported but unused +src/suhail_pipeline/logging_utils.py:27:1: F401 'weakref' imported but unused +src/suhail_pipeline/logging_utils.py:38:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:50:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:54:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:57:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:60:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:63:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:67:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:73:80: E501 line too long (122 > 79 characters) +src/suhail_pipeline/logging_utils.py:75:80: E501 line too long (111 > 79 characters) +src/suhail_pipeline/logging_utils.py:77:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/logging_utils.py:83:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:84:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/logging_utils.py:91:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:96:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:105:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:109:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:115:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:121:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/logging_utils.py:123:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:128:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:136:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:143:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:147:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:152:47: W291 trailing whitespace +src/suhail_pipeline/logging_utils.py:155:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:163:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/logging_utils.py:164:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:167:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/logging_utils.py:169:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:174:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:181:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:185:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:194:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:208:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:218:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:233:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:234:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/logging_utils.py:246:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:252:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:256:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/logging_utils.py:258:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:267:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/logging_utils.py:268:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/logging_utils.py:274:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:280:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:284:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:290:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:293:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/logging_utils.py:294:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:297:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:301:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:305:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:309:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:313:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:317:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:322:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:325:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/logging_utils.py:332:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:351:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:358:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:361:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:367:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:369:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/logging_utils.py:372:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:376:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:379:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:381:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/logging_utils.py:383:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:392:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/logging_utils.py:393:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/logging_utils.py:394:80: E501 line too long (101 > 79 characters) +src/suhail_pipeline/logging_utils.py:406:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:408:29: W291 trailing whitespace +src/suhail_pipeline/logging_utils.py:409:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/logging_utils.py:411:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:416:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:427:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:430:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/logging_utils.py:432:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:441:80: E501 line too long (99 > 79 characters) +src/suhail_pipeline/logging_utils.py:442:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:448:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:449:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/logging_utils.py:451:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:466:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:471:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:474:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:476:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/logging_utils.py:478:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:489:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/logging_utils.py:491:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:495:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/logging_utils.py:497:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:499:9: F811 redefinition of unused 'asyncio' from line 18 +src/suhail_pipeline/logging_utils.py:504:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:508:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/logging_utils.py:514:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:519:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:520:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/logging_utils.py:521:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:527:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/logging_utils.py:537:1: W293 blank line contains whitespace +src/suhail_pipeline/logging_utils.py:543:16: W291 trailing whitespace +src/suhail_pipeline/memory_utils.py:47:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:56:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:62:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/memory_utils.py:64:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/memory_utils.py:66:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:69:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:75:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/memory_utils.py:79:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:85:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:87:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:92:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:97:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:101:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:106:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:109:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:114:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:117:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:120:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:130:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:132:80: E501 line too long (107 > 79 characters) +src/suhail_pipeline/memory_utils.py:135:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:137:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:143:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:144:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/memory_utils.py:150:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:154:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:160:56: W291 trailing whitespace +src/suhail_pipeline/memory_utils.py:163:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:169:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:171:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:175:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:181:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:182:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/memory_utils.py:184:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:186:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:205:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/memory_utils.py:208:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:210:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:213:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:218:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/memory_utils.py:223:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:224:5: F841 local variable 'e' is assigned to but never used +src/suhail_pipeline/memory_utils.py:229:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:233:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:236:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/memory_utils.py:245:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/memory_utils.py:251:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:255:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:258:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:262:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:266:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:269:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:276:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:280:80: E501 line too long (108 > 79 characters) +src/suhail_pipeline/memory_utils.py:284:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:286:80: E501 line too long (95 > 79 characters) +src/suhail_pipeline/memory_utils.py:288:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:291:80: E501 line too long (99 > 79 characters) +src/suhail_pipeline/memory_utils.py:292:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:300:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:303:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:306:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/memory_utils.py:307:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/memory_utils.py:309:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:314:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/memory_utils.py:316:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:329:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:332:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:339:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/memory_utils.py:342:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:344:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/memory_utils.py:345:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:350:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:353:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:358:1: W293 blank line contains whitespace +src/suhail_pipeline/memory_utils.py:362:14: W291 trailing whitespace +src/suhail_pipeline/persistence/db.py:16:21: F821 undefined name 'settings' +src/suhail_pipeline/persistence/db.py:16:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/persistence/db.py:19:19: F821 undefined name 'settings' +src/suhail_pipeline/persistence/db.py:20:22: F821 undefined name 'settings' +src/suhail_pipeline/persistence/db.py:20:50: F821 undefined name 'settings' +src/suhail_pipeline/persistence/db.py:21:23: F821 undefined name 'settings' +src/suhail_pipeline/persistence/db.py:22:22: F821 undefined name 'settings' +src/suhail_pipeline/persistence/db.py:23:14: F821 undefined name 'settings' +src/suhail_pipeline/persistence/db.py:25:22: F821 undefined name 'settings' +src/suhail_pipeline/persistence/db.py:27:32: F821 undefined name 'settings' +src/suhail_pipeline/persistence/db.py:52:80: E501 line too long (114 > 79 characters) +src/suhail_pipeline/persistence/db.py:57:80: E501 line too long (109 > 79 characters) +src/suhail_pipeline/persistence/db.py:72:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/persistence/db.py:73:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/persistence/db.py:78:25: W291 trailing whitespace +src/suhail_pipeline/persistence/db.py:78:26: W292 no newline at end of file +src/suhail_pipeline/persistence/enrichment_persister.py:5:1: F401 'math.ceil' imported but unused +src/suhail_pipeline/persistence/enrichment_persister.py:19:80: E501 line too long (112 > 79 characters) +src/suhail_pipeline/persistence/enrichment_persister.py:46:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/persistence/enrichment_persister.py:82:80: E501 line too long (102 > 79 characters) +src/suhail_pipeline/persistence/enrichment_persister.py:105:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/persistence/enrichment_persister.py:120:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/persistence/enrichment_persister.py:136:48: W291 trailing whitespace +src/suhail_pipeline/persistence/models.py:2:1: F401 'sqlalchemy.create_engine' imported but unused +src/suhail_pipeline/persistence/models.py:2:1: F401 'sqlalchemy.select' imported but unused +src/suhail_pipeline/persistence/models.py:2:1: F401 'sqlalchemy.update' imported but unused +src/suhail_pipeline/persistence/models.py:18:1: F401 'sqlalchemy.orm.load_only' imported but unused +src/suhail_pipeline/persistence/models.py:35:80: E501 line too long (100 > 79 characters) +src/suhail_pipeline/persistence/models.py:43:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/persistence/models.py:50:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/persistence/models.py:67:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/persistence/models.py:73:1: W293 blank line contains whitespace +src/suhail_pipeline/persistence/models.py:77:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/persistence/models.py:84:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/persistence/models.py:89:80: E501 line too long (100 > 79 characters) +src/suhail_pipeline/persistence/models.py:90:1: W293 blank line contains whitespace +src/suhail_pipeline/persistence/models.py:96:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/persistence/models.py:103:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/persistence/models.py:121:1: W293 blank line contains whitespace +src/suhail_pipeline/persistence/models.py:125:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/persistence/models.py:127:1: W293 blank line contains whitespace +src/suhail_pipeline/persistence/models.py:128:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:135:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/persistence/models.py:144:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/persistence/models.py:147:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:179:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:189:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/persistence/models.py:193:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:198:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:202:1: W293 blank line contains whitespace +src/suhail_pipeline/persistence/models.py:205:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:210:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:217:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:225:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:233:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:241:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:248:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:255:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:261:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:267:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/models.py:280:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/persistence/models.py:301:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/persistence/models.py:303:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/persistence/models.py:306:80: E501 line too long (102 > 79 characters) +src/suhail_pipeline/persistence/models.py:312:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/persistence/models.py:313:80: E501 line too long (194 > 79 characters) +src/suhail_pipeline/persistence/models.py:324:33: E711 comparison to None should be 'if cond is not None:' +src/suhail_pipeline/persistence/models.py:326:80: E501 line too long (113 > 79 characters) +src/suhail_pipeline/persistence/models.py:328:23: W291 trailing whitespace +src/suhail_pipeline/persistence/models.py:328:24: W292 no newline at end of file +src/suhail_pipeline/persistence/postgis_persister.py:26:1: E305 expected 2 blank lines after class or function definition, found 1 +src/suhail_pipeline/persistence/postgis_persister.py:131:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/persistence/postgis_persister.py:134:71: W291 trailing whitespace +src/suhail_pipeline/persistence/postgis_persister.py:137:1: W293 blank line contains whitespace +src/suhail_pipeline/persistence/postgis_persister.py:142:1: W293 blank line contains whitespace +src/suhail_pipeline/persistence/postgis_persister.py:143:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:151:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:160:80: E501 line too long (99 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:162:80: E501 line too long (114 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:163:80: E501 line too long (101 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:172:80: E501 line too long (120 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:179:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:183:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:205:80: E501 line too long (100 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:207:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:209:80: E501 line too long (100 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:211:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:217:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:219:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:221:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:248:80: E501 line too long (99 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:250:80: E501 line too long (96 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:258:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:259:80: E501 line too long (96 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:265:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:268:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:275:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:276:1: W293 blank line contains whitespace +src/suhail_pipeline/persistence/postgis_persister.py:279:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:282:80: E501 line too long (98 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:284:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:291:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:295:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:309:80: E501 line too long (115 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:310:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:312:80: E501 line too long (101 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:314:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:326:80: E501 line too long (126 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:340:80: E501 line too long (107 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:360:80: E501 line too long (111 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:362:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:363:80: E501 line too long (104 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:366:80: E501 line too long (121 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:368:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:375:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:377:80: E501 line too long (96 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:402:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:415:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:424:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/persistence/postgis_persister.py:429:36: W291 trailing whitespace +src/suhail_pipeline/persistence/table_management.py:7:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/persistence/table_management.py:16:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/persistence/table_management.py:18:75: W291 trailing whitespace +src/suhail_pipeline/persistence/table_management.py:18:76: W292 no newline at end of file +src/suhail_pipeline/pipeline_orchestrator.py:4:1: F401 'typing.Dict' imported but unused +src/suhail_pipeline/pipeline_orchestrator.py:6:1: F401 'concurrent.futures' imported but unused +src/suhail_pipeline/pipeline_orchestrator.py:12:1: F401 'sqlalchemy.create_engine' imported but unused +src/suhail_pipeline/pipeline_orchestrator.py:22:1: F401 'sqlalchemy.text' imported but unused +src/suhail_pipeline/pipeline_orchestrator.py:77:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:82:80: E501 line too long (107 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:85:80: E501 line too long (110 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:109:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:114:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:171:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:172:80: E501 line too long (89 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:210:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:211:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:229:80: E501 line too long (98 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:231:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:252:5: F841 local variable 'stitcher' is assigned to but never used +src/suhail_pipeline/pipeline_orchestrator.py:252:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:264:80: E501 line too long (104 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:271:80: E501 line too long (101 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:274:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:295:80: E501 line too long (141 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:296:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:299:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:304:80: E501 line too long (96 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:305:80: E501 line too long (108 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:310:80: E501 line too long (111 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:315:80: E501 line too long (109 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:316:80: E501 line too long (126 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:333:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:340:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:354:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:359:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:365:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:369:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:378:80: E501 line too long (112 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:381:80: E501 line too long (119 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:388:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:394:80: E501 line too long (114 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:399:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/pipeline_orchestrator.py:409:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:10:1: F401 'typing.Optional' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:11:1: F401 'sqlalchemy.create_engine' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:12:1: F401 'sqlalchemy.ext.asyncio.create_async_engine' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:12:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:13:1: F401 'sqlalchemy.dialects.postgresql.insert as pg_insert' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:14:1: F401 'datetime.timedelta' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:23:1: F401 'suhail_pipeline.persistence.models.Base' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:23:1: F401 'suhail_pipeline.persistence.models.Transaction' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:23:1: F401 'suhail_pipeline.persistence.models.ParcelPriceMetric' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:23:1: F401 'suhail_pipeline.persistence.models.BuildingRule' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:23:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:29:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:30:1: F401 'logging' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:30:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.BigInteger' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.Column' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.DateTime' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.Float' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.Integer' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.String' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:31:1: F401 'sqlalchemy.JSON' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:31:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:31:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:32:1: F401 'sqlalchemy.orm.sessionmaker' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:32:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:33:1: F401 'sqlalchemy.ForeignKey' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:33:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:34:1: F401 'suhail_pipeline.exceptions.ValidationException' imported but unused +src/suhail_pipeline/run_enrichment_pipeline.py:34:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:35:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:36:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:43:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:44:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:48:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:51:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:52:1: E402 module level import not at top of file +src/suhail_pipeline/run_enrichment_pipeline.py:72:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_enrichment_pipeline.py:75:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:81:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:85:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:91:1: W293 blank line contains whitespace +src/suhail_pipeline/run_enrichment_pipeline.py:99:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:107:80: E501 line too long (127 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:111:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_enrichment_pipeline.py:115:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:117:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_enrichment_pipeline.py:122:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:126:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:128:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_enrichment_pipeline.py:134:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:144:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_enrichment_pipeline.py:152:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:155:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_enrichment_pipeline.py:160:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:162:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:165:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:168:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:171:80: E501 line too long (117 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:173:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:176:1: W293 blank line contains whitespace +src/suhail_pipeline/run_enrichment_pipeline.py:187:80: E501 line too long (89 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:193:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:208:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:209:80: E501 line too long (89 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:213:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:223:80: E501 line too long (104 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:225:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:226:80: E501 line too long (106 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:230:80: E501 line too long (89 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:231:80: E501 line too long (101 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:234:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:239:32: F541 f-string is missing placeholders +src/suhail_pipeline/run_enrichment_pipeline.py:240:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:243:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:244:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:245:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:251:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:263:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:264:80: E501 line too long (112 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:294:80: E501 line too long (99 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:300:80: E501 line too long (98 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:303:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:305:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:306:1: W293 blank line contains whitespace +src/suhail_pipeline/run_enrichment_pipeline.py:309:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_enrichment_pipeline.py:311:80: E501 line too long (105 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:312:80: E501 line too long (108 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:313:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:314:80: E501 line too long (82 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:317:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:318:1: W293 blank line contains whitespace +src/suhail_pipeline/run_enrichment_pipeline.py:323:1: W293 blank line contains whitespace +src/suhail_pipeline/run_enrichment_pipeline.py:327:1: W293 blank line contains whitespace +src/suhail_pipeline/run_enrichment_pipeline.py:330:80: E501 line too long (89 > 79 characters) +src/suhail_pipeline/run_enrichment_pipeline.py:332:1: W293 blank line contains whitespace +src/suhail_pipeline/run_enrichment_pipeline.py:337:1: W293 blank line contains whitespace +src/suhail_pipeline/run_enrichment_pipeline.py:341:1: W293 blank line contains whitespace +src/suhail_pipeline/run_enrichment_pipeline.py:347:1: W293 blank line contains whitespace +src/suhail_pipeline/run_enrichment_pipeline.py:355:1: E305 expected 2 blank lines after class or function definition, found 1 +src/suhail_pipeline/run_enrichment_pipeline.py:356:10: W291 trailing whitespace +src/suhail_pipeline/run_geometric_pipeline.py:3:1: F401 'subprocess' imported but unused +src/suhail_pipeline/run_geometric_pipeline.py:4:1: F401 'sys' imported but unused +src/suhail_pipeline/run_geometric_pipeline.py:5:1: F401 'typing.Tuple' imported but unused +src/suhail_pipeline/run_geometric_pipeline.py:17:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_geometric_pipeline.py:23:80: E501 line too long (100 > 79 characters) +src/suhail_pipeline/run_geometric_pipeline.py:29:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/run_geometric_pipeline.py:48:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/run_geometric_pipeline.py:63:1: W293 blank line contains whitespace +src/suhail_pipeline/run_geometric_pipeline.py:65:51: W291 trailing whitespace +src/suhail_pipeline/run_geometric_pipeline.py:77:80: E501 line too long (86 > 79 characters) +src/suhail_pipeline/run_geometric_pipeline.py:79:1: W293 blank line contains whitespace +src/suhail_pipeline/run_geometric_pipeline.py:83:1: W293 blank line contains whitespace +src/suhail_pipeline/run_geometric_pipeline.py:105:1: E305 expected 2 blank lines after class or function definition, found 1 +src/suhail_pipeline/run_geometric_pipeline.py:106:10: W291 trailing whitespace +src/suhail_pipeline/run_monitoring.py:4:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/run_monitoring.py:17:1: E402 module level import not at top of file +src/suhail_pipeline/run_monitoring.py:21:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_monitoring.py:25:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:28:80: E501 line too long (118 > 79 characters) +src/suhail_pipeline/run_monitoring.py:29:80: E501 line too long (123 > 79 characters) +src/suhail_pipeline/run_monitoring.py:30:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/run_monitoring.py:31:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:34:19: W291 trailing whitespace +src/suhail_pipeline/run_monitoring.py:35:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/run_monitoring.py:36:80: E501 line too long (92 > 79 characters) +src/suhail_pipeline/run_monitoring.py:37:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/run_monitoring.py:38:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/run_monitoring.py:41:32: W291 trailing whitespace +src/suhail_pipeline/run_monitoring.py:44:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:47:15: F541 f-string is missing placeholders +src/suhail_pipeline/run_monitoring.py:53:15: F541 f-string is missing placeholders +src/suhail_pipeline/run_monitoring.py:55:80: E501 line too long (99 > 79 characters) +src/suhail_pipeline/run_monitoring.py:57:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:59:19: F541 f-string is missing placeholders +src/suhail_pipeline/run_monitoring.py:61:65: W291 trailing whitespace +src/suhail_pipeline/run_monitoring.py:66:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:68:80: E501 line too long (107 > 79 characters) +src/suhail_pipeline/run_monitoring.py:70:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_monitoring.py:74:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:76:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/run_monitoring.py:77:9: F841 local variable 'trigger_parcels' is assigned to but never used +src/suhail_pipeline/run_monitoring.py:79:80: E501 line too long (84 > 79 characters) +src/suhail_pipeline/run_monitoring.py:82:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:84:80: E501 line too long (121 > 79 characters) +src/suhail_pipeline/run_monitoring.py:85:80: E501 line too long (121 > 79 characters) +src/suhail_pipeline/run_monitoring.py:87:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:92:40: W291 trailing whitespace +src/suhail_pipeline/run_monitoring.py:95:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:96:48: W291 trailing whitespace +src/suhail_pipeline/run_monitoring.py:100:40: W291 trailing whitespace +src/suhail_pipeline/run_monitoring.py:103:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:106:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:108:80: E501 line too long (104 > 79 characters) +src/suhail_pipeline/run_monitoring.py:109:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:110:15: F541 f-string is missing placeholders +src/suhail_pipeline/run_monitoring.py:112:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/run_monitoring.py:115:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:118:80: E501 line too long (98 > 79 characters) +src/suhail_pipeline/run_monitoring.py:119:80: E501 line too long (114 > 79 characters) +src/suhail_pipeline/run_monitoring.py:120:19: F541 f-string is missing placeholders +src/suhail_pipeline/run_monitoring.py:120:80: E501 line too long (108 > 79 characters) +src/suhail_pipeline/run_monitoring.py:121:80: E501 line too long (97 > 79 characters) +src/suhail_pipeline/run_monitoring.py:125:19: F541 f-string is missing placeholders +src/suhail_pipeline/run_monitoring.py:125:80: E501 line too long (108 > 79 characters) +src/suhail_pipeline/run_monitoring.py:127:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:131:19: F541 f-string is missing placeholders +src/suhail_pipeline/run_monitoring.py:132:19: F541 f-string is missing placeholders +src/suhail_pipeline/run_monitoring.py:132:80: E501 line too long (111 > 79 characters) +src/suhail_pipeline/run_monitoring.py:135:77: W291 trailing whitespace +src/suhail_pipeline/run_monitoring.py:136:19: F541 f-string is missing placeholders +src/suhail_pipeline/run_monitoring.py:136:80: E501 line too long (112 > 79 characters) +src/suhail_pipeline/run_monitoring.py:138:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:143:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/run_monitoring.py:149:1: W293 blank line contains whitespace +src/suhail_pipeline/run_monitoring.py:152:80: E501 line too long (118 > 79 characters) +src/suhail_pipeline/run_monitoring.py:154:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_monitoring.py:155:80: E501 line too long (94 > 79 characters) +src/suhail_pipeline/run_monitoring.py:156:80: E501 line too long (98 > 79 characters) +src/suhail_pipeline/run_monitoring.py:158:80: E501 line too long (88 > 79 characters) +src/suhail_pipeline/run_monitoring.py:166:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_monitoring.py:166:15: W291 trailing whitespace +src/suhail_pipeline/run_monitoring.py:175:80: E501 line too long (87 > 79 characters) +src/suhail_pipeline/run_monitoring.py:176:80: E501 line too long (91 > 79 characters) +src/suhail_pipeline/run_monitoring.py:180:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/run_monitoring.py:181:80: E501 line too long (85 > 79 characters) +src/suhail_pipeline/run_monitoring.py:189:80: E501 line too long (81 > 79 characters) +src/suhail_pipeline/run_monitoring.py:191:1: E305 expected 2 blank lines after class or function definition, found 1 +src/suhail_pipeline/run_monitoring.py:192:10: W291 trailing whitespace +src/suhail_pipeline/run_tile_pipeline.py:14:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_tile_pipeline.py:28:80: E501 line too long (105 > 79 characters) +src/suhail_pipeline/run_tile_pipeline.py:30:80: E501 line too long (95 > 79 characters) +src/suhail_pipeline/run_tile_pipeline.py:37:80: E501 line too long (93 > 79 characters) +src/suhail_pipeline/run_tile_pipeline.py:39:80: E501 line too long (83 > 79 characters) +src/suhail_pipeline/run_tile_pipeline.py:42:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/run_tile_pipeline.py:51:80: E501 line too long (107 > 79 characters) +src/suhail_pipeline/run_tile_pipeline.py:59:1: E305 expected 2 blank lines after class or function definition, found 1 +src/suhail_pipeline/run_tile_pipeline.py:60:11: W291 trailing whitespace +src/suhail_pipeline/run_tile_pipeline.py:60:12: W292 no newline at end of file +src/suhail_pipeline/show_discovery_summary.py:4:80: E501 line too long (80 > 79 characters) +src/suhail_pipeline/show_discovery_summary.py:12:1: E402 module level import not at top of file +src/suhail_pipeline/show_discovery_summary.py:13:1: E402 module level import not at top of file +src/suhail_pipeline/show_discovery_summary.py:15:1: E302 expected 2 blank lines, found 1 +src/suhail_pipeline/show_discovery_summary.py:19:1: W293 blank line contains whitespace +src/suhail_pipeline/show_discovery_summary.py:21:80: E501 line too long (107 > 79 characters) +src/suhail_pipeline/show_discovery_summary.py:27:1: W293 blank line contains whitespace +src/suhail_pipeline/show_discovery_summary.py:28:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:31:1: W293 blank line contains whitespace +src/suhail_pipeline/show_discovery_summary.py:32:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:35:1: W293 blank line contains whitespace +src/suhail_pipeline/show_discovery_summary.py:41:1: W293 blank line contains whitespace +src/suhail_pipeline/show_discovery_summary.py:42:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:43:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:44:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:45:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:46:1: W293 blank line contains whitespace +src/suhail_pipeline/show_discovery_summary.py:47:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:48:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:49:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:50:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:51:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:52:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:53:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:54:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:55:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:55:80: E501 line too long (90 > 79 characters) +src/suhail_pipeline/show_discovery_summary.py:56:1: W293 blank line contains whitespace +src/suhail_pipeline/show_discovery_summary.py:57:11: F541 f-string is missing placeholders +src/suhail_pipeline/show_discovery_summary.py:59:1: E305 expected 2 blank lines after class or function definition, found 1 +src/suhail_pipeline/show_discovery_summary.py:60:11: W291 trailing whitespace +src/suhail_pipeline/utils/tile_list_generator.py:10:80: E501 line too long (80 > 79 characters) tests/__init__.py:1:1: W391 blank line at end of file tests/integration/test_pipeline_integration.py:28:80: E501 line too long (84 > 79 characters) tests/integration/test_pipeline_integration.py:29:80: E501 line too long (80 > 79 characters) @@ -981,7 +981,7 @@ tests/unit/test_delta_enrichment_cli.py:17:80: E501 line too long (84 > 79 chara tests/unit/test_delta_enrichment_cli.py:19:80: E501 line too long (94 > 79 characters) tests/unit/test_delta_enrichment_cli.py:64:80: E501 line too long (86 > 79 characters) tests/unit/test_discovery.py:13:13: F821 undefined name 'get_tile_coordinates_for_grid' -tests/unit/test_province_loader.py:2:1: F401 'meshic_pipeline.utils.tile_list_generator.tiles_from_bbox_z' imported but unused +tests/unit/test_province_loader.py:2:1: F401 'suhail_pipeline.utils.tile_list_generator.tiles_from_bbox_z' imported but unused tests/unit/test_province_loader.py:13:80: E501 line too long (84 > 79 characters) tests/unit/test_smart_pipeline_enrich.py:6:5: E306 expected 1 blank line before a nested definition, found 0 tests/unit/test_smart_pipeline_enrich.py:8:80: E501 line too long (81 > 79 characters) diff --git a/docs/archive/generated-txt/geometric_help.txt b/docs/archive/generated-txt/geometric_help.txt index c555ecd..656de67 100644 --- a/docs/archive/generated-txt/geometric_help.txt +++ b/docs/archive/generated-txt/geometric_help.txt @@ -1,5 +1,5 @@ - Usage: meshic-pipeline geometric [OPTIONS] + Usage: suhail-pipeline geometric [OPTIONS] Run the geometric pipeline (Stage 1). diff --git a/docs/archive/generated-txt/geometric_help_full.txt b/docs/archive/generated-txt/geometric_help_full.txt index fd2fcf5..113f310 100644 --- a/docs/archive/generated-txt/geometric_help_full.txt +++ b/docs/archive/generated-txt/geometric_help_full.txt @@ -1,5 +1,5 @@ - Usage: python -m src.meshic_pipeline.run_geometric_pipeline + Usage: python -m src.suhail_pipeline.run_geometric_pipeline [OPTIONS] 🚀 Enhanced geometric processing pipeline with province discovery. diff --git a/docs/archive/generated-txt/geometric_test_results.txt b/docs/archive/generated-txt/geometric_test_results.txt index 7ef08e5..9d83c00 100644 --- a/docs/archive/generated-txt/geometric_test_results.txt +++ b/docs/archive/generated-txt/geometric_test_results.txt @@ -1,101 +1,101 @@ -2025-07-07 08:22:36,624 - meshic_pipeline.pipeline_orchestrator - INFO - Starting pipeline run for AOI: grid at zoom 15 -2025-07-07 08:22:36,625 - meshic_pipeline.pipeline_orchestrator - INFO - 🔢 Grid mode: Discovered 9 tiles for 3x3 grid -2025-07-07 08:22:36,625 - meshic_pipeline.pipeline_orchestrator - INFO - 🌐 Using tile server: https://tiles.suhail.ai/maps/riyadh -2025-07-07 08:22:36,738 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: neighborhoods --- -2025-07-07 08:22:37,650 - meshic_pipeline.pipeline_orchestrator - WARNING - Dropped 16 duplicate rows for primary key 'neighborhood_id' in layer 'neighborhoods' before DB write. -2025-07-07 08:22:37,665 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_neighborhoods from known columns -2025-07-07 08:22:37,755 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 7 features to public.temp_neighborhoods using mode 'replace' +2025-07-07 08:22:36,624 - suhail_pipeline.pipeline_orchestrator - INFO - Starting pipeline run for AOI: grid at zoom 15 +2025-07-07 08:22:36,625 - suhail_pipeline.pipeline_orchestrator - INFO - 🔢 Grid mode: Discovered 9 tiles for 3x3 grid +2025-07-07 08:22:36,625 - suhail_pipeline.pipeline_orchestrator - INFO - 🌐 Using tile server: https://tiles.suhail.ai/maps/riyadh +2025-07-07 08:22:36,738 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: neighborhoods --- +2025-07-07 08:22:37,650 - suhail_pipeline.pipeline_orchestrator - WARNING - Dropped 16 duplicate rows for primary key 'neighborhood_id' in layer 'neighborhoods' before DB write. +2025-07-07 08:22:37,665 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_neighborhoods from known columns +2025-07-07 08:22:37,755 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 7 features to public.temp_neighborhoods using mode 'replace' 2025-07-07 08:22:37,846 - pyogrio._io - INFO - Created 7 records -2025-07-07 08:22:37,848 - meshic_pipeline.pipeline_orchestrator - INFO - Persisting 7 features for layer 'neighborhoods' to table 'neighborhoods' -2025-07-07 08:22:37,851 - meshic_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.neighborhoods using ID column 'neighborhood_id' -2025-07-07 08:22:37,857 - meshic_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 7 rows in public.neighborhoods. -2025-07-07 08:22:37,858 - meshic_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: neighborhoods --- -2025-07-07 08:22:37,858 - meshic_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'neighborhoods': 0.00MB -2025-07-07 08:22:37,858 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: subdivisions --- -2025-07-07 08:22:38,625 - meshic_pipeline.pipeline_orchestrator - WARNING - Dropped 32 duplicate rows for primary key 'subdivision_id' in layer 'subdivisions' before DB write. -2025-07-07 08:22:38,631 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_subdivisions from known columns -2025-07-07 08:22:38,652 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 34 features to public.temp_subdivisions using mode 'replace' +2025-07-07 08:22:37,848 - suhail_pipeline.pipeline_orchestrator - INFO - Persisting 7 features for layer 'neighborhoods' to table 'neighborhoods' +2025-07-07 08:22:37,851 - suhail_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.neighborhoods using ID column 'neighborhood_id' +2025-07-07 08:22:37,857 - suhail_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 7 rows in public.neighborhoods. +2025-07-07 08:22:37,858 - suhail_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: neighborhoods --- +2025-07-07 08:22:37,858 - suhail_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'neighborhoods': 0.00MB +2025-07-07 08:22:37,858 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: subdivisions --- +2025-07-07 08:22:38,625 - suhail_pipeline.pipeline_orchestrator - WARNING - Dropped 32 duplicate rows for primary key 'subdivision_id' in layer 'subdivisions' before DB write. +2025-07-07 08:22:38,631 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_subdivisions from known columns +2025-07-07 08:22:38,652 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 34 features to public.temp_subdivisions using mode 'replace' 2025-07-07 08:22:38,658 - pyogrio._io - INFO - Created 34 records -2025-07-07 08:22:38,661 - meshic_pipeline.pipeline_orchestrator - INFO - Persisting 34 features for layer 'subdivisions' to table 'subdivisions' -2025-07-07 08:22:38,664 - meshic_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.subdivisions using ID column 'subdivision_id' -2025-07-07 08:22:38,672 - meshic_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 34 rows in public.subdivisions. -2025-07-07 08:22:38,673 - meshic_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: subdivisions --- -2025-07-07 08:22:38,673 - meshic_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'subdivisions': 0.00MB -2025-07-07 08:22:38,673 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: parcels --- -2025-07-07 08:22:40,585 - meshic_pipeline.pipeline_orchestrator - WARNING - Dropped 774 duplicate rows for primary key 'parcel_objectid' in layer 'parcels' before DB write. +2025-07-07 08:22:38,661 - suhail_pipeline.pipeline_orchestrator - INFO - Persisting 34 features for layer 'subdivisions' to table 'subdivisions' +2025-07-07 08:22:38,664 - suhail_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.subdivisions using ID column 'subdivision_id' +2025-07-07 08:22:38,672 - suhail_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 34 rows in public.subdivisions. +2025-07-07 08:22:38,673 - suhail_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: subdivisions --- +2025-07-07 08:22:38,673 - suhail_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'subdivisions': 0.00MB +2025-07-07 08:22:38,673 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: parcels --- +2025-07-07 08:22:40,585 - suhail_pipeline.pipeline_orchestrator - WARNING - Dropped 774 duplicate rows for primary key 'parcel_objectid' in layer 'parcels' before DB write. [INFO] All ruleid values present in zoning_rules. -2025-07-07 08:22:40,634 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_parcels from known columns -2025-07-07 08:22:40,851 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 9007 features to public.temp_parcels using mode 'replace' -2025-07-07 08:22:40,851 - meshic_pipeline.pipeline_orchestrator - INFO - Enriching parcels with region_id via spatial join... -2025-07-07 08:22:40,944 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.parcels_enriched from known columns -2025-07-07 08:22:41,282 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 9007 features to public.parcels_enriched using mode 'replace' -2025-07-07 08:22:41,284 - meshic_pipeline.pipeline_orchestrator - INFO - Enrichment complete: region_id assigned where possible. +2025-07-07 08:22:40,634 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_parcels from known columns +2025-07-07 08:22:40,851 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 9007 features to public.temp_parcels using mode 'replace' +2025-07-07 08:22:40,851 - suhail_pipeline.pipeline_orchestrator - INFO - Enriching parcels with region_id via spatial join... +2025-07-07 08:22:40,944 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.parcels_enriched from known columns +2025-07-07 08:22:41,282 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 9007 features to public.parcels_enriched using mode 'replace' +2025-07-07 08:22:41,284 - suhail_pipeline.pipeline_orchestrator - INFO - Enrichment complete: region_id assigned where possible. 2025-07-07 08:22:41,608 - pyogrio._io - INFO - Created 9,007 records -2025-07-07 08:22:41,757 - meshic_pipeline.pipeline_orchestrator - INFO - Persisting 9007 features for layer 'parcels' to table 'parcels' -2025-07-07 08:22:41,882 - meshic_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.parcels using ID column 'parcel_objectid' -2025-07-07 08:22:42,231 - meshic_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 9007 rows in public.parcels. -2025-07-07 08:22:42,242 - meshic_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: parcels --- -2025-07-07 08:22:42,243 - meshic_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'parcels': 0.00MB -2025-07-07 08:22:42,243 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: parcels-centroids --- -2025-07-07 08:22:42,245 - meshic_pipeline.pipeline_orchestrator - WARNING - Skipping layer 'parcels-centroids': production table 'parcels-centroids' does not exist. -2025-07-07 08:22:42,245 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: neighborhoods-centroids --- -2025-07-07 08:22:42,245 - meshic_pipeline.pipeline_orchestrator - WARNING - Skipping layer 'neighborhoods-centroids': production table 'neighborhoods-centroids' does not exist. -2025-07-07 08:22:42,245 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: metro_lines --- -2025-07-07 08:22:43,136 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_metro_lines from known columns -2025-07-07 08:22:43,160 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 6 features to public.temp_metro_lines using mode 'replace' +2025-07-07 08:22:41,757 - suhail_pipeline.pipeline_orchestrator - INFO - Persisting 9007 features for layer 'parcels' to table 'parcels' +2025-07-07 08:22:41,882 - suhail_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.parcels using ID column 'parcel_objectid' +2025-07-07 08:22:42,231 - suhail_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 9007 rows in public.parcels. +2025-07-07 08:22:42,242 - suhail_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: parcels --- +2025-07-07 08:22:42,243 - suhail_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'parcels': 0.00MB +2025-07-07 08:22:42,243 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: parcels-centroids --- +2025-07-07 08:22:42,245 - suhail_pipeline.pipeline_orchestrator - WARNING - Skipping layer 'parcels-centroids': production table 'parcels-centroids' does not exist. +2025-07-07 08:22:42,245 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: neighborhoods-centroids --- +2025-07-07 08:22:42,245 - suhail_pipeline.pipeline_orchestrator - WARNING - Skipping layer 'neighborhoods-centroids': production table 'neighborhoods-centroids' does not exist. +2025-07-07 08:22:42,245 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: metro_lines --- +2025-07-07 08:22:43,136 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_metro_lines from known columns +2025-07-07 08:22:43,160 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 6 features to public.temp_metro_lines using mode 'replace' 2025-07-07 08:22:43,163 - pyogrio._io - INFO - Created 6 records -2025-07-07 08:22:43,164 - meshic_pipeline.pipeline_orchestrator - INFO - Persisting 6 features for layer 'metro_lines' to table 'metro_lines' -2025-07-07 08:22:43,169 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.metro_lines from known columns -2025-07-07 08:22:43,187 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 6 features to public.metro_lines using mode 'replace' -2025-07-07 08:22:43,187 - meshic_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: metro_lines --- -2025-07-07 08:22:43,187 - meshic_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'metro_lines': 0.00MB -2025-07-07 08:22:43,187 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: bus_lines --- -2025-07-07 08:22:43,961 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_bus_lines from known columns -2025-07-07 08:22:43,982 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 52 features to public.temp_bus_lines using mode 'replace' +2025-07-07 08:22:43,164 - suhail_pipeline.pipeline_orchestrator - INFO - Persisting 6 features for layer 'metro_lines' to table 'metro_lines' +2025-07-07 08:22:43,169 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.metro_lines from known columns +2025-07-07 08:22:43,187 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 6 features to public.metro_lines using mode 'replace' +2025-07-07 08:22:43,187 - suhail_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: metro_lines --- +2025-07-07 08:22:43,187 - suhail_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'metro_lines': 0.00MB +2025-07-07 08:22:43,187 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: bus_lines --- +2025-07-07 08:22:43,961 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_bus_lines from known columns +2025-07-07 08:22:43,982 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 52 features to public.temp_bus_lines using mode 'replace' 2025-07-07 08:22:43,987 - pyogrio._io - INFO - Created 52 records -2025-07-07 08:22:43,990 - meshic_pipeline.pipeline_orchestrator - INFO - Persisting 52 features for layer 'bus_lines' to table 'bus_lines' -2025-07-07 08:22:43,998 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.bus_lines from known columns -2025-07-07 08:22:44,015 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 52 features to public.bus_lines using mode 'replace' -2025-07-07 08:22:44,015 - meshic_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: bus_lines --- -2025-07-07 08:22:44,015 - meshic_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'bus_lines': 0.00MB -2025-07-07 08:22:44,015 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: metro_stations --- -2025-07-07 08:22:44,771 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_metro_stations from known columns -2025-07-07 08:22:44,787 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 5 features to public.temp_metro_stations using mode 'replace' +2025-07-07 08:22:43,990 - suhail_pipeline.pipeline_orchestrator - INFO - Persisting 52 features for layer 'bus_lines' to table 'bus_lines' +2025-07-07 08:22:43,998 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.bus_lines from known columns +2025-07-07 08:22:44,015 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 52 features to public.bus_lines using mode 'replace' +2025-07-07 08:22:44,015 - suhail_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: bus_lines --- +2025-07-07 08:22:44,015 - suhail_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'bus_lines': 0.00MB +2025-07-07 08:22:44,015 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: metro_stations --- +2025-07-07 08:22:44,771 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_metro_stations from known columns +2025-07-07 08:22:44,787 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 5 features to public.temp_metro_stations using mode 'replace' 2025-07-07 08:22:44,791 - pyogrio._io - INFO - Created 5 records -2025-07-07 08:22:44,792 - meshic_pipeline.pipeline_orchestrator - INFO - Persisting 5 features for layer 'metro_stations' to table 'metro_stations' -2025-07-07 08:22:44,794 - meshic_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.metro_stations using ID column 'station_code' -2025-07-07 08:22:44,800 - meshic_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 5 rows in public.metro_stations. -2025-07-07 08:22:44,801 - meshic_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: metro_stations --- -2025-07-07 08:22:44,801 - meshic_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'metro_stations': 0.00MB -2025-07-07 08:22:44,801 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: riyadh_bus_stations --- -2025-07-07 08:22:45,564 - meshic_pipeline.pipeline_orchestrator - WARNING - Dropped 6 duplicate rows for primary key 'station_code' in layer 'riyadh_bus_stations' before DB write. -2025-07-07 08:22:45,571 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_riyadh_bus_stations from known columns -2025-07-07 08:22:45,590 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 31 features to public.temp_riyadh_bus_stations using mode 'replace' +2025-07-07 08:22:44,792 - suhail_pipeline.pipeline_orchestrator - INFO - Persisting 5 features for layer 'metro_stations' to table 'metro_stations' +2025-07-07 08:22:44,794 - suhail_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.metro_stations using ID column 'station_code' +2025-07-07 08:22:44,800 - suhail_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 5 rows in public.metro_stations. +2025-07-07 08:22:44,801 - suhail_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: metro_stations --- +2025-07-07 08:22:44,801 - suhail_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'metro_stations': 0.00MB +2025-07-07 08:22:44,801 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: riyadh_bus_stations --- +2025-07-07 08:22:45,564 - suhail_pipeline.pipeline_orchestrator - WARNING - Dropped 6 duplicate rows for primary key 'station_code' in layer 'riyadh_bus_stations' before DB write. +2025-07-07 08:22:45,571 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_riyadh_bus_stations from known columns +2025-07-07 08:22:45,590 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 31 features to public.temp_riyadh_bus_stations using mode 'replace' 2025-07-07 08:22:45,593 - pyogrio._io - INFO - Created 31 records -2025-07-07 08:22:45,595 - meshic_pipeline.pipeline_orchestrator - INFO - Persisting 31 features for layer 'riyadh_bus_stations' to table 'riyadh_bus_stations' -2025-07-07 08:22:45,597 - meshic_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.riyadh_bus_stations using ID column 'station_code' -2025-07-07 08:22:45,603 - meshic_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 31 rows in public.riyadh_bus_stations. -2025-07-07 08:22:45,604 - meshic_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: riyadh_bus_stations --- -2025-07-07 08:22:45,604 - meshic_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'riyadh_bus_stations': 0.00MB -2025-07-07 08:22:45,604 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: qi_population_metrics --- -2025-07-07 08:22:46,410 - meshic_pipeline.pipeline_orchestrator - WARNING - Dropped 64 duplicate rows for primary key 'grid_id' in layer 'qi_population_metrics' before DB write. -2025-07-07 08:22:46,416 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_qi_population_metrics from known columns -2025-07-07 08:22:46,456 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 136 features to public.temp_qi_population_metrics using mode 'replace' +2025-07-07 08:22:45,595 - suhail_pipeline.pipeline_orchestrator - INFO - Persisting 31 features for layer 'riyadh_bus_stations' to table 'riyadh_bus_stations' +2025-07-07 08:22:45,597 - suhail_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.riyadh_bus_stations using ID column 'station_code' +2025-07-07 08:22:45,603 - suhail_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 31 rows in public.riyadh_bus_stations. +2025-07-07 08:22:45,604 - suhail_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: riyadh_bus_stations --- +2025-07-07 08:22:45,604 - suhail_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'riyadh_bus_stations': 0.00MB +2025-07-07 08:22:45,604 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: qi_population_metrics --- +2025-07-07 08:22:46,410 - suhail_pipeline.pipeline_orchestrator - WARNING - Dropped 64 duplicate rows for primary key 'grid_id' in layer 'qi_population_metrics' before DB write. +2025-07-07 08:22:46,416 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_qi_population_metrics from known columns +2025-07-07 08:22:46,456 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 136 features to public.temp_qi_population_metrics using mode 'replace' 2025-07-07 08:22:46,463 - pyogrio._io - INFO - Created 136 records -2025-07-07 08:22:46,466 - meshic_pipeline.pipeline_orchestrator - INFO - Persisting 136 features for layer 'qi_population_metrics' to table 'qi_population_metrics' -2025-07-07 08:22:46,471 - meshic_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.qi_population_metrics using ID column 'grid_id' -2025-07-07 08:22:46,481 - meshic_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 136 rows in public.qi_population_metrics. -2025-07-07 08:22:46,483 - meshic_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: qi_population_metrics --- -2025-07-07 08:22:46,483 - meshic_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'qi_population_metrics': 0.00MB -2025-07-07 08:22:46,483 - meshic_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: qi_stripes --- -2025-07-07 08:22:47,376 - meshic_pipeline.pipeline_orchestrator - WARNING - Dropped 105 duplicate rows for primary key 'strip_id' in layer 'qi_stripes' before DB write. -2025-07-07 08:22:47,382 - meshic_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_qi_stripes from known columns -2025-07-07 08:22:47,403 - meshic_pipeline.persistence.postgis_persister - INFO - Persisted 259 features to public.temp_qi_stripes using mode 'replace' +2025-07-07 08:22:46,466 - suhail_pipeline.pipeline_orchestrator - INFO - Persisting 136 features for layer 'qi_population_metrics' to table 'qi_population_metrics' +2025-07-07 08:22:46,471 - suhail_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.qi_population_metrics using ID column 'grid_id' +2025-07-07 08:22:46,481 - suhail_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 136 rows in public.qi_population_metrics. +2025-07-07 08:22:46,483 - suhail_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: qi_population_metrics --- +2025-07-07 08:22:46,483 - suhail_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'qi_population_metrics': 0.00MB +2025-07-07 08:22:46,483 - suhail_pipeline.pipeline_orchestrator - INFO - --- Starting processing for layer: qi_stripes --- +2025-07-07 08:22:47,376 - suhail_pipeline.pipeline_orchestrator - WARNING - Dropped 105 duplicate rows for primary key 'strip_id' in layer 'qi_stripes' before DB write. +2025-07-07 08:22:47,382 - suhail_pipeline.persistence.postgis_persister - INFO - Successfully created table public.temp_qi_stripes from known columns +2025-07-07 08:22:47,403 - suhail_pipeline.persistence.postgis_persister - INFO - Persisted 259 features to public.temp_qi_stripes using mode 'replace' 2025-07-07 08:22:47,415 - pyogrio._io - INFO - Created 259 records -2025-07-07 08:22:47,419 - meshic_pipeline.pipeline_orchestrator - INFO - Persisting 259 features for layer 'qi_stripes' to table 'qi_stripes' -2025-07-07 08:22:47,424 - meshic_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.qi_stripes using ID column 'strip_id' -2025-07-07 08:22:47,437 - meshic_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 259 rows in public.qi_stripes. -2025-07-07 08:22:47,439 - meshic_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: qi_stripes --- -2025-07-07 08:22:47,440 - meshic_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'qi_stripes': 0.00MB -2025-07-07 08:22:47,440 - meshic_pipeline.pipeline_orchestrator - INFO - 🎉 Pipeline finished successfully. 🎉 -2025-07-07 08:22:47,440 - meshic_pipeline.pipeline_orchestrator - INFO - Total memory change during run: 0.00MB +2025-07-07 08:22:47,419 - suhail_pipeline.pipeline_orchestrator - INFO - Persisting 259 features for layer 'qi_stripes' to table 'qi_stripes' +2025-07-07 08:22:47,424 - suhail_pipeline.persistence.postgis_persister - INFO - Performing upsert on public.qi_stripes using ID column 'strip_id' +2025-07-07 08:22:47,437 - suhail_pipeline.persistence.postgis_persister - INFO - Upsert complete. Affected 259 rows in public.qi_stripes. +2025-07-07 08:22:47,439 - suhail_pipeline.pipeline_orchestrator - INFO - --- Finished processing for layer: qi_stripes --- +2025-07-07 08:22:47,440 - suhail_pipeline.pipeline_orchestrator - INFO - Memory delta for layer 'qi_stripes': 0.00MB +2025-07-07 08:22:47,440 - suhail_pipeline.pipeline_orchestrator - INFO - 🎉 Pipeline finished successfully. 🎉 +2025-07-07 08:22:47,440 - suhail_pipeline.pipeline_orchestrator - INFO - Total memory change during run: 0.00MB diff --git a/docs/archive/generated-txt/vulture_report.txt b/docs/archive/generated-txt/vulture_report.txt index 9fe2e52..021efd8 100644 --- a/docs/archive/generated-txt/vulture_report.txt +++ b/docs/archive/generated-txt/vulture_report.txt @@ -1,152 +1,152 @@ -src/meshic_pipeline/cli.py:13: unused function 'geometric' (60% confidence) -src/meshic_pipeline/cli.py:79: unused function 'smart_pipeline' (60% confidence) -src/meshic_pipeline/cli.py:105: unused function 'province_geometric' (60% confidence) -src/meshic_pipeline/cli.py:121: unused function 'saudi_arabia_geometric' (60% confidence) -src/meshic_pipeline/cli.py:136: unused function 'discovery_summary' (60% confidence) -src/meshic_pipeline/cli.py:143: unused function 'province_pipeline' (60% confidence) -src/meshic_pipeline/cli.py:158: unused function 'saudi_pipeline' (60% confidence) -src/meshic_pipeline/config.py:40: unused variable 'STAGING' (60% confidence) -src/meshic_pipeline/config.py:78: unused variable 'model_config' (60% confidence) -src/meshic_pipeline/config.py:100: unused method 'build_urls' (60% confidence) -src/meshic_pipeline/config.py:118: unused variable 'model_config' (60% confidence) -src/meshic_pipeline/config.py:196: unused variable 'aggregation_rules_per_layer' (60% confidence) -src/meshic_pipeline/config.py:303: unused method 'ensure_dirs' (60% confidence) -src/meshic_pipeline/decoder/mvt_decoder.py:48: unused attribute 'quarantined_features' (60% confidence) -src/meshic_pipeline/decoder/mvt_decoder.py:195: unused method 'decode_to_gdf' (60% confidence) -src/meshic_pipeline/discovery/tile_discovery.py:19: unused import 'wkb' (90% confidence) -src/meshic_pipeline/discovery/tile_discovery.py:24: unused variable 'ZOOM15' (60% confidence) -src/meshic_pipeline/downloader/async_tile_downloader.py:53: unused variable 'exc_type' (100% confidence) -src/meshic_pipeline/downloader/async_tile_downloader.py:53: unused variable 'tb' (100% confidence) -src/meshic_pipeline/enrichment/api_client.py:264: unused function 'apply_arabic_column_mapping_dict' (60% confidence) -src/meshic_pipeline/exceptions.py:19: unused import 'Union' (90% confidence) -src/meshic_pipeline/exceptions.py:26: unused variable 'T' (60% confidence) -src/meshic_pipeline/exceptions.py:47: unused variable 'AUTHENTICATION' (60% confidence) -src/meshic_pipeline/exceptions.py:120: unused class 'DatabaseException' (60% confidence) -src/meshic_pipeline/exceptions.py:132: unused class 'ValidationException' (60% confidence) -src/meshic_pipeline/exceptions.py:144: unused class 'ConfigurationException' (60% confidence) -src/meshic_pipeline/exceptions.py:157: unused class 'ProcessingException' (60% confidence) -src/meshic_pipeline/exceptions.py:181: unused class 'FileSystemException' (60% confidence) -src/meshic_pipeline/exceptions.py:258: unused method 'get_error_stats' (60% confidence) -src/meshic_pipeline/exceptions.py:398: unused function 'handle_exceptions' (60% confidence) -src/meshic_pipeline/geometry/stitcher.py:22: unused function '_dissolve_group' (60% confidence) -src/meshic_pipeline/geometry/stitcher.py:25: unused variable 'geom_column' (100% confidence) -src/meshic_pipeline/geometry/stitcher.py:111: unused method 'stitch_geometries' (60% confidence) -src/meshic_pipeline/geometry/stitcher.py:204: unused method 'stitch_from_table' (60% confidence) -src/meshic_pipeline/logging_utils.py:24: unused import 'Union' (90% confidence) -src/meshic_pipeline/logging_utils.py:148: unused method 'shouldFlush' (60% confidence) -src/meshic_pipeline/logging_utils.py:234: unused method 'log_metrics' (60% confidence) -src/meshic_pipeline/logging_utils.py:247: unused method 'get_operation_stats' (60% confidence) -src/meshic_pipeline/logging_utils.py:281: unused method 'set_correlation_id' (60% confidence) -src/meshic_pipeline/logging_utils.py:314: unused method 'critical' (60% confidence) -src/meshic_pipeline/logging_utils.py:323: unused method 'operation_context' (60% confidence) -src/meshic_pipeline/logging_utils.py:508: unused function 'log_function_calls' (60% confidence) -src/meshic_pipeline/memory_utils.py:36: unused variable 'total_mb' (60% confidence) -src/meshic_pipeline/memory_utils.py:37: unused variable 'available_mb' (60% confidence) -src/meshic_pipeline/memory_utils.py:38: unused variable 'used_mb' (60% confidence) -src/meshic_pipeline/memory_utils.py:39: unused variable 'percent_used' (60% confidence) -src/meshic_pipeline/memory_utils.py:41: unused variable 'gc_collections' (60% confidence) -src/meshic_pipeline/memory_utils.py:144: unused method 'register_object' (60% confidence) -src/meshic_pipeline/memory_utils.py:155: unused method 'get_memory_history' (60% confidence) -src/meshic_pipeline/memory_utils.py:164: unused method 'get_memory_trend' (60% confidence) -src/meshic_pipeline/memory_utils.py:338: unused function 'batch_processor' (60% confidence) -src/meshic_pipeline/persistence/db.py:44: unused function 'setup_database' (60% confidence) -src/meshic_pipeline/persistence/enrichment_persister.py:5: unused import 'ceil' (90% confidence) -src/meshic_pipeline/persistence/models.py:2: unused import 'select' (90% confidence) -src/meshic_pipeline/persistence/models.py:18: unused import 'load_only' (90% confidence) -src/meshic_pipeline/persistence/models.py:24: unused class 'Parcel' (60% confidence) -src/meshic_pipeline/persistence/models.py:30: unused variable 'landuseagroup' (60% confidence) -src/meshic_pipeline/persistence/models.py:31: unused variable 'landuseadetailed' (60% confidence) -src/meshic_pipeline/persistence/models.py:32: unused variable 'subdivision_no' (60% confidence) -src/meshic_pipeline/persistence/models.py:35: unused variable 'neighborhood_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:36: unused variable 'block_no' (60% confidence) -src/meshic_pipeline/persistence/models.py:37: unused variable 'neighborhood_ar' (60% confidence) -src/meshic_pipeline/persistence/models.py:38: unused variable 'subdivision_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:40: unused variable 'shape_area' (60% confidence) -src/meshic_pipeline/persistence/models.py:42: unused variable 'ruleid' (60% confidence) -src/meshic_pipeline/persistence/models.py:43: unused variable 'province_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:44: unused variable 'municipality_ar' (60% confidence) -src/meshic_pipeline/persistence/models.py:45: unused variable 'parcel_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:46: unused variable 'parcel_no' (60% confidence) -src/meshic_pipeline/persistence/models.py:49: unused variable 'created_at' (60% confidence) -src/meshic_pipeline/persistence/models.py:50: unused variable 'updated_at' (60% confidence) -src/meshic_pipeline/persistence/models.py:51: unused variable 'is_active' (60% confidence) -src/meshic_pipeline/persistence/models.py:52: unused variable 'geometry_hash' (60% confidence) -src/meshic_pipeline/persistence/models.py:53: unused variable 'enriched_at' (60% confidence) -src/meshic_pipeline/persistence/models.py:55: unused variable 'neighborhood' (60% confidence) -src/meshic_pipeline/persistence/models.py:57: unused variable 'zoning_rule' (60% confidence) -src/meshic_pipeline/persistence/models.py:59: unused variable 'price_metrics' (60% confidence) -src/meshic_pipeline/persistence/models.py:60: unused variable 'building_rules' (60% confidence) -src/meshic_pipeline/persistence/models.py:74: unused variable 'parcel' (60% confidence) -src/meshic_pipeline/persistence/models.py:83: unused variable 'metric_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:89: unused variable 'neighborhood_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:91: unused variable 'parcel' (60% confidence) -src/meshic_pipeline/persistence/models.py:92: unused variable 'neighborhood' (60% confidence) -src/meshic_pipeline/persistence/models.py:122: unused variable 'parcel' (60% confidence) -src/meshic_pipeline/persistence/models.py:128: unused class 'Neighborhood' (60% confidence) -src/meshic_pipeline/persistence/models.py:130: unused variable 'neighborhood_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:132: unused variable 'neighborhood_name' (60% confidence) -src/meshic_pipeline/persistence/models.py:133: unused variable 'neighborhood_ar' (60% confidence) -src/meshic_pipeline/persistence/models.py:134: unused variable 'region_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:135: unused variable 'province_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:137: unused variable 'shape_area' (60% confidence) -src/meshic_pipeline/persistence/models.py:141: unused variable 'geometry_hash' (60% confidence) -src/meshic_pipeline/persistence/models.py:144: unused variable 'price_metrics' (60% confidence) -src/meshic_pipeline/persistence/models.py:160: unused variable 'province_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:162: unused variable 'province_name_ar' (60% confidence) -src/meshic_pipeline/persistence/models.py:169: unused variable 'bbox_sw_lon' (60% confidence) -src/meshic_pipeline/persistence/models.py:170: unused variable 'bbox_sw_lat' (60% confidence) -src/meshic_pipeline/persistence/models.py:171: unused variable 'bbox_ne_lon' (60% confidence) -src/meshic_pipeline/persistence/models.py:172: unused variable 'bbox_ne_lat' (60% confidence) -src/meshic_pipeline/persistence/models.py:173: unused variable 'region_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:177: unused variable 'subdivisions' (60% confidence) -src/meshic_pipeline/persistence/models.py:179: unused class 'Subdivision' (60% confidence) -src/meshic_pipeline/persistence/models.py:181: unused variable 'subdivision_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:183: unused variable 'subdivision_no' (60% confidence) -src/meshic_pipeline/persistence/models.py:184: unused variable 'shape_area' (60% confidence) -src/meshic_pipeline/persistence/models.py:189: unused variable 'province_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:193: unused class 'ParcelsBase' (60% confidence) -src/meshic_pipeline/persistence/models.py:195: unused variable 'parcel_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:198: unused class 'ZoningRule' (60% confidence) -src/meshic_pipeline/persistence/models.py:200: unused variable 'ruleid' (60% confidence) -src/meshic_pipeline/persistence/models.py:205: unused class 'LandUseGroup' (60% confidence) -src/meshic_pipeline/persistence/models.py:207: unused variable 'landuse_group' (60% confidence) -src/meshic_pipeline/persistence/models.py:210: unused class 'NeighborhoodsCentroids' (60% confidence) -src/meshic_pipeline/persistence/models.py:214: unused variable 'neighborh_aname' (60% confidence) -src/meshic_pipeline/persistence/models.py:215: unused variable 'province_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:217: unused class 'MetroLines' (60% confidence) -src/meshic_pipeline/persistence/models.py:221: unused variable 'track_color' (60% confidence) -src/meshic_pipeline/persistence/models.py:222: unused variable 'track_length' (60% confidence) -src/meshic_pipeline/persistence/models.py:223: unused variable 'track_name' (60% confidence) -src/meshic_pipeline/persistence/models.py:225: unused class 'ParcelsCentroids' (60% confidence) -src/meshic_pipeline/persistence/models.py:227: unused variable 'parcel_no' (60% confidence) -src/meshic_pipeline/persistence/models.py:233: unused class 'BusLines' (60% confidence) -src/meshic_pipeline/persistence/models.py:237: unused variable 'busroute' (60% confidence) -src/meshic_pipeline/persistence/models.py:238: unused variable 'route_name' (60% confidence) -src/meshic_pipeline/persistence/models.py:239: unused variable 'route_type' (60% confidence) -src/meshic_pipeline/persistence/models.py:241: unused class 'MetroStations' (60% confidence) -src/meshic_pipeline/persistence/models.py:243: unused variable 'station_code' (60% confidence) -src/meshic_pipeline/persistence/models.py:245: unused variable 'station_name' (60% confidence) -src/meshic_pipeline/persistence/models.py:246: unused variable 'line' (60% confidence) -src/meshic_pipeline/persistence/models.py:248: unused class 'RiyadhBusStations' (60% confidence) -src/meshic_pipeline/persistence/models.py:250: unused variable 'station_code' (60% confidence) -src/meshic_pipeline/persistence/models.py:252: unused variable 'station_name' (60% confidence) -src/meshic_pipeline/persistence/models.py:253: unused variable 'route' (60% confidence) -src/meshic_pipeline/persistence/models.py:255: unused class 'QIPopulationMetrics' (60% confidence) -src/meshic_pipeline/persistence/models.py:257: unused variable 'grid_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:258: unused variable 'population' (60% confidence) -src/meshic_pipeline/persistence/models.py:261: unused class 'QIStripes' (60% confidence) -src/meshic_pipeline/persistence/models.py:263: unused variable 'strip_id' (60% confidence) -src/meshic_pipeline/persistence/models.py:272: unused variable 'zoom_level' (60% confidence) -src/meshic_pipeline/persistence/models.py:279: unused variable 'created_at' (60% confidence) -src/meshic_pipeline/persistence/models.py:280: unused variable 'updated_at' (60% confidence) -src/meshic_pipeline/persistence/models.py:282: unused method 'fetch_tiles_by_status' (60% confidence) -src/meshic_pipeline/pipeline_orchestrator.py:6: unused import 'concurrent' (90% confidence) -src/meshic_pipeline/pipeline_orchestrator.py:65: unused function 'enrich_parcels_with_region_id' (60% confidence) -src/meshic_pipeline/run_enrichment_pipeline.py:32: unused import 'sessionmaker' (90% confidence) -src/meshic_pipeline/run_enrichment_pipeline.py:34: unused import 'ValidationException' (90% confidence) -src/meshic_pipeline/run_monitoring.py:70: unused function 'recommend' (60% confidence) -src/meshic_pipeline/run_monitoring.py:77: unused variable 'trigger_parcels' (60% confidence) -src/meshic_pipeline/run_monitoring.py:166: unused function 'schedule_info' (60% confidence) +src/suhail_pipeline/cli.py:13: unused function 'geometric' (60% confidence) +src/suhail_pipeline/cli.py:79: unused function 'smart_pipeline' (60% confidence) +src/suhail_pipeline/cli.py:105: unused function 'province_geometric' (60% confidence) +src/suhail_pipeline/cli.py:121: unused function 'saudi_arabia_geometric' (60% confidence) +src/suhail_pipeline/cli.py:136: unused function 'discovery_summary' (60% confidence) +src/suhail_pipeline/cli.py:143: unused function 'province_pipeline' (60% confidence) +src/suhail_pipeline/cli.py:158: unused function 'saudi_pipeline' (60% confidence) +src/suhail_pipeline/config.py:40: unused variable 'STAGING' (60% confidence) +src/suhail_pipeline/config.py:78: unused variable 'model_config' (60% confidence) +src/suhail_pipeline/config.py:100: unused method 'build_urls' (60% confidence) +src/suhail_pipeline/config.py:118: unused variable 'model_config' (60% confidence) +src/suhail_pipeline/config.py:196: unused variable 'aggregation_rules_per_layer' (60% confidence) +src/suhail_pipeline/config.py:303: unused method 'ensure_dirs' (60% confidence) +src/suhail_pipeline/decoder/mvt_decoder.py:48: unused attribute 'quarantined_features' (60% confidence) +src/suhail_pipeline/decoder/mvt_decoder.py:195: unused method 'decode_to_gdf' (60% confidence) +src/suhail_pipeline/discovery/tile_discovery.py:19: unused import 'wkb' (90% confidence) +src/suhail_pipeline/discovery/tile_discovery.py:24: unused variable 'ZOOM15' (60% confidence) +src/suhail_pipeline/downloader/async_tile_downloader.py:53: unused variable 'exc_type' (100% confidence) +src/suhail_pipeline/downloader/async_tile_downloader.py:53: unused variable 'tb' (100% confidence) +src/suhail_pipeline/enrichment/api_client.py:264: unused function 'apply_arabic_column_mapping_dict' (60% confidence) +src/suhail_pipeline/exceptions.py:19: unused import 'Union' (90% confidence) +src/suhail_pipeline/exceptions.py:26: unused variable 'T' (60% confidence) +src/suhail_pipeline/exceptions.py:47: unused variable 'AUTHENTICATION' (60% confidence) +src/suhail_pipeline/exceptions.py:120: unused class 'DatabaseException' (60% confidence) +src/suhail_pipeline/exceptions.py:132: unused class 'ValidationException' (60% confidence) +src/suhail_pipeline/exceptions.py:144: unused class 'ConfigurationException' (60% confidence) +src/suhail_pipeline/exceptions.py:157: unused class 'ProcessingException' (60% confidence) +src/suhail_pipeline/exceptions.py:181: unused class 'FileSystemException' (60% confidence) +src/suhail_pipeline/exceptions.py:258: unused method 'get_error_stats' (60% confidence) +src/suhail_pipeline/exceptions.py:398: unused function 'handle_exceptions' (60% confidence) +src/suhail_pipeline/geometry/stitcher.py:22: unused function '_dissolve_group' (60% confidence) +src/suhail_pipeline/geometry/stitcher.py:25: unused variable 'geom_column' (100% confidence) +src/suhail_pipeline/geometry/stitcher.py:111: unused method 'stitch_geometries' (60% confidence) +src/suhail_pipeline/geometry/stitcher.py:204: unused method 'stitch_from_table' (60% confidence) +src/suhail_pipeline/logging_utils.py:24: unused import 'Union' (90% confidence) +src/suhail_pipeline/logging_utils.py:148: unused method 'shouldFlush' (60% confidence) +src/suhail_pipeline/logging_utils.py:234: unused method 'log_metrics' (60% confidence) +src/suhail_pipeline/logging_utils.py:247: unused method 'get_operation_stats' (60% confidence) +src/suhail_pipeline/logging_utils.py:281: unused method 'set_correlation_id' (60% confidence) +src/suhail_pipeline/logging_utils.py:314: unused method 'critical' (60% confidence) +src/suhail_pipeline/logging_utils.py:323: unused method 'operation_context' (60% confidence) +src/suhail_pipeline/logging_utils.py:508: unused function 'log_function_calls' (60% confidence) +src/suhail_pipeline/memory_utils.py:36: unused variable 'total_mb' (60% confidence) +src/suhail_pipeline/memory_utils.py:37: unused variable 'available_mb' (60% confidence) +src/suhail_pipeline/memory_utils.py:38: unused variable 'used_mb' (60% confidence) +src/suhail_pipeline/memory_utils.py:39: unused variable 'percent_used' (60% confidence) +src/suhail_pipeline/memory_utils.py:41: unused variable 'gc_collections' (60% confidence) +src/suhail_pipeline/memory_utils.py:144: unused method 'register_object' (60% confidence) +src/suhail_pipeline/memory_utils.py:155: unused method 'get_memory_history' (60% confidence) +src/suhail_pipeline/memory_utils.py:164: unused method 'get_memory_trend' (60% confidence) +src/suhail_pipeline/memory_utils.py:338: unused function 'batch_processor' (60% confidence) +src/suhail_pipeline/persistence/db.py:44: unused function 'setup_database' (60% confidence) +src/suhail_pipeline/persistence/enrichment_persister.py:5: unused import 'ceil' (90% confidence) +src/suhail_pipeline/persistence/models.py:2: unused import 'select' (90% confidence) +src/suhail_pipeline/persistence/models.py:18: unused import 'load_only' (90% confidence) +src/suhail_pipeline/persistence/models.py:24: unused class 'Parcel' (60% confidence) +src/suhail_pipeline/persistence/models.py:30: unused variable 'landuseagroup' (60% confidence) +src/suhail_pipeline/persistence/models.py:31: unused variable 'landuseadetailed' (60% confidence) +src/suhail_pipeline/persistence/models.py:32: unused variable 'subdivision_no' (60% confidence) +src/suhail_pipeline/persistence/models.py:35: unused variable 'neighborhood_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:36: unused variable 'block_no' (60% confidence) +src/suhail_pipeline/persistence/models.py:37: unused variable 'neighborhood_ar' (60% confidence) +src/suhail_pipeline/persistence/models.py:38: unused variable 'subdivision_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:40: unused variable 'shape_area' (60% confidence) +src/suhail_pipeline/persistence/models.py:42: unused variable 'ruleid' (60% confidence) +src/suhail_pipeline/persistence/models.py:43: unused variable 'province_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:44: unused variable 'municipality_ar' (60% confidence) +src/suhail_pipeline/persistence/models.py:45: unused variable 'parcel_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:46: unused variable 'parcel_no' (60% confidence) +src/suhail_pipeline/persistence/models.py:49: unused variable 'created_at' (60% confidence) +src/suhail_pipeline/persistence/models.py:50: unused variable 'updated_at' (60% confidence) +src/suhail_pipeline/persistence/models.py:51: unused variable 'is_active' (60% confidence) +src/suhail_pipeline/persistence/models.py:52: unused variable 'geometry_hash' (60% confidence) +src/suhail_pipeline/persistence/models.py:53: unused variable 'enriched_at' (60% confidence) +src/suhail_pipeline/persistence/models.py:55: unused variable 'neighborhood' (60% confidence) +src/suhail_pipeline/persistence/models.py:57: unused variable 'zoning_rule' (60% confidence) +src/suhail_pipeline/persistence/models.py:59: unused variable 'price_metrics' (60% confidence) +src/suhail_pipeline/persistence/models.py:60: unused variable 'building_rules' (60% confidence) +src/suhail_pipeline/persistence/models.py:74: unused variable 'parcel' (60% confidence) +src/suhail_pipeline/persistence/models.py:83: unused variable 'metric_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:89: unused variable 'neighborhood_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:91: unused variable 'parcel' (60% confidence) +src/suhail_pipeline/persistence/models.py:92: unused variable 'neighborhood' (60% confidence) +src/suhail_pipeline/persistence/models.py:122: unused variable 'parcel' (60% confidence) +src/suhail_pipeline/persistence/models.py:128: unused class 'Neighborhood' (60% confidence) +src/suhail_pipeline/persistence/models.py:130: unused variable 'neighborhood_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:132: unused variable 'neighborhood_name' (60% confidence) +src/suhail_pipeline/persistence/models.py:133: unused variable 'neighborhood_ar' (60% confidence) +src/suhail_pipeline/persistence/models.py:134: unused variable 'region_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:135: unused variable 'province_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:137: unused variable 'shape_area' (60% confidence) +src/suhail_pipeline/persistence/models.py:141: unused variable 'geometry_hash' (60% confidence) +src/suhail_pipeline/persistence/models.py:144: unused variable 'price_metrics' (60% confidence) +src/suhail_pipeline/persistence/models.py:160: unused variable 'province_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:162: unused variable 'province_name_ar' (60% confidence) +src/suhail_pipeline/persistence/models.py:169: unused variable 'bbox_sw_lon' (60% confidence) +src/suhail_pipeline/persistence/models.py:170: unused variable 'bbox_sw_lat' (60% confidence) +src/suhail_pipeline/persistence/models.py:171: unused variable 'bbox_ne_lon' (60% confidence) +src/suhail_pipeline/persistence/models.py:172: unused variable 'bbox_ne_lat' (60% confidence) +src/suhail_pipeline/persistence/models.py:173: unused variable 'region_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:177: unused variable 'subdivisions' (60% confidence) +src/suhail_pipeline/persistence/models.py:179: unused class 'Subdivision' (60% confidence) +src/suhail_pipeline/persistence/models.py:181: unused variable 'subdivision_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:183: unused variable 'subdivision_no' (60% confidence) +src/suhail_pipeline/persistence/models.py:184: unused variable 'shape_area' (60% confidence) +src/suhail_pipeline/persistence/models.py:189: unused variable 'province_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:193: unused class 'ParcelsBase' (60% confidence) +src/suhail_pipeline/persistence/models.py:195: unused variable 'parcel_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:198: unused class 'ZoningRule' (60% confidence) +src/suhail_pipeline/persistence/models.py:200: unused variable 'ruleid' (60% confidence) +src/suhail_pipeline/persistence/models.py:205: unused class 'LandUseGroup' (60% confidence) +src/suhail_pipeline/persistence/models.py:207: unused variable 'landuse_group' (60% confidence) +src/suhail_pipeline/persistence/models.py:210: unused class 'NeighborhoodsCentroids' (60% confidence) +src/suhail_pipeline/persistence/models.py:214: unused variable 'neighborh_aname' (60% confidence) +src/suhail_pipeline/persistence/models.py:215: unused variable 'province_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:217: unused class 'MetroLines' (60% confidence) +src/suhail_pipeline/persistence/models.py:221: unused variable 'track_color' (60% confidence) +src/suhail_pipeline/persistence/models.py:222: unused variable 'track_length' (60% confidence) +src/suhail_pipeline/persistence/models.py:223: unused variable 'track_name' (60% confidence) +src/suhail_pipeline/persistence/models.py:225: unused class 'ParcelsCentroids' (60% confidence) +src/suhail_pipeline/persistence/models.py:227: unused variable 'parcel_no' (60% confidence) +src/suhail_pipeline/persistence/models.py:233: unused class 'BusLines' (60% confidence) +src/suhail_pipeline/persistence/models.py:237: unused variable 'busroute' (60% confidence) +src/suhail_pipeline/persistence/models.py:238: unused variable 'route_name' (60% confidence) +src/suhail_pipeline/persistence/models.py:239: unused variable 'route_type' (60% confidence) +src/suhail_pipeline/persistence/models.py:241: unused class 'MetroStations' (60% confidence) +src/suhail_pipeline/persistence/models.py:243: unused variable 'station_code' (60% confidence) +src/suhail_pipeline/persistence/models.py:245: unused variable 'station_name' (60% confidence) +src/suhail_pipeline/persistence/models.py:246: unused variable 'line' (60% confidence) +src/suhail_pipeline/persistence/models.py:248: unused class 'RiyadhBusStations' (60% confidence) +src/suhail_pipeline/persistence/models.py:250: unused variable 'station_code' (60% confidence) +src/suhail_pipeline/persistence/models.py:252: unused variable 'station_name' (60% confidence) +src/suhail_pipeline/persistence/models.py:253: unused variable 'route' (60% confidence) +src/suhail_pipeline/persistence/models.py:255: unused class 'QIPopulationMetrics' (60% confidence) +src/suhail_pipeline/persistence/models.py:257: unused variable 'grid_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:258: unused variable 'population' (60% confidence) +src/suhail_pipeline/persistence/models.py:261: unused class 'QIStripes' (60% confidence) +src/suhail_pipeline/persistence/models.py:263: unused variable 'strip_id' (60% confidence) +src/suhail_pipeline/persistence/models.py:272: unused variable 'zoom_level' (60% confidence) +src/suhail_pipeline/persistence/models.py:279: unused variable 'created_at' (60% confidence) +src/suhail_pipeline/persistence/models.py:280: unused variable 'updated_at' (60% confidence) +src/suhail_pipeline/persistence/models.py:282: unused method 'fetch_tiles_by_status' (60% confidence) +src/suhail_pipeline/pipeline_orchestrator.py:6: unused import 'concurrent' (90% confidence) +src/suhail_pipeline/pipeline_orchestrator.py:65: unused function 'enrich_parcels_with_region_id' (60% confidence) +src/suhail_pipeline/run_enrichment_pipeline.py:32: unused import 'sessionmaker' (90% confidence) +src/suhail_pipeline/run_enrichment_pipeline.py:34: unused import 'ValidationException' (90% confidence) +src/suhail_pipeline/run_monitoring.py:70: unused function 'recommend' (60% confidence) +src/suhail_pipeline/run_monitoring.py:77: unused variable 'trigger_parcels' (60% confidence) +src/suhail_pipeline/run_monitoring.py:166: unused function 'schedule_info' (60% confidence) tests/integration/test_pipeline_integration.py:59: unused variable 'exc_type' (100% confidence) tests/integration/test_pipeline_integration.py:59: unused variable 'tb' (100% confidence) tests/integration/test_pipeline_integration.py:152: unused variable 'exc_type' (100% confidence) diff --git a/docs/archive/legacy-root-md/CONFIGURATION_FIXES_TEST_REPORT.md b/docs/archive/legacy-root-md/CONFIGURATION_FIXES_TEST_REPORT.md index 56a0b89..ffc30eb 100644 --- a/docs/archive/legacy-root-md/CONFIGURATION_FIXES_TEST_REPORT.md +++ b/docs/archive/legacy-root-md/CONFIGURATION_FIXES_TEST_REPORT.md @@ -132,7 +132,7 @@ The most important finding is that **16,807 tiles from 3 major regions were succ 3. Fixing this could instantly populate 10 empty provinces **Next investigation should focus on:** -- `src/meshic_pipeline/run_db_geometric.py` - province assignment logic +- `src/suhail_pipeline/run_db_geometric.py` - province assignment logic - Province polygon boundaries in PostGIS - Spatial join queries and coordinate transformations diff --git a/docs/archive/legacy-root-md/CONFIGURATION_ISSUES_ANALYSIS.md b/docs/archive/legacy-root-md/CONFIGURATION_ISSUES_ANALYSIS.md index 06931b9..47423c9 100644 --- a/docs/archive/legacy-root-md/CONFIGURATION_ISSUES_ANALYSIS.md +++ b/docs/archive/legacy-root-md/CONFIGURATION_ISSUES_ANALYSIS.md @@ -169,9 +169,9 @@ WHERE tile_server_url = 'https://tiles.suhail.ai/makkah_region/'; ```bash # After fixes applied -meshic-pipeline province-geometric makkah -meshic-pipeline province-geometric eastern_region -meshic-pipeline province-geometric al_madenieh +suhail-pipeline province-geometric makkah +suhail-pipeline province-geometric eastern_region +suhail-pipeline province-geometric al_madenieh # ... etc ``` diff --git a/docs/archive/legacy-root-md/DATABASE_ARCHITECTURE_ANALYSIS.md b/docs/archive/legacy-root-md/DATABASE_ARCHITECTURE_ANALYSIS.md index 94fb987..5942a55 100644 --- a/docs/archive/legacy-root-md/DATABASE_ARCHITECTURE_ANALYSIS.md +++ b/docs/archive/legacy-root-md/DATABASE_ARCHITECTURE_ANALYSIS.md @@ -1,13 +1,13 @@ # Database Architecture Analysis Report -**Project**: Meshic Real Estate Data Pipeline +**Project**: Suhail Real Estate Data Pipeline **Date**: December 10, 2024 **Analyst**: Senior Database Architect **Status**: Production System (2.3M+ parcels) ## Executive Summary -The Meshic pipeline database demonstrates **solid foundational design** for a geospatial real estate platform but suffers from **critical performance bottlenecks** that will severely impact operations at the current scale. Analysis reveals missing indexes on 2.3M+ parcel foreign keys, data redundancy across geographic hierarchy levels, and normalization violations that create maintenance overhead. +The Suhail pipeline database demonstrates **solid foundational design** for a geospatial real estate platform but suffers from **critical performance bottlenecks** that will severely impact operations at the current scale. Analysis reveals missing indexes on 2.3M+ parcel foreign keys, data redundancy across geographic hierarchy levels, and normalization violations that create maintenance overhead. **Key Finding**: The system can achieve **60-80% performance improvement** through safe index additions without breaking the existing ETL pipeline. @@ -239,7 +239,7 @@ ORDER BY idx_scan DESC; ## Conclusion -The Meshic database architecture demonstrates **strong foundational design** but requires **immediate performance optimization** to support current scale effectively. The most critical improvements (indexes) can be implemented safely without pipeline disruption, providing substantial performance gains. +The Suhail database architecture demonstrates **strong foundational design** but requires **immediate performance optimization** to support current scale effectively. The most critical improvements (indexes) can be implemented safely without pipeline disruption, providing substantial performance gains. **Recommended Action**: Proceed immediately with Phase 1 implementation for critical performance improvements, then evaluate Phase 2+ based on operational priorities and development bandwidth. diff --git a/docs/archive/legacy-root-md/FINAL_RECOVERY_REPORT.md b/docs/archive/legacy-root-md/FINAL_RECOVERY_REPORT.md index 637790a..a6a747a 100644 --- a/docs/archive/legacy-root-md/FINAL_RECOVERY_REPORT.md +++ b/docs/archive/legacy-root-md/FINAL_RECOVERY_REPORT.md @@ -7,7 +7,7 @@ ## 🎯 Executive Summary -**MASSIVE SUCCESS**: We successfully diagnosed, fixed, and recovered from critical configuration and spatial assignment issues in the Meshic pipeline, resulting in the recovery of nearly 800,000 previously "lost" parcels and achieving 99.98% province assignment rate across Saudi Arabia. +**MASSIVE SUCCESS**: We successfully diagnosed, fixed, and recovered from critical configuration and spatial assignment issues in the Suhail pipeline, resulting in the recovery of nearly 800,000 previously "lost" parcels and achieving 99.98% province assignment rate across Saudi Arabia. --- @@ -179,7 +179,7 @@ The system is now **production-ready** for: 4. **Enrichment pipeline**: Begin API enrichment with assigned parcels ### **System Capabilities** -The Meshic pipeline now supports: +The Suhail pipeline now supports: - ✅ **Multi-province concurrent processing** - ✅ **Automatic spatial assignments** - ✅ **Proper Arabic/English naming** diff --git a/docs/archive/legacy-root-md/PERFORMANCE_OPTIMIZATION_REPORT.md b/docs/archive/legacy-root-md/PERFORMANCE_OPTIMIZATION_REPORT.md index 8edacad..74cd3b4 100644 --- a/docs/archive/legacy-root-md/PERFORMANCE_OPTIMIZATION_REPORT.md +++ b/docs/archive/legacy-root-md/PERFORMANCE_OPTIMIZATION_REPORT.md @@ -1,7 +1,7 @@ # Database Performance Optimization Report **Date**: December 10, 2024 -**Project**: Meshic Real Estate Data Pipeline +**Project**: Suhail Real Estate Data Pipeline **Migration**: `19c587b33197_add_critical_performance_indexes` **Status**: ✅ **COMPLETED SUCCESSFULLY** @@ -9,7 +9,7 @@ ## 🎯 Executive Summary -Critical database performance optimizations have been **successfully implemented** for the Meshic pipeline, providing **60-80% performance improvement** for core operations without breaking existing ETL pipeline functionality. +Critical database performance optimizations have been **successfully implemented** for the Suhail pipeline, providing **60-80% performance improvement** for core operations without breaking existing ETL pipeline functionality. ### 🔑 Key Achievements - ✅ **27 performance indexes** created across critical tables @@ -200,7 +200,7 @@ Based on operational needs and development bandwidth: ## 📊 Validation Results ### Database Connection -✅ Connected to database: `meshic` +✅ Connected to database: `suhail` ### Performance Indexes ✅ **27 performance indexes** successfully created @@ -219,7 +219,7 @@ Based on operational needs and development bandwidth: ## 🏆 Conclusion -The Meshic database performance optimization has been **successfully completed** with **zero risk** to existing operations. The pipeline is now ready for: +The Suhail database performance optimization has been **successfully completed** with **zero risk** to existing operations. The pipeline is now ready for: - **✅ Province-wide processing** with optimal performance - **✅ All-Saudi scrapes** handling 6M+ parcels efficiently diff --git a/docs/archive/memory-bank/activeContext.md b/docs/archive/memory-bank/activeContext.md index 8433b15..91dbbef 100644 --- a/docs/archive/memory-bank/activeContext.md +++ b/docs/archive/memory-bank/activeContext.md @@ -122,8 +122,8 @@ source .venv/bin/activate uv add -e . # Core testing commands -meshic-pipeline geometric # 3x3 baseline test -meshic-pipeline fast-enrich --limit 100 # Enrichment validation +suhail-pipeline geometric # 3x3 baseline test +suhail-pipeline fast-enrich --limit 100 # Enrichment validation python scripts/check_db.py # Database validation ``` @@ -226,20 +226,20 @@ See `docs/archive/legacy-root-md/DATABASE_ARCHITECTURE_ANALYSIS.md` for comprehe > For a complete, up-to-date audit, see [`docs/CLI_COMMAND_AUDIT.md`](../docs/CLI_COMMAND_AUDIT.md) and the README. ### Core Commands -- `meshic-pipeline geometric [--bbox ...] [--recreate-db] [--save-as-temp ...]` -- `meshic-pipeline fast-enrich [--batch-size ...] [--limit ...]` -- `meshic-pipeline incremental-enrich [--batch-size ...] [--days-old ...] [--limit ...]` -- `meshic-pipeline full-refresh [--batch-size ...] [--limit ...]` -- `meshic-pipeline delta-enrich [--batch-size ...] [--limit ...] [--fresh-table ...] [--auto-geometric] [--show-details/--no-details]` +- `suhail-pipeline geometric [--bbox ...] [--recreate-db] [--save-as-temp ...]` +- `suhail-pipeline fast-enrich [--batch-size ...] [--limit ...]` +- `suhail-pipeline incremental-enrich [--batch-size ...] [--days-old ...] [--limit ...]` +- `suhail-pipeline full-refresh [--batch-size ...] [--limit ...]` +- `suhail-pipeline delta-enrich [--batch-size ...] [--limit ...] [--fresh-table ...] [--auto-geometric] [--show-details/--no-details]` ### Advanced/Composite Commands -- `meshic-pipeline smart-pipeline [--geometric-first] [--batch-size ...] [--bbox ...]` -- `meshic-pipeline monitor ` -- `meshic-pipeline province-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` -- `meshic-pipeline saudi-arabia-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` -- `meshic-pipeline discovery-summary` -- `meshic-pipeline province-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` -- `meshic-pipeline saudi-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` +- `suhail-pipeline smart-pipeline [--geometric-first] [--batch-size ...] [--bbox ...]` +- `suhail-pipeline monitor ` +- `suhail-pipeline province-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` +- `suhail-pipeline saudi-arabia-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` +- `suhail-pipeline discovery-summary` +- `suhail-pipeline province-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` +- `suhail-pipeline saudi-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` ### Workflow Recommendations - **Baseline/Small Grid:** Use `geometric` and `fast-enrich` for initial validation. diff --git a/docs/archive/memory-bank/progress.md b/docs/archive/memory-bank/progress.md index cebf47d..24148e5 100644 --- a/docs/archive/memory-bank/progress.md +++ b/docs/archive/memory-bank/progress.md @@ -31,7 +31,7 @@ **Status**: Complete **Tasks**: -- [x] **Package Installation**: Install meshic-pipeline with `uv sync --all-groups` +- [x] **Package Installation**: Install suhail-pipeline with `uv sync --all-groups` - [x] **Environment Activation**: Source virtual environment - [x] **Database Connection**: Verify PostgreSQL/PostGIS connectivity - [x] **Configuration Validation**: Confirm pipeline settings @@ -47,7 +47,7 @@ - **Purpose**: Confirm basic pipeline functionality with fresh database **Tasks**: -- [x] **Execute Geometric Pipeline**: Run `meshic-pipeline geometric` +- [x] **Execute Geometric Pipeline**: Run `suhail-pipeline geometric` - [x] **Verify Database Population**: Check parcels and reference tables - [x] **Validate Data Types**: Confirm proper schema alignment - [x] **Check Foreign Keys**: Verify relationship integrity @@ -63,7 +63,7 @@ **Status**: Complete **Tasks**: -- [x] **API Integration Test**: Run `meshic-pipeline fast-enrich --limit 100` (**Success: 100 parcels processed, 10 transactions, 0 building rules, 200 price metrics added**) +- [x] **API Integration Test**: Run `suhail-pipeline fast-enrich --limit 100` (**Success: 100 parcels processed, 10 transactions, 0 building rules, 200 price metrics added**) - [x] **Success Rate Monitoring**: Track enrichment coverage percentage (see Results below) - [x] **Endpoint Validation**: All 3 API endpoints responsive (transactions, building rules, price metrics) - [x] **Data Quality Check**: Enrichment data written to DB, no errors @@ -308,20 +308,20 @@ CREATE INDEX idx_parcel_price_metrics_neighborhood_id ON parcel_price_metrics(ne > For a complete, up-to-date audit, see [`docs/CLI_COMMAND_AUDIT.md`](../docs/CLI_COMMAND_AUDIT.md) and the README. ### Core Commands -- `meshic-pipeline geometric [--bbox ...] [--recreate-db] [--save-as-temp ...]` -- `meshic-pipeline fast-enrich [--batch-size ...] [--limit ...]` -- `meshic-pipeline incremental-enrich [--batch-size ...] [--days-old ...] [--limit ...]` -- `meshic-pipeline full-refresh [--batch-size ...] [--limit ...]` -- `meshic-pipeline delta-enrich [--batch-size ...] [--limit ...] [--fresh-table ...] [--auto-geometric] [--show-details/--no-details]` +- `suhail-pipeline geometric [--bbox ...] [--recreate-db] [--save-as-temp ...]` +- `suhail-pipeline fast-enrich [--batch-size ...] [--limit ...]` +- `suhail-pipeline incremental-enrich [--batch-size ...] [--days-old ...] [--limit ...]` +- `suhail-pipeline full-refresh [--batch-size ...] [--limit ...]` +- `suhail-pipeline delta-enrich [--batch-size ...] [--limit ...] [--fresh-table ...] [--auto-geometric] [--show-details/--no-details]` ### Advanced/Composite Commands -- `meshic-pipeline smart-pipeline [--geometric-first] [--batch-size ...] [--bbox ...]` -- `meshic-pipeline monitor ` -- `meshic-pipeline province-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` -- `meshic-pipeline saudi-arabia-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` -- `meshic-pipeline discovery-summary` -- `meshic-pipeline province-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` -- `meshic-pipeline saudi-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` +- `suhail-pipeline smart-pipeline [--geometric-first] [--batch-size ...] [--bbox ...]` +- `suhail-pipeline monitor ` +- `suhail-pipeline province-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` +- `suhail-pipeline saudi-arabia-geometric [--strategy ...] [--recreate-db] [--save-as-temp ...]` +- `suhail-pipeline discovery-summary` +- `suhail-pipeline province-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` +- `suhail-pipeline saudi-pipeline [--strategy ...] [--batch-size ...] [--geometric-first]` ### Phase-by-Phase Command Usage - **Baseline/3x3 Grid:** diff --git a/docs/archive/memory-bank/projectbrief.md b/docs/archive/memory-bank/projectbrief.md index 0db8306..e13c7b2 100644 --- a/docs/archive/memory-bank/projectbrief.md +++ b/docs/archive/memory-bank/projectbrief.md @@ -1,4 +1,4 @@ -# Project Brief: Meshic Geospatial Data Pipeline +# Project Brief: Suhail Geospatial Data Pipeline ## Project Overview Commercial geospatial data processing pipeline for capturing Saudi Arabian land parcel data from MVT tiles. Built to extract and enrich real estate data for client sales and analytics products. diff --git a/docs/archive/memory-bank/systemPatterns.md b/docs/archive/memory-bank/systemPatterns.md index b079d92..1b46540 100644 --- a/docs/archive/memory-bank/systemPatterns.md +++ b/docs/archive/memory-bank/systemPatterns.md @@ -3,7 +3,7 @@ ## 🏗️ **Core Architecture: DB-Driven, Functional Async Design** ### **Design Philosophy** -The Meshic pipeline uses **database-driven, functional async patterns** for high-performance geospatial data processing at scale. All tile discovery and orchestration is managed via the `tile_urls` table in the database, supporting province-wide and all-Saudi scrapes with resumable processing. +The Suhail pipeline uses **database-driven, functional async patterns** for high-performance geospatial data processing at scale. All tile discovery and orchestration is managed via the `tile_urls` table in the database, supporting province-wide and all-Saudi scrapes with resumable processing. ### **Key Architectural Principles** 1. **DB-Driven Tile Orchestration**: All tiles to be processed are stored in the `tile_urls` table; pipeline queries for pending/failed tiles and updates status as it processes. diff --git a/docs/archive/memory-bank/techContext.md b/docs/archive/memory-bank/techContext.md index 8fcd929..14c2120 100644 --- a/docs/archive/memory-bank/techContext.md +++ b/docs/archive/memory-bank/techContext.md @@ -1,4 +1,4 @@ -# Technical Context: Meshic Geospatial Pipeline +# Technical Context: Suhail Geospatial Pipeline ## 🛠️ **Technology Stack** @@ -46,7 +46,7 @@ Stage 2: PostGIS → API Enrichment → Enhanced PostGIS ## 🔄 **Current Implementation Status** ### **Environment Setup: READY** -- **Package**: meshic-pipeline ready for installation via `uv add -e .` +- **Package**: suhail-pipeline ready for installation via `uv add -e .` - **Dependencies**: All required packages specified in pyproject.toml - **Configuration**: Province-specific settings configured in pipeline_config.yaml - **Database**: Fresh schema with spatial extensions enabled @@ -92,7 +92,7 @@ Stage 2: PostGIS → API Enrichment → Enhanced PostGIS database: host: ${DB_HOST:localhost} port: ${DB_PORT:5432} - name: ${DB_NAME:meshic_pipeline} + name: ${DB_NAME:suhail_pipeline} processing: max_concurrent_downloads: 5 @@ -112,8 +112,8 @@ source .venv/bin/activate uv add -e . # Core testing commands -meshic-pipeline geometric # 3x3 baseline test -meshic-pipeline fast-enrich --limit 100 # Enrichment validation +suhail-pipeline geometric # 3x3 baseline test +suhail-pipeline fast-enrich --limit 100 # Enrichment validation python scripts/check_db.py # Database validation ``` diff --git a/docs/brownfield-architecture.md b/docs/brownfield-architecture.md index 426b0ac..47c9bcd 100644 --- a/docs/brownfield-architecture.md +++ b/docs/brownfield-architecture.md @@ -18,11 +18,11 @@ This document captures the **CURRENT STATE** of the Suhail Final geospatial data ### Critical Files for Understanding the System -- **Main CLI**: `src/meshic_pipeline/cli.py` (Comprehensive command-line interface with 15+ commands) -- **Configuration**: `pipeline_config.yaml`, `src/meshic_pipeline/config.py` (Multi-environment configuration) -- **Geometric Pipeline**: `src/meshic_pipeline/run_geometric_pipeline.py` (Stage 1: MVT processing) -- **Enrichment Pipeline**: `src/meshic_pipeline/run_enrichment_pipeline.py` (Stage 2: Business intelligence) -- **Database Models**: `src/meshic_pipeline/persistence/models.py` (SQLAlchemy/PostGIS models) +- **Main CLI**: `src/suhail_pipeline/cli.py` (Comprehensive command-line interface with 15+ commands) +- **Configuration**: `pipeline_config.yaml`, `src/suhail_pipeline/config.py` (Multi-environment configuration) +- **Geometric Pipeline**: `src/suhail_pipeline/run_geometric_pipeline.py` (Stage 1: MVT processing) +- **Enrichment Pipeline**: `src/suhail_pipeline/run_enrichment_pipeline.py` (Stage 2: Business intelligence) +- **Database Models**: `src/suhail_pipeline/persistence/models.py` (SQLAlchemy/PostGIS models) - **Database Schema**: `schema_dump.sql` (Complete PostgreSQL schema with 20+ tables) ### Current Implementation State @@ -65,7 +65,7 @@ Suhail Final is a sophisticated geospatial data processing pipeline that transfo ``` /Users/raedmund/Projects/suhail_final/ -├── src/meshic_pipeline/ # ✅ IMPLEMENTED: Core pipeline modules +├── src/suhail_pipeline/ # ✅ IMPLEMENTED: Core pipeline modules │ ├── cli.py # ✅ IMPLEMENTED: CLI with 15+ commands │ ├── config.py # ✅ IMPLEMENTED: Pydantic configuration │ ├── decoder/ # ✅ IMPLEMENTED: MVT tile decoding @@ -142,7 +142,7 @@ The enrichment system integrates with Suhail APIs through sophisticated patterns ### Critical Technical Debt 1. **Complex CLI Architecture** - - Location: `src/meshic_pipeline/cli.py` + - Location: `src/suhail_pipeline/cli.py` - Impact: 15+ commands with subprocess calls instead of direct imports - Status: ⚠️ **MAINTENANCE**: Some commands use subprocess instead of direct function calls @@ -194,7 +194,7 @@ The enrichment system integrates with Suhail APIs through sophisticated patterns 2. **Dependencies**: `uv sync --all-groups` for reproducible dependency management 3. **Database**: Local PostgreSQL with PostGIS extensions 4. **Configuration**: Environment variables + YAML config files -5. **Execution**: `meshic-pipeline` CLI with 15+ commands +5. **Execution**: `suhail-pipeline` CLI with 15+ commands ### Build and Deployment Process (Actual) @@ -253,25 +253,25 @@ uv run pytest tests/integration # Integration tests ```bash # Geometric pipeline (Stage 1) -meshic-pipeline geometric --bbox 46.4 24.3 47.0 24.8 -meshic-pipeline province-geometric riyadh --strategy optimal -meshic-pipeline saudi-arabia-geometric --strategy efficient +suhail-pipeline geometric --bbox 46.4 24.3 47.0 24.8 +suhail-pipeline province-geometric riyadh --strategy optimal +suhail-pipeline saudi-arabia-geometric --strategy efficient # Enrichment pipeline (Stage 2) - Multiple strategies -meshic-pipeline fast-enrich --batch-size 400 # New parcels only -meshic-pipeline incremental-enrich --days-old 7 --batch-size 100 # Weekly updates -meshic-pipeline delta-enrich --auto-geometric # Change detection -meshic-pipeline full-refresh --batch-size 50 # Complete refresh +suhail-pipeline fast-enrich --batch-size 400 # New parcels only +suhail-pipeline incremental-enrich --days-old 7 --batch-size 100 # Weekly updates +suhail-pipeline delta-enrich --auto-geometric # Change detection +suhail-pipeline full-refresh --batch-size 50 # Complete refresh # Monitoring and management -meshic-pipeline monitor status # Current status -meshic-pipeline monitor recommend # Smart recommendations -meshic-pipeline monitor schedule-info # Scheduling guidance +suhail-pipeline monitor status # Current status +suhail-pipeline monitor recommend # Smart recommendations +suhail-pipeline monitor schedule-info # Scheduling guidance # Complete workflows -meshic-pipeline smart-pipeline --batch-size 300 --geometric-first -meshic-pipeline province-pipeline riyadh --strategy optimal --batch-size 300 -meshic-pipeline saudi-pipeline --strategy efficient --batch-size 500 +suhail-pipeline smart-pipeline --batch-size 300 --geometric-first +suhail-pipeline province-pipeline riyadh --strategy optimal --batch-size 300 +suhail-pipeline saudi-pipeline --strategy efficient --batch-size 500 # Database management alembic upgrade head # Apply migrations diff --git a/docs/docs-distillate/01-requirements-and-product.md b/docs/docs-distillate/01-requirements-and-product.md index 9e05ae3..fc4e8a8 100644 --- a/docs/docs-distillate/01-requirements-and-product.md +++ b/docs/docs-distillate/01-requirements-and-product.md @@ -94,9 +94,9 @@ This section covers requirements, product scope, and acceptance themes. Part 1 o - Index migrations via Alembic without downtime; baseline and post-migration timings recorded and linked in runbook. ## Acceptance: monitoring (Epic 2) -- `meshic-pipeline monitor status`: queue counts by status, top errors, age of oldest `in_progress`. -- `meshic-pipeline monitor recommend`: scheduling guidance (next enrichment strategy, batch sizes). -- `meshic-pipeline monitor schedule-info`: recommended cadence from freshness. +- `suhail-pipeline monitor status`: queue counts by status, top errors, age of oldest `in_progress`. +- `suhail-pipeline monitor recommend`: scheduling guidance (next enrichment strategy, batch sizes). +- `suhail-pipeline monitor schedule-info`: recommended cadence from freshness. - Scheduled job resets stale `in_progress` after configurable threshold (example 60 minutes); log count affected. - Tiles/hour and enrichment counts per run in logs; optional CSV or dashboard export. @@ -146,4 +146,4 @@ This section covers requirements, product scope, and acceptance themes. Part 1 o ## PRD process and references - PRD author Mary (Business Analyst); date 2025-10-16; project level 2 (focused PRD + solutioning handoff). -- Technical references: `docs/BROWNFIELD_PROJECT_DOCUMENTATION.md`, `docs/archive/legacy-root-md/DATABASE_ARCHITECTURE_ANALYSIS.md`, Alembic index migration `19c587b33197_add_critical_performance_indexes.py`, `src/meshic_pipeline/cli.py`, `models.py`, `run_db_geometric.py`. +- Technical references: `docs/BROWNFIELD_PROJECT_DOCUMENTATION.md`, `docs/archive/legacy-root-md/DATABASE_ARCHITECTURE_ANALYSIS.md`, Alembic index migration `19c587b33197_add_critical_performance_indexes.py`, `src/suhail_pipeline/cli.py`, `models.py`, `run_db_geometric.py`. diff --git a/docs/docs-distillate/02-architecture-and-data.md b/docs/docs-distillate/02-architecture-and-data.md index 1a48ff4..c0bfb19 100644 --- a/docs/docs-distillate/02-architecture-and-data.md +++ b/docs/docs-distillate/02-architecture-and-data.md @@ -1,18 +1,18 @@ This section covers architecture, schema, and ground-truth production state. Part 2 of 3. ## Identity and scope -- Meshic / Suhail Final: two-stage geospatial pipeline for Saudi Arabian parcel intelligence (MVT geometry + Suhail API enrichment). +- Suhail / Suhail Final: two-stage geospatial pipeline for Saudi Arabian parcel intelligence (MVT geometry + Suhail API enrichment). - Lean architecture doc dated 2025-10-16 v0.1; brownfield doc “current state”; brownfield project doc dated 2025-10-16 analyst Mary; production narrative cites 2.16M+ parcels in executive summary. ## Inputs and storage - Inputs: province-specific Mapbox Vector Tiles (MVT); Suhail APIs (transactions, building rules, price metrics). -- System of record: PostgreSQL + PostGIS; control plane: Typer CLI `meshic-pipeline`. +- System of record: PostgreSQL + PostGIS; control plane: Typer CLI `suhail-pipeline`. ## Stage 1 geometric - DB-queued tile downloads, decode, stitch, validate, persist; seed from `provinces.bbox_z15` into `tile_urls`. - Workers claim with `SELECT ... FOR UPDATE SKIP LOCKED`; statuses pending → in_progress → processed/failed. - Downloader: aiohttp concurrent fetch with retry/backoff; decoder: MVT → EPSG:4326; stitcher: PostGIS dissolve across tiles (temp tables); persister: schema-driven upsert. -- Files: `src/meshic_pipeline/run_db_geometric.py`, `decoder/mvt_decoder.py`, `geometry/stitcher.py`, `persistence/postgis_persister.py`, `persistence/table_management.py`; brownfield also cites `run_geometric_pipeline.py` for Stage 1 MVT processing. +- Files: `src/suhail_pipeline/run_db_geometric.py`, `decoder/mvt_decoder.py`, `geometry/stitcher.py`, `persistence/postgis_persister.py`, `persistence/table_management.py`; brownfield also cites `run_geometric_pipeline.py` for Stage 1 MVT processing. - Flow: seed `tile_urls` → claim batch → fetch → decode (Arabic column normalization) → validate CRS/geometry → stitch → persist temp then canonical → update queue processed/failed. - Adaptive geometric concurrency 5–20; architecture doc: multiple geometric workers share `tile_urls` queue. - Brownfield project doc: ~0.05s delay between requests; chunked DB writes ~5000 rows per batch; garbage collection triggers for large runs. @@ -25,14 +25,14 @@ This section covers architecture, schema, and ground-truth production state. Par - universal-metrics: all parcels including those without transaction_price. - delta-enrich: parcels with changed transaction_price via MVT comparison; optional auto geometric; FULL OUTER JOIN `parcels` vs fresh table on `parcel_objectid`. - API client batch/async; transform JSON → SQLAlchemy; upsert with conflict handling; update `parcels.enriched_at`; brownfield project doc notes `ON CONFLICT DO NOTHING` on persistence path. -- Files: `src/meshic_pipeline/run_enrichment_pipeline.py`, `enrichment/strategies.py`, `enrichment/api_client.py`, `persistence/enrichment_persister.py`; brownfield adds `enrichment/processor.py`, `metrics_only_processor.py`. +- Files: `src/suhail_pipeline/run_enrichment_pipeline.py`, `enrichment/strategies.py`, `enrichment/api_client.py`, `persistence/enrichment_persister.py`; brownfield adds `enrichment/processor.py`, `metrics_only_processor.py`. ## Suhail HTTP surface - `/parcel/buildingRules`; `/api/parcel/metrics/priceOfMeter`; `/transactions`; API key/session; rate limiting and retry/backoff referenced. ## Queue and models - `tile_urls(id PK, url UNIQUE, z/x/y, status, retry_count, last_checked_at, error_message, ...)`. -- `TileURL.claim_tiles_for_processing(..., SKIP LOCKED)` and `reset_stale_in_progress(..., stale_minutes=60)` in `src/meshic_pipeline/persistence/models.py`. +- `TileURL.claim_tiles_for_processing(..., SKIP LOCKED)` and `reset_stale_in_progress(..., stale_minutes=60)` in `src/suhail_pipeline/persistence/models.py`. - Stale protection: periodic reset stuck in_progress tiles; schedule example stale_minutes=60. ## Core tables and keys @@ -55,7 +55,7 @@ This section covers architecture, schema, and ground-truth production state. Par - `building_rules`: 130,112 rows. - `tile_urls`: 34,726 rows; processed 33,938 (97.7%); in_progress 788 (2.3%). - `provinces`: 12; `neighborhoods`: 812. -- Database name `meshic` (not `meshic_pipeline` as some docs); owner `postgres`. +- Database name `suhail` (not `suhail_pipeline` as some docs); owner `postgres`. ## Older / alternate scale figures (deduped caveat) - Brownfield architecture doc: 1M+ parcels, 45K+ transactions, 68K+ building rules, 752K+ parcel_price_metrics — superseded by ground-truth table counts where they conflict. @@ -74,18 +74,18 @@ This section covers architecture, schema, and ground-truth production state. Par - Composite: `smart-pipeline`, `province-pipeline`, `saudi-pipeline`. - Ops: `seed-tiles` (province, provinces, region-slugs, stride, limit), `discovery-summary`, `monitor status|recommend|schedule-info`. - Command count: ground-truth doc states 18 commands in 5 categories; brownfield architecture says 15+; architecture lists representative subset without total. -- Entry: `src/meshic_pipeline/cli.py`. +- Entry: `src/suhail_pipeline/cli.py`. ## Configuration -- `src/meshic_pipeline/config.py`: Pydantic settings; DB URL, API endpoints, layers, batch sizes; provinces loaded from DB via `load_provinces_from_db`; failure falls back to empty provinces with warning. +- `src/suhail_pipeline/config.py`: Pydantic settings; DB URL, API endpoints, layers, batch sizes; provinces loaded from DB via `load_provinces_from_db`; failure falls back to empty provinces with warning. - `pipeline_config.yaml` + environment variables; province tile URL template pattern `https://tiles.suhail.ai/maps/{slug}/{z}/{x}/{y}.vector.pbf` (Riyadh example in doc). - Secrets: `.env`; no secrets in code; least-privilege DB role; UTF-8 Arabic. ## Tech stack -- Python 3.9+; package `meshic-pipeline` v0.1.0 per doc snippet; SQLAlchemy 2.0+ async; GeoAlchemy2; GeoPandas, Shapely, h3; Alembic; Typer; aiohttp; mapbox-vector-tile/protobuf for MVT. +- Python 3.9+; package `suhail-pipeline` v0.1.0 per doc snippet; SQLAlchemy 2.0+ async; GeoAlchemy2; GeoPandas, Shapely, h3; Alembic; Typer; aiohttp; mapbox-vector-tile/protobuf for MVT. ## Repo layout -- `src/meshic_pipeline/` cli, config, decoder, discovery, downloader, enrichment, geometry, persistence, `pipeline_orchestrator.py`, utils; `alembic/`, `tests/`, `scripts/`, `logs/`, `docs/`, `pyproject.toml`. +- `src/suhail_pipeline/` cli, config, decoder, discovery, downloader, enrichment, geometry, persistence, `pipeline_orchestrator.py`, utils; `alembic/`, `tests/`, `scripts/`, `logs/`, `docs/`, `pyproject.toml`. ## NFR targets (architecture) - Geometric: adaptive concurrency, minimal retries; query p95 < 500 ms post-index for core joins/metrics reads. @@ -101,7 +101,7 @@ This section covers architecture, schema, and ground-truth production state. Par - Risks: missing indexes, stuck in_progress tiles, API rate limits, outdated stakeholder docs. ## Deployment and ops -- Dev/staging/prod with shared migrations; local: PostgreSQL+PostGIS, `uv sync --all-groups`, `uv run meshic-pipeline …`, `uv run alembic upgrade head`. +- Dev/staging/prod with shared migrations; local: PostgreSQL+PostGIS, `uv sync --all-groups`, `uv run suhail-pipeline …`, `uv run alembic upgrade head`. - Temp table policy: pipeline-owned `temp_*` only. ## Roadmap and recommendations (compressed) diff --git a/docs/docs-distillate/03-stories-ops-cli.md b/docs/docs-distillate/03-stories-ops-cli.md index 71ccbd5..6295bca 100644 --- a/docs/docs-distillate/03-stories-ops-cli.md +++ b/docs/docs-distillate/03-stories-ops-cli.md @@ -5,7 +5,7 @@ This section covers implementation stories, CLI inventory, ops, and runbooks. Pa - STORY-001: Verify Alembic migration `19c587b33197_add_critical_performance_indexes.py` in staging; run in prod low-traffic; confirm via `pg_indexes` and `EXPLAIN`; record timings in `docs/reports/perf-post-.md`. - STORY-001: Acceptance: indexes per `docs/ACCEPTANCE_CRITERIA.md`; p95 < 500 ms; migration without downtime/errors; DoD: reports + runbook. - STORY-002: Baseline/post-index p50/p95 for 3 representative queries; `docs/reports/perf-baseline-.md` and `perf-post-.md`; p95 target < 500 ms or remediation plan. -- STORY-003: Enhance `meshic-pipeline monitor`: `status` (queue by status, oldest `in_progress`, top errors); `recommend` (next action + batch sizes); `schedule-info` (cadence); structured log summary per run. +- STORY-003: Enhance `suhail-pipeline monitor`: `status` (queue by status, oldest `in_progress`, top errors); `recommend` (next action + batch sizes); `schedule-info` (cadence); structured log summary per run. - STORY-004: Schedule `TileURL.reset_stale_in_progress(..., stale_minutes=60)`; log reset count; alert if excessive repeats; runbook + monitoring reflects queue health. - STORY-005: Delta enrichment: `--auto-geometric` fresh-table lifecycle; change stats/summary; cron guidance; fresh table dropped on success, retained on failure with warning. - STORY-006: Data quality Phase 1: completeness `%` with `neighborhood_id`, `province_id`, `enriched_at`; outlier thresholds `price_of_meter`, `transaction_price`; `docs/reports/data-quality-.md` by province. @@ -20,12 +20,12 @@ This section covers implementation stories, CLI inventory, ops, and runbooks. Pa - Index targets (`parcel_price_metrics`): `neighborhood_id`; composites `(metrics_type, year, month)` and `(neighborhood_id, metrics_type)`. - Index targets (`transactions`): `transaction_date`, `transaction_price`. - Measurement queries: (1) `parcels ↔ neighborhoods` on `neighborhood_id`; (2) `parcels` filter `transaction_price > 0` and `enriched_at`; (3) time-series `parcel_price_metrics` by `(neighborhood_id, metrics_type, year, month)`. -- `meshic-pipeline monitor status`: queue counts by `status`, failed count, oldest `in_progress` age; enrichment: total parcels, with `enriched_at`, % with `transaction_price > 0`. -- `meshic-pipeline monitor recommend`: next action among fast-enrich / incremental / delta with batch sizes. -- `meshic-pipeline monitor schedule-info`: suggested daily/weekly/monthly cadence from freshness. -- AC mentions: `meshic-pipeline monitor reset-stale -- --stale-minutes 60`; reports `docs/reports/perf-baseline-*.md` and `docs/reports/perf-post-index-*.md` (naming variant vs stories). +- `suhail-pipeline monitor status`: queue counts by `status`, failed count, oldest `in_progress` age; enrichment: total parcels, with `enriched_at`, % with `transaction_price > 0`. +- `suhail-pipeline monitor recommend`: next action among fast-enrich / incremental / delta with batch sizes. +- `suhail-pipeline monitor schedule-info`: suggested daily/weekly/monthly cadence from freshness. +- AC mentions: `suhail-pipeline monitor reset-stale -- --stale-minutes 60`; reports `docs/reports/perf-baseline-*.md` and `docs/reports/perf-post-index-*.md` (naming variant vs stories). - Risks: index build time/locks → `IF NOT EXISTS`, off-hours; plan regressions → `EXPLAIN ANALYZE`; limit CLI scope vs dashboards. -- Utilities referenced: `claim_tiles_for_processing(...)`, `TileURL.reset_stale_in_progress(...)` in `src/meshic_pipeline/persistence/models.py`. +- Utilities referenced: `claim_tiles_for_processing(...)`, `TileURL.reset_stale_in_progress(...)` in `src/suhail_pipeline/persistence/models.py`. ## CLI inventory and documentation gaps (`CLI_COMMAND_AUDIT.md`) - Commands: `geometric` (`--bbox`, `--recreate-db`, `--save-as-temp`); `fast-enrich` (`--batch-size`, `--limit`); `incremental-enrich` (`--batch-size`, `--days-old`, `--limit`); `full-refresh` (`--batch-size`, `--limit`). @@ -42,9 +42,9 @@ This section covers implementation stories, CLI inventory, ops, and runbooks. Pa ## Province-wide scraping (DB-driven MVT) - Authoritative: `provinces` metadata; `tile_urls` tile list/status; config from DB; no hard-coded province dicts/YAML tile lists. - Downloader: async; province + tiles from DB; caching, retry, concurrency; resumable multi/all-province. -- CLI examples: `meshic-pipeline geometric --province riyadh`; `meshic-pipeline geometric --all-provinces`; multi `--province riyadh --province makkah`. +- CLI examples: `suhail-pipeline geometric --province riyadh`; `suhail-pipeline geometric --all-provinces`; multi `--province riyadh --province makkah`. - Performance: index `tile_urls` by status/province; spatial/lookup indices on parcels/neighborhoods at >1M/province; ~5000-row chunks (settings-tunable); disk tile cache. -- CI suggestion: nightly `meshic-pipeline geometric --province riyadh --limit-test`. +- CI suggestion: nightly `suhail-pipeline geometric --province riyadh --limit-test`. - Rollout phases: DB metadata+generator → downloader/refactor → orchestrator+CLI `--all-provinces` → tests/CI → docs → full Riyadh → six provinces (success metrics in doc table). - Risks: rate limit (0.05s delay, backoff); memory (stream, chunk tuning); DB province drift (runtime validation). @@ -74,7 +74,7 @@ This section covers implementation stories, CLI inventory, ops, and runbooks. Pa - `brew services stop --all`; `brew uninstall postgresql postgresql@14 postgresql@15 postgis`. - Remove data dirs: Apple Silicon `rm -rf /opt/homebrew/var/postgres*`; Intel `rm -rf /usr/local/var/postgres*`. - `brew install postgis` (or pin `postgresql@14` + postgis); example `initdb --locale=C -E UTF-8 /opt/homebrew/var/postgresql@16`. -- `brew services start postgresql@16`; `createdb meshic -T template0`; `psql -d meshic -c "CREATE EXTENSION postgis;"`. +- `brew services start postgresql@16`; `createdb suhail -T template0`; `psql -d suhail -c "CREATE EXTENSION postgis;"`. - Python: from repo root, `uv sync --all-groups` (see `README.md`). - Alembic: init, configure, autogenerate initial revision, manual spatial index checks, `uv run alembic upgrade head`. -- Verify: `psql -d meshic -c "\dt"`, `\d+ your_spatial_table`, `SELECT * FROM geometry_columns;`; run app tests. +- Verify: `psql -d suhail -c "\dt"`, `\d+ your_spatial_table`, `SELECT * FROM geometry_columns;`; run app tests. diff --git a/docs/docs-distillate/_index-validation-report.md b/docs/docs-distillate/_index-validation-report.md index 9aed5ed..6f1981e 100644 --- a/docs/docs-distillate/_index-validation-report.md +++ b/docs/docs-distillate/_index-validation-report.md @@ -32,7 +32,7 @@ created: "2026-03-20" - **Per-story narrative and DoD detail:** `docs/stories/STORY-*.md` body text is folded into theme bullets in `03-stories-ops-cli.md`; edge acceptance notes and formatting from each story file are not losslessly encoded. - **Line-level ops prose:** `docs/ops/migrations.md`, `province_wide_scraping_plan.md`, and `CLI_COMMAND_AUDIT.md` tables/examples: distillate keeps command names and recommendations, not every example block. -- **BROWNFIELD_PROJECT_DOCUMENTATION.md:** Long sections (e.g. extended code excerpts, full table attribute lists) are compressed; **counts and DB name `meshic`** are present; some secondary tables and narrative asides may be absent from bullets. +- **BROWNFIELD_PROJECT_DOCUMENTATION.md:** Long sections (e.g. extended code excerpts, full table attribute lists) are compressed; **counts and DB name `suhail`** are present; some secondary tables and narrative asides may be absent from bullets. - **EPIC-001** risk tables and long checklist prose: captured as index/monitoring/stale-reset bullets; not every risk row is duplicated. - **Reconstructor self-markers:** `reconstruction-PRD-bundle.md` flags **[POSSIBLE GAP]** for full SQL specs, exit-code matrix, and complete CLI error catalog — these are thin or absent in sources too; treat as “distillate correctly reflects source depth.” @@ -52,7 +52,7 @@ created: "2026-03-20" | PRD Stage 1–2 + delta | `docs/PRD.md` §Requirements | `01-requirements-and-product.md` — aligned | | Monitoring FR | `docs/PRD.md` §4 | `01` acceptance + `03` EPIC-001/stories — aligned | | Scale 2.16M / 76M metrics | PRD context + brownfield | `01` + `02` — aligned | -| DB name `meshic` | BROWNFIELD doc | `02` — aligned | +| DB name `suhail` | BROWNFIELD doc | `02` — aligned | | STORY-001–008 IDs | stories | `03` — aligned | ## Recommendation diff --git a/docs/docs-distillate/_validation/reconstruction-ARCH-bundle.md b/docs/docs-distillate/_validation/reconstruction-ARCH-bundle.md index 292869c..8cd4b24 100644 --- a/docs/docs-distillate/_validation/reconstruction-ARCH-bundle.md +++ b/docs/docs-distillate/_validation/reconstruction-ARCH-bundle.md @@ -13,11 +13,11 @@ This document expands the **docs distillate** (architecture-and-data section plu ## System identity and scope -**Meshic / Suhail Final** is a **two-stage geospatial pipeline** for **Saudi Arabian parcel intelligence**: **MVT geometry** ingestion plus **Suhail API enrichment**. The lean architecture document is dated **2025-10-16**, version **0.1**; the brownfield document describes **current state**; the brownfield project documentation is dated **2025-10-16** (analyst Mary). An executive summary in production narrative cites **2.16M+ parcels**. +**Suhail / Suhail Final** is a **two-stage geospatial pipeline** for **Saudi Arabian parcel intelligence**: **MVT geometry** ingestion plus **Suhail API enrichment**. The lean architecture document is dated **2025-10-16**, version **0.1**; the brownfield document describes **current state**; the brownfield project documentation is dated **2025-10-16** (analyst Mary). An executive summary in production narrative cites **2.16M+ parcels**. ## Inputs, system of record, and control plane -**Inputs** are **province-specific Mapbox Vector Tiles (MVT)** and **Suhail APIs** for **transactions**, **building rules**, and **price metrics**. The **system of record** is **PostgreSQL with PostGIS**. The **control plane** is the **Typer** CLI **`meshic-pipeline`**. +**Inputs** are **province-specific Mapbox Vector Tiles (MVT)** and **Suhail APIs** for **transactions**, **building rules**, and **price metrics**. The **system of record** is **PostgreSQL with PostGIS**. The **control plane** is the **Typer** CLI **`suhail-pipeline`**. ## Stage 1: geometric pipeline (architecture) @@ -25,7 +25,7 @@ The geometric stage performs **DB-queued tile downloads**, **decode**, **stitch* The **downloader** uses **aiohttp** with **concurrent fetch**, **retry**, and **backoff**. The **decoder** converts **MVT to EPSG:4326**. The **stitcher** uses **PostGIS dissolve** across tiles (**temporary tables**). The **persister** performs **schema-driven upsert**. -Implementation paths cited: **`src/meshic_pipeline/run_db_geometric.py`**, **`decoder/mvt_decoder.py`**, **`geometry/stitcher.py`**, **`persistence/postgis_persister.py`**, **`persistence/table_management.py`**. Brownfield also references **`run_geometric_pipeline.py`** for Stage 1 MVT processing **[POSSIBLE GAP: relationship and deprecation story between `run_db_geometric` and `run_geometric_pipeline`].** +Implementation paths cited: **`src/suhail_pipeline/run_db_geometric.py`**, **`decoder/mvt_decoder.py`**, **`geometry/stitcher.py`**, **`persistence/postgis_persister.py`**, **`persistence/table_management.py`**. Brownfield also references **`run_geometric_pipeline.py`** for Stage 1 MVT processing **[POSSIBLE GAP: relationship and deprecation story between `run_db_geometric` and `run_geometric_pipeline`].** End-to-end flow: **seed `tile_urls`** → **claim batch** → **fetch** → **decode** (with **Arabic column normalization**) → **validate CRS/geometry** → **stitch** → **persist temp then canonical** → **update queue** to processed or failed. **Adaptive geometric concurrency** is described as **5–20**; **multiple geometric workers** share the **`tile_urls`** queue. Brownfield project notes include roughly **0.05s delay between requests**, **chunked DB writes** around **5000 rows per batch**, and **garbage collection** triggers for large runs. @@ -51,7 +51,7 @@ Endpoints named: **`/parcel/buildingRules`**; **`/api/parcel/metrics/priceOfMete **`tile_urls`** includes **`id` PK**, **`url` UNIQUE**, **z/x/y**, **`status`**, **`retry_count`**, **`last_checked_at`**, **`error_message`**, and further columns implied by ellipsis in the distillate. -**`TileURL.claim_tiles_for_processing(..., SKIP LOCKED)`** and **`reset_stale_in_progress(..., stale_minutes=60)`** live in **`src/meshic_pipeline/persistence/models.py`**. **Stale protection** means **periodic reset** of stuck **`in_progress`** tiles; schedule example **`stale_minutes=60`**. +**`TileURL.claim_tiles_for_processing(..., SKIP LOCKED)`** and **`reset_stale_in_progress(..., stale_minutes=60)`** live in **`src/suhail_pipeline/persistence/models.py`**. **Stale protection** means **periodic reset** of stuck **`in_progress`** tiles; schedule example **`stale_minutes=60`**. ## Core relational schema (distilled) @@ -71,7 +71,7 @@ Endpoints named: **`/parcel/buildingRules`**; **`/api/parcel/metrics/priceOfMete ## Production scale (ground-truth narrative) -**`parcels`:** **2,163,003** rows. **`parcel_price_metrics`:** **76,080,728** rows. **`transactions`:** **70,787** rows. **`building_rules`:** **130,112** rows. **`tile_urls`:** **34,726** rows; **processed 33,938 (97.7%)**; **`in_progress` 788 (2.3%)**. **`provinces`:** **12**; **`neighborhoods`:** **812**. Database name **`meshic`** (not **`meshic_pipeline`** as some documents suggest); owner **`postgres`**. +**`parcels`:** **2,163,003** rows. **`parcel_price_metrics`:** **76,080,728** rows. **`transactions`:** **70,787** rows. **`building_rules`:** **130,112** rows. **`tile_urls`:** **34,726** rows; **processed 33,938 (97.7%)**; **`in_progress` 788 (2.3%)**. **`provinces`:** **12**; **`neighborhoods`:** **812**. Database name **`suhail`** (not **`suhail_pipeline`** as some documents suggest); owner **`postgres`**. ## Superseded and memory-bank figures @@ -93,19 +93,19 @@ Alembic revision **`19c587b33197_add_critical_performance_indexes.py`** is the r **Command count:** ground-truth doc states **18 commands in 5 categories**; brownfield architecture says **15+**; architecture lists a **representative subset** without a single total **[POSSIBLE GAP: authoritative canonical list and category mapping]**. -**Entry point:** **`src/meshic_pipeline/cli.py`**. +**Entry point:** **`src/suhail_pipeline/cli.py`**. ## Configuration (architecture) -**`src/meshic_pipeline/config.py`:** **Pydantic settings**; **DB URL**, **API endpoints**, **layers**, **batch sizes**; **provinces** loaded from DB via **`load_provinces_from_db`**; failure **falls back to empty provinces with warning**. **`pipeline_config.yaml`** plus **environment variables**. Example tile URL template: **`https://tiles.suhail.ai/maps/{slug}/{z}/{x}/{y}.vector.pbf`** (Riyadh example). **Secrets** in **`.env`**; **no secrets in code**; **least-privilege DB role**; **UTF-8 Arabic**. +**`src/suhail_pipeline/config.py`:** **Pydantic settings**; **DB URL**, **API endpoints**, **layers**, **batch sizes**; **provinces** loaded from DB via **`load_provinces_from_db`**; failure **falls back to empty provinces with warning**. **`pipeline_config.yaml`** plus **environment variables**. Example tile URL template: **`https://tiles.suhail.ai/maps/{slug}/{z}/{x}/{y}.vector.pbf`** (Riyadh example). **Secrets** in **`.env`**; **no secrets in code**; **least-privilege DB role**; **UTF-8 Arabic**. ## Technology stack -**Python 3.9+**; package **`meshic-pipeline` v0.1.0** per doc snippet; **SQLAlchemy 2.0+ async**; **GeoAlchemy2**; **GeoPandas**, **Shapely**, **h3**; **Alembic**; **Typer**; **aiohttp**; **mapbox-vector-tile/protobuf** for MVT. +**Python 3.9+**; package **`suhail-pipeline` v0.1.0** per doc snippet; **SQLAlchemy 2.0+ async**; **GeoAlchemy2**; **GeoPandas**, **Shapely**, **h3**; **Alembic**; **Typer**; **aiohttp**; **mapbox-vector-tile/protobuf** for MVT. ## Repository layout -**`src/meshic_pipeline/`** — cli, config, decoder, discovery, downloader, enrichment, geometry, persistence, **`pipeline_orchestrator.py`**, utils; **`alembic/`**, **`tests/`**, **`scripts/`**, **`logs/`**, **`docs/`**, **`pyproject.toml`**. +**`src/suhail_pipeline/`** — cli, config, decoder, discovery, downloader, enrichment, geometry, persistence, **`pipeline_orchestrator.py`**, utils; **`alembic/`**, **`tests/`**, **`scripts/`**, **`logs/`**, **`docs/`**, **`pyproject.toml`**. ## Non-functional targets (architecture) @@ -123,7 +123,7 @@ Alembic revision **`19c587b33197_add_critical_performance_indexes.py`** is the r ## Deployment and operations -**Dev, staging, prod** share **migrations**. **Local:** PostgreSQL+PostGIS, **`uv sync --all-groups`**, **`uv run meshic-pipeline`**, **`uv run alembic upgrade head`**. **Temp table policy:** **pipeline-owned `temp_*` only**. +**Dev, staging, prod** share **migrations**. **Local:** PostgreSQL+PostGIS, **`uv sync --all-groups`**, **`uv run suhail-pipeline`**, **`uv run alembic upgrade head`**. **Temp table policy:** **pipeline-owned `temp_*` only**. **Migrations (`docs/ops/migrations.md` themes):** Prereqs **`DATABASE_URL`** in **`.env`**; Postgres + PostGIS. Upgrade via **`uv run python scripts/db/upgrade.py`** or **`uv run alembic upgrade head`**. Downgrade via **`uv run python scripts/db/downgrade.py -1`** or **`... base`**. Critical indexes migration includes **`temp_*` cleanup**; apply in **low-traffic** windows with **baseline/post timings**. **Repair incomplete province metadata** with **`uv run python scripts/util/backfill_province_metadata.py --province-id `** before **auto-geometric delta** pipelines. @@ -145,7 +145,7 @@ Upsert-enabled examples: **`parcels_centroids.parcel_no`**; **`metro_stations.st ## Province-wide scraping and DB-driven MVT (themes) -**Authoritative** sources: **`provinces`** metadata; **`tile_urls`** list and status; **config from DB**; **no hard-coded province dicts** or YAML tile lists. **Downloader:** async; province and tiles from DB; caching, retry, concurrency; **resumable** multi/all-province. **CLI examples:** **`meshic-pipeline geometric --province riyadh`**; **`--all-provinces`**; multi **`--province`** flags. **Performance:** index **`tile_urls`** by status/province; spatial/lookup indices on parcels/neighborhoods at **>1M/province**; **~5000-row chunks**; **disk tile cache**. **CI suggestion:** nightly geometric with **`--province riyadh --limit-test`**. **Rollout phases** span metadata+generator through full Riyadh and six provinces with success metrics in a doc table **[POSSIBLE GAP: exact phase gates and metric table not reproduced here]**. +**Authoritative** sources: **`provinces`** metadata; **`tile_urls`** list and status; **config from DB**; **no hard-coded province dicts** or YAML tile lists. **Downloader:** async; province and tiles from DB; caching, retry, concurrency; **resumable** multi/all-province. **CLI examples:** **`suhail-pipeline geometric --province riyadh`**; **`--all-provinces`**; multi **`--province`** flags. **Performance:** index **`tile_urls`** by status/province; spatial/lookup indices on parcels/neighborhoods at **>1M/province**; **~5000-row chunks**; **disk tile cache**. **CI suggestion:** nightly geometric with **`--province riyadh --limit-test`**. **Rollout phases** span metadata+generator through full Riyadh and six provinces with success metrics in a doc table **[POSSIBLE GAP: exact phase gates and metric table not reproduced here]**. ## CLI audit themes (documentation vs implementation) @@ -153,11 +153,11 @@ Inventory includes flags such as **`--bbox`**, **`--recreate-db`**, **`--save-as ## Clean slate protocol (destructive local reset only) -**Warning:** wipes **local** PostgreSQL/PostGIS data; **dev-only**. Steps include stopping Homebrew services, uninstalling Postgres/PostGIS variants, removing data directories (**Apple Silicon** vs **Intel** paths), reinstalling PostGIS/Postgres (example **`postgresql@16`**), **`createdb meshic`**, **`CREATE EXTENSION postgis`**, Python venv and Alembic workflow, verification queries. **[POSSIBLE GAP: full ordered checklist and version pinning policy for teams not on Homebrew/macOS.]** +**Warning:** wipes **local** PostgreSQL/PostGIS data; **dev-only**. Steps include stopping Homebrew services, uninstalling Postgres/PostGIS variants, removing data directories (**Apple Silicon** vs **Intel** paths), reinstalling PostGIS/Postgres (example **`postgresql@16`**), **`createdb suhail`**, **`CREATE EXTENSION postgis`**, Python venv and Alembic workflow, verification queries. **[POSSIBLE GAP: full ordered checklist and version pinning policy for teams not on Homebrew/macOS.]** ## EPIC-001 cross-reference (performance and monitoring) -EPIC-001 scope: **Alembic critical indexes**; **baseline→post measurements**; **CLI monitoring** for tile queue and enrichment coverage; **automate stale tile resets**; **out of scope:** full **Prometheus/Grafana**. Migration **`19c587b33197_add_critical_performance_indexes.py`**: indexes plus **safe cleanup** of stray **`temp_*`**. Monitor commands expose **queue counts**, **failed count**, **oldest `in_progress` age**, enrichment **totals and percentages**. **`monitor recommend`** chooses next action among **fast-enrich / incremental / delta** with **batch sizes**. **`schedule-info`** suggests **daily/weekly/monthly** cadence from freshness. Acceptance mentions **`meshic-pipeline monitor reset-stale -- --stale-minutes 60`**; report naming variants (**`perf-post-.md`** vs **`perf-post-index-*.md`**) appear across stories and AC **[POSSIBLE GAP: single standard report filename pattern]**. +EPIC-001 scope: **Alembic critical indexes**; **baseline→post measurements**; **CLI monitoring** for tile queue and enrichment coverage; **automate stale tile resets**; **out of scope:** full **Prometheus/Grafana**. Migration **`19c587b33197_add_critical_performance_indexes.py`**: indexes plus **safe cleanup** of stray **`temp_*`**. Monitor commands expose **queue counts**, **failed count**, **oldest `in_progress` age**, enrichment **totals and percentages**. **`monitor recommend`** chooses next action among **fast-enrich / incremental / delta** with **batch sizes**. **`schedule-info`** suggests **daily/weekly/monthly** cadence from freshness. Acceptance mentions **`suhail-pipeline monitor reset-stale -- --stale-minutes 60`**; report naming variants (**`perf-post-.md`** vs **`perf-post-index-*.md`**) appear across stories and AC **[POSSIBLE GAP: single standard report filename pattern]**. ## References cited in architecture distillate diff --git a/docs/docs-distillate/_validation/reconstruction-PRD-bundle.md b/docs/docs-distillate/_validation/reconstruction-PRD-bundle.md index 43d1a50..0a0aa47 100644 --- a/docs/docs-distillate/_validation/reconstruction-PRD-bundle.md +++ b/docs/docs-distillate/_validation/reconstruction-PRD-bundle.md @@ -106,7 +106,7 @@ Production must contain the listed indexes (verified via **`pg_indexes`**): on * ### Epic 2 — Monitoring -**`meshic-pipeline monitor status`:** queue counts by status, top errors, age of oldest **`in_progress`**. **`recommend`:** scheduling guidance (next enrichment strategy, batch sizes). **`schedule-info`:** recommended cadence from freshness. A **scheduled job** resets stale **`in_progress`** after a **configurable threshold** (example **60 minutes**), logging the count affected. **Tiles per hour** and **enrichment counts per run** appear in logs, with **optional CSV or dashboard export**. +**`suhail-pipeline monitor status`:** queue counts by status, top errors, age of oldest **`in_progress`**. **`recommend`:** scheduling guidance (next enrichment strategy, batch sizes). **`schedule-info`:** recommended cadence from freshness. A **scheduled job** resets stale **`in_progress`** after a **configurable threshold** (example **60 minutes**), logging the count affected. **Tiles per hour** and **enrichment counts per run** appear in logs, with **optional CSV or dashboard export**. ### Epic 3 — Delta productization @@ -146,7 +146,7 @@ Out of delta PRD scope: **GUI**, **advanced query optimization**, **upstream MVT ## Process and references (PRD metadata) -PRD author **Mary** (Business Analyst); date **2025-10-16**; project **level 2** (focused PRD plus solutioning handoff). Technical references named in the distillate include **`docs/BROWNFIELD_PROJECT_DOCUMENTATION.md`**, Alembic revision **`19c587b33197_add_critical_performance_indexes.py`**, **`src/meshic_pipeline/cli.py`**, **`models.py`**, and **`run_db_geometric.py`**. +PRD author **Mary** (Business Analyst); date **2025-10-16**; project **level 2** (focused PRD plus solutioning handoff). Technical references named in the distillate include **`docs/BROWNFIELD_PROJECT_DOCUMENTATION.md`**, Alembic revision **`19c587b33197_add_critical_performance_indexes.py`**, **`src/suhail_pipeline/cli.py`**, **`models.py`**, and **`run_db_geometric.py`**. ## Index and distillate housekeeping diff --git a/docs/index.md b/docs/index.md index bdb68ea..9cc5356 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,7 +26,7 @@ Generated and curated reference for humans and LLMs. Descriptions reflect file p - **[Delta_enrichment_PRD.md](./Delta_enrichment_PRD.md)** — Product requirements for delta enrichment and `delta-enrich` CLI. - **[handoff_upsert_bottleneck.md](./handoff_upsert_bottleneck.md)** — Reference-data bottleneck analysis (municipalities/neighborhoods). - **[pipeline_table_uniqueness_review.md](./pipeline_table_uniqueness_review.md)** — Upsert keys and table uniqueness review. -- **[PRD.md](./PRD.md)** — Main product requirements: Meshic geospatial pipeline. +- **[PRD.md](./PRD.md)** — Main product requirements: Suhail geospatial pipeline. - **[PROJECT_BRIEF.md](./PROJECT_BRIEF.md)** — Executive brief: goals, MVP, personas. - **[province_wide_scraping_plan.md](./province_wide_scraping_plan.md)** — DB-driven province and national scrape rollout plan. - **[technical-decisions-template.md](./technical-decisions-template.md)** — Template for recording ADRs or technical decisions. diff --git a/docs/ops/MACHINE_POSTGRES_AUDIT.md b/docs/ops/MACHINE_POSTGRES_AUDIT.md index 04c0ff6..73fecd9 100644 --- a/docs/ops/MACHINE_POSTGRES_AUDIT.md +++ b/docs/ops/MACHINE_POSTGRES_AUDIT.md @@ -27,7 +27,7 @@ | 17 | `/opt/homebrew/var/postgresql@17` | **~6.5 GB** | **5432** | | 18 | `/opt/homebrew/var/postgresql@18` | ~73 MB | default **5432** (commented `#port = 5432`) | -**Interpretation:** The large **@17** cluster likely holds historical work (e.g. former `meshic` DB). **@18** is the current target for this project (`suhail_pipeline` per repo `.env.example`). +**Interpretation:** The large **@17** cluster likely holds historical work (e.g. former `suhail` DB). **@18** is the current target for this project (`suhail_pipeline` per repo `.env.example`). ## `brew services` snapshot diff --git a/docs/province_wide_scraping_plan.md b/docs/province_wide_scraping_plan.md index 9ddd0a3..2f8c3a2 100644 --- a/docs/province_wide_scraping_plan.md +++ b/docs/province_wide_scraping_plan.md @@ -1,7 +1,7 @@ -# Province-Wide Scraping Plan for Meshic Geospatial Pipeline +# Province-Wide Scraping Plan for Suhail Geospatial Pipeline ## 1 Overview -This document specifies the DB-driven approach for scraping **all six Saudi provinces** using Mapbox Vector Tiles (MVT) and persisting them into the Meshic PostGIS schema. +This document specifies the DB-driven approach for scraping **all six Saudi provinces** using Mapbox Vector Tiles (MVT) and persisting them into the Suhail PostGIS schema. The plan addresses: 1. Centralised province metadata (now stored in the database) @@ -50,9 +50,9 @@ Shortcomings for province-wide rollout (now resolved): * `--province` (multi-use) and `--all-provinces` flags. * Examples ```bash - meshic-pipeline geometric --province riyadh - meshic-pipeline geometric --all-provinces - meshic-pipeline geometric --province riyadh --province makkah + suhail-pipeline geometric --province riyadh + suhail-pipeline geometric --all-provinces + suhail-pipeline geometric --province riyadh --province makkah ``` * Help text updated accordingly. @@ -71,7 +71,7 @@ Shortcomings for province-wide rollout (now resolved): ### 3.8 Documentation & CI * This doc (updated) reflects the DB-driven approach. * README and CLI `--help` examples updated. -* CI job runs `meshic-pipeline geometric --province riyadh --limit-test` nightly. +* CI job runs `suhail-pipeline geometric --province riyadh --limit-test` nightly. --- diff --git a/docs/stories/STORY-003-monitoring-cli-outputs.md b/docs/stories/STORY-003-monitoring-cli-outputs.md index 146a9d7..b79c931 100644 --- a/docs/stories/STORY-003-monitoring-cli-outputs.md +++ b/docs/stories/STORY-003-monitoring-cli-outputs.md @@ -6,7 +6,7 @@ Owner: Platform Engineering Status: Ready for Dev ## Goal -Provide actionable status and recommendations via `meshic-pipeline monitor`. +Provide actionable status and recommendations via `suhail-pipeline monitor`. ## Tasks - Ensure `monitor status` outputs: tile queue counts by status, oldest in_progress age, top errors. diff --git a/docs/tech-specs/EPIC-001-performance-and-monitoring.md b/docs/tech-specs/EPIC-001-performance-and-monitoring.md index c16cc52..d204287 100644 --- a/docs/tech-specs/EPIC-001-performance-and-monitoring.md +++ b/docs/tech-specs/EPIC-001-performance-and-monitoring.md @@ -8,7 +8,7 @@ Status: Draft for Architecture Review ## 1. Objective & Scope -Improve query performance and operational visibility for the Meshic pipeline by: +Improve query performance and operational visibility for the Suhail pipeline by: - Applying critical indexes (safe, additive) via Alembic. - Establishing baseline → post-change performance measurements. - Providing monitoring outputs for tile queue health and enrichment coverage. @@ -48,12 +48,12 @@ Out of scope: Full observability stack (Prometheus/Grafana). This phase focuses ### 3.3 Monitoring Outputs (CLI) -- `meshic-pipeline monitor status` should include: +- `suhail-pipeline monitor status` should include: - Tile queue summary: counts by `status`, number of failed, oldest `in_progress` age - Enrichment coverage: total parcels, parcels with `enriched_at`, % with `transaction_price > 0` -- `meshic-pipeline monitor recommend`: +- `suhail-pipeline monitor recommend`: - Next best action (fast-enrich / incremental / delta) with recommended batch sizes -- `meshic-pipeline monitor schedule-info`: +- `suhail-pipeline monitor schedule-info`: - Suggested cadence (daily/weekly/monthly) based on current freshness ### 3.4 Stale Reset Automation @@ -78,7 +78,7 @@ Out of scope: Full observability stack (Prometheus/Grafana). This phase focuses - All specified indexes exist in production; Alembic migration completes without downtime. - Sample query timings captured (baseline vs post-index) at `docs/reports/perf-baseline-*.md` and `docs/reports/perf-post-index-*.md`. -- Monitoring outputs present and informative; stale reset job operational (`meshic-pipeline monitor reset-stale -- --stale-minutes 60`). +- Monitoring outputs present and informative; stale reset job operational (`suhail-pipeline monitor reset-stale -- --stale-minutes 60`). - Investigate any regressions (e.g., broad metrics aggregation still performs full table scan—documented for follow-up tuning). --- @@ -105,5 +105,5 @@ Out of scope: Full observability stack (Prometheus/Grafana). This phase focuses - docs/BROWNFIELD_PROJECT_DOCUMENTATION.md - alembic/versions/19c587b33197_add_critical_performance_indexes.py -- src/meshic_pipeline/persistence/models.py (TileURL methods) +- src/suhail_pipeline/persistence/models.py (TileURL methods) - docs/PRD.md, docs/ACCEPTANCE_CRITERIA.md diff --git a/docs/working/TODO.md b/docs/working/TODO.md index 099e5af..7013a64 100644 --- a/docs/working/TODO.md +++ b/docs/working/TODO.md @@ -1,4 +1,4 @@ -# Meshic Geospatial Pipeline Implementation Plan +# Suhail Geospatial Pipeline Implementation Plan ## 🚦 **Next Actions Summary (as of DB-Driven Pipeline Phase)** @@ -15,9 +15,9 @@ --- -# Meshic Geospatial Pipeline Implementation Plan +# Suhail Geospatial Pipeline Implementation Plan -This document outlines the implementation plan for the Meshic Geospatial Data Pipeline following the transition to a fully DB-driven pipeline. All tile discovery and orchestration is now managed via the `tile_urls` table in the database, supporting province-wide and all-Saudi scrapes with resumable processing. +This document outlines the implementation plan for the Suhail Geospatial Data Pipeline following the transition to a fully DB-driven pipeline. All tile discovery and orchestration is now managed via the `tile_urls` table in the database, supporting province-wide and all-Saudi scrapes with resumable processing. ## 🎯 **Current Project Status** @@ -52,7 +52,7 @@ This document outlines the implementation plan for the Meshic Geospatial Data Pi **Tasks**: - [x] **Source Environment**: Activate `.venv` before running commands -- [x] **Run Geometric Pipeline**: `meshic-pipeline geometric` (or `python -m meshic_pipeline.cli geometric`) +- [x] **Run Geometric Pipeline**: `suhail-pipeline geometric` (or `python -m suhail_pipeline.cli geometric`) - [x] **Verify Database Population**: Check parcels, neighborhoods, subdivisions tables - [x] **Validate Schema**: Confirm data types and foreign key relationships - [x] **Performance Check**: Measure processing time and memory usage @@ -116,7 +116,7 @@ git tag v0.1.1-3x3-validated **Prerequisites**: 3x3 baseline test completed successfully **Tasks**: -- [x] **Test Fast Enrichment**: `meshic-pipeline fast-enrich --limit 100` (and full batch) +- [x] **Test Fast Enrichment**: `suhail-pipeline fast-enrich --limit 100` (and full batch) - [x] **Monitor Success Rate**: Track enrichment coverage percentage - [x] **Verify API Integration**: Check all 3 API endpoints working - [x] **Data Quality Check**: Ensure no garbage data in enrichment tables @@ -208,22 +208,22 @@ uv add -e . source .venv/bin/activate # or appropriate activation for your shell # Verify installation -meshic-pipeline --help +suhail-pipeline --help ``` ### **Testing Commands** ```bash # Basic geometric pipeline (3x3 grid) -meshic-pipeline geometric +suhail-pipeline geometric # Alternative if package not in PATH -python -m meshic_pipeline.cli geometric +python -m suhail_pipeline.cli geometric # Enrichment testing -meshic-pipeline fast-enrich --limit 100 +suhail-pipeline fast-enrich --limit 100 # Monitor pipeline status -meshic-pipeline monitor status +suhail-pipeline monitor status # Database validation python scripts/check_db.py @@ -391,18 +391,18 @@ Provide minimal pytest examples for each module to guide future implementations. ### **1. Environment Setup** - [ ] Install dependencies using `uv add -e .` - [x] Activate the virtual environment: `source .venv/bin/activate` -- [x] Verify package installation: `meshic-pipeline --help` or `python -m meshic_pipeline.cli --help` - - meshic-pipeline CLI is available and functional +- [x] Verify package installation: `suhail-pipeline --help` or `python -m suhail_pipeline.cli --help` + - suhail-pipeline CLI is available and functional - [x] Check database connectivity and configuration - Database is accessible and tables are present (see `list-tables` output) ### **2. Run Baseline 3x3 Riyadh Test** -- [x] Execute geometric pipeline: `meshic-pipeline geometric` +- [x] Execute geometric pipeline: `suhail-pipeline geometric` - **Note:** Pipeline ran successfully for most layers, but failed for 'parcels' due to `invalid input syntax for type bigint: "9.0"` (zoning_id). All other layers processed and persisted as expected. This error must be fixed before scaling or enrichment. - [x] Verify database population: Check that parcels and reference tables are populated - [x] Validate schema: Confirm data types and foreign key relationships - [ ] Monitor performance: Record processing time and memory usage -- [x] Run enrichment pipeline: `meshic-pipeline fast-enrich --limit 100` (validated and operational) +- [x] Run enrichment pipeline: `suhail-pipeline fast-enrich --limit 100` (validated and operational) ## Comprehensive Debugging & Remediation Plan for Pipeline/DB Issues (July 2025) @@ -480,7 +480,7 @@ Provide minimal pytest examples for each module to guide future implementations. ### Handoff Notes - This checklist is designed for another developer to pick up and systematically resolve the pipeline/DB issues. - Please document all findings and fixes directly in this file or in a new issue tracker as appropriate. -- If you need more context, review the pipeline logs, stitched GeoJSONs, and the code in `src/meshic_pipeline/`. +- If you need more context, review the pipeline logs, stitched GeoJSONs, and the code in `src/suhail_pipeline/`. ## ✅ Baseline Validation Results (July 2025) - All tests run successfully (`unit_test_results.txt`) @@ -503,7 +503,7 @@ Provide minimal pytest examples for each module to guide future implementations. ## Notes - Enrichment pipeline must be run as: ```bash - PYTHONPATH=$(pwd) python src/meshic_pipeline/run_enrichment_pipeline.py fast-enrich --limit 100 + PYTHONPATH=$(pwd) python src/suhail_pipeline/run_enrichment_pipeline.py fast-enrich --limit 100 ``` - Pydantic deprecation warning is present but does not affect current functionality. diff --git a/logs/check_stale.py b/logs/check_stale.py index a3952a3..90cb83c 100644 --- a/logs/check_stale.py +++ b/logs/check_stale.py @@ -1,5 +1,5 @@ from sqlalchemy import create_engine, text -from meshic_pipeline.config import settings +from suhail_pipeline.config import settings eng = create_engine(str(settings.database_url)) with eng.connect() as c: diff --git a/logs/enrichment_monitor.py b/logs/enrichment_monitor.py index dd06da6..0524c18 100644 --- a/logs/enrichment_monitor.py +++ b/logs/enrichment_monitor.py @@ -7,7 +7,7 @@ # Add project root to path sys.path.insert(0, str(Path(__file__).parent.parent)) -from src.meshic_pipeline.config import settings +from src.suhail_pipeline.config import settings def monitor_enrichment(): engine = create_engine(str(settings.database_url)) diff --git a/logs/enrichment_watcher.py b/logs/enrichment_watcher.py index 982f3d2..d6e1af3 100644 --- a/logs/enrichment_watcher.py +++ b/logs/enrichment_watcher.py @@ -9,7 +9,7 @@ # Add project root to path sys.path.insert(0, str(Path(__file__).parent.parent)) -from src.meshic_pipeline.config import settings +from src.suhail_pipeline.config import settings from sqlalchemy import create_engine def check_process_running(pid_file): @@ -89,7 +89,7 @@ def main(): if not process_running and remaining > 0: print(f"\n⚠️ WARNING: Process stopped but {remaining:,} parcels remain!") print(f"📝 Check logs/enrichment-full.log for errors") - print(f"🔄 You may need to restart: meshic-pipeline full-refresh") + print(f"🔄 You may need to restart: suhail-pipeline full-refresh") break except KeyboardInterrupt: diff --git a/logs/live_monitor.py b/logs/live_monitor.py index 96ebc28..76a9057 100644 --- a/logs/live_monitor.py +++ b/logs/live_monitor.py @@ -9,7 +9,7 @@ from datetime import datetime, timedelta from tqdm import tqdm from sqlalchemy import create_engine, text -from meshic_pipeline.config import settings +from suhail_pipeline.config import settings class PipelineMonitor: def __init__(self): diff --git a/logs/progress_watcher.py b/logs/progress_watcher.py index 0735330..2deb1e6 100644 --- a/logs/progress_watcher.py +++ b/logs/progress_watcher.py @@ -1,6 +1,6 @@ import sys, time from sqlalchemy import create_engine, text -from meshic_pipeline.config import settings +from suhail_pipeline.config import settings def snapshot(): e = create_engine(str(settings.database_url)) diff --git a/logs/reset_stale.py b/logs/reset_stale.py index 9d203af..140a934 100644 --- a/logs/reset_stale.py +++ b/logs/reset_stale.py @@ -1,5 +1,5 @@ from sqlalchemy import create_engine, text -from meshic_pipeline.config import settings +from suhail_pipeline.config import settings eng = create_engine(str(settings.database_url)) with eng.connect() as c: diff --git a/logs/universal_metrics_monitor.py b/logs/universal_metrics_monitor.py index 4107fff..0374fef 100644 --- a/logs/universal_metrics_monitor.py +++ b/logs/universal_metrics_monitor.py @@ -9,7 +9,7 @@ # Add project root to path sys.path.insert(0, str(Path(__file__).parent.parent)) -from src.meshic_pipeline.config import settings +from src.suhail_pipeline.config import settings from sqlalchemy import create_engine, text from tqdm import tqdm diff --git a/pipeline_config_enhanced.yaml b/pipeline_config_enhanced.yaml index be0dd13..f1ac102 100644 --- a/pipeline_config_enhanced.yaml +++ b/pipeline_config_enhanced.yaml @@ -76,7 +76,7 @@ layers: # Integration examples (documentation) usage_examples: - single_province: "python -m meshic_pipeline.cli province-geometric riyadh" - all_saudi: "python -m meshic_pipeline.cli saudi-arabia-geometric" - efficient_mode: "python -m meshic_pipeline.cli saudi-arabia-geometric --strategy efficient" - complete_pipeline: "python -m meshic_pipeline.cli province-pipeline al_qassim" \ No newline at end of file + single_province: "python -m suhail_pipeline.cli province-geometric riyadh" + all_saudi: "python -m suhail_pipeline.cli saudi-arabia-geometric" + efficient_mode: "python -m suhail_pipeline.cli saudi-arabia-geometric --strategy efficient" + complete_pipeline: "python -m suhail_pipeline.cli province-pipeline al_qassim" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5a1a6cb..7aa3b04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,12 +3,12 @@ requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] -name = "meshic-pipeline" +name = "suhail-pipeline" version = "0.1.0" authors = [ { name="Suhail.AI", email="engineering@suhail.ai" }, ] -description = "Meshic data processing pipeline" +description = "Suhail data processing pipeline" readme = "README.md" requires-python = ">=3.9" classifiers = [ @@ -43,8 +43,8 @@ dependencies = [ ] [project.scripts] -check_db = "meshic_pipeline.utils.db_checker:app" -meshic-pipeline = "meshic_pipeline.cli:app" +check_db = "suhail_pipeline.utils.db_checker:app" +suhail-pipeline = "suhail_pipeline.cli:app" [tool.setuptools.packages.find] where = ["src"] diff --git a/run_pipeline.sh b/run_pipeline.sh index d9727f1..2504765 100755 --- a/run_pipeline.sh +++ b/run_pipeline.sh @@ -19,11 +19,11 @@ if [[ -z "${DATABASE_URL:-}" ]]; then fi echo "Seeding pilot tiles for two provinces (riyadh, eastern)..." -uv run meshic-pipeline seed-tiles --provinces riyadh --provinces eastern --limit 200 --stride 20 +uv run suhail-pipeline seed-tiles --provinces riyadh --provinces eastern --limit 200 --stride 20 echo "Running DB-driven geometric pilot..." -uv run meshic-pipeline db-geometric --batch-size 200 --concurrency 20 --request-delay 0.05 +uv run suhail-pipeline db-geometric --batch-size 200 --concurrency 20 --request-delay 0.05 echo "Pilot complete. To proceed countrywide:" -echo " 1) uv run meshic-pipeline seed-tiles" -echo " 2) uv run meshic-pipeline db-geometric --batch-size 1000 --concurrency 20 --request-delay 0.05" +echo " 1) uv run suhail-pipeline seed-tiles" +echo " 2) uv run suhail-pipeline db-geometric --batch-size 1000 --concurrency 20 --request-delay 0.05" diff --git a/scripts/generate_riyadh_transactions_report.py b/scripts/generate_riyadh_transactions_report.py index 88a11c5..c925624 100755 --- a/scripts/generate_riyadh_transactions_report.py +++ b/scripts/generate_riyadh_transactions_report.py @@ -20,8 +20,8 @@ src_path = project_root / "src" sys.path.insert(0, str(src_path)) -from meshic_pipeline.config import Settings -from meshic_pipeline.persistence.models import Parcel, Transaction, Neighborhood, Province +from suhail_pipeline.config import Settings +from suhail_pipeline.persistence.models import Parcel, Transaction, Neighborhood, Province def get_database_connection(): @@ -155,7 +155,7 @@ def generate_markdown_report(df, stats, output_path): **Generated on:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} **Report Period:** Last 6 months -**Data Source:** Meshic Pipeline Database +**Data Source:** Suhail Pipeline Database ## Executive Summary @@ -258,7 +258,7 @@ def generate_markdown_report(df, stats, output_path): ## Methodology -This report is generated from the Meshic Pipeline database, which captures real estate data from official sources and enriches it with additional market information. The analysis includes: +This report is generated from the Suhail Pipeline database, which captures real estate data from official sources and enriches it with additional market information. The analysis includes: 1. **Data Source:** Direct database queries from the `parcels` and `transactions` tables 2. **Time Period:** Last 6 months from the report generation date @@ -273,7 +273,7 @@ def generate_markdown_report(df, stats, output_path): - **Data Export:** CSV format available in companion file --- -*Report generated by Meshic Pipeline Analytics System* +*Report generated by Suhail Pipeline Analytics System* """ # Write the report diff --git a/scripts/util/backfill_province_metadata.py b/scripts/util/backfill_province_metadata.py index 17655f8..b54b13a 100644 --- a/scripts/util/backfill_province_metadata.py +++ b/scripts/util/backfill_province_metadata.py @@ -18,8 +18,8 @@ import mercantile from sqlalchemy import text -from meshic_pipeline.persistence.db import get_db_engine -from meshic_pipeline.config import settings +from suhail_pipeline.persistence.db import get_db_engine +from suhail_pipeline.config import settings SLUG_REGEX = re.compile(r"https?://[^/]+/maps/([^/]+)/") diff --git a/scripts/util/capture_test_fixtures.py b/scripts/util/capture_test_fixtures.py index 94a9412..11541c3 100644 --- a/scripts/util/capture_test_fixtures.py +++ b/scripts/util/capture_test_fixtures.py @@ -10,10 +10,10 @@ src_path = Path(__file__).parent.parent / "src" sys.path.insert(0, str(src_path)) -from meshic_pipeline.config import settings -from meshic_pipeline.downloader.async_tile_downloader import AsyncTileDownloader -from meshic_pipeline.enrichment.api_client import SuhailAPIClient -from meshic_pipeline.logging_utils import get_logger +from suhail_pipeline.config import settings +from suhail_pipeline.downloader.async_tile_downloader import AsyncTileDownloader +from suhail_pipeline.enrichment.api_client import SuhailAPIClient +from suhail_pipeline.logging_utils import get_logger logger = get_logger(__name__) diff --git a/scripts/util/check_db.py b/scripts/util/check_db.py index 2d8fd9c..59c316d 100644 --- a/scripts/util/check_db.py +++ b/scripts/util/check_db.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Optional from sqlalchemy import create_engine, inspect, text, exc -from meshic_pipeline.config import settings +from suhail_pipeline.config import settings from rich.table import Table as RichTable from rich.console import Console from sqlalchemy.exc import NoSuchTableError diff --git a/scripts/util/enhanced_province_discovery.py b/scripts/util/enhanced_province_discovery.py index 37550d5..fbeb750 100644 --- a/scripts/util/enhanced_province_discovery.py +++ b/scripts/util/enhanced_province_discovery.py @@ -16,7 +16,7 @@ import typer import time -from meshic_pipeline.decoder.mvt_decoder import MVTDecoder +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder # --- Enhanced Configuration --- LOG_LEVEL = logging.INFO diff --git a/scripts/util/export_to_geojson.py b/scripts/util/export_to_geojson.py index 13611ec..1974564 100644 --- a/scripts/util/export_to_geojson.py +++ b/scripts/util/export_to_geojson.py @@ -30,7 +30,7 @@ def export_table_to_geojson(table_name: str, output_file: str, limit: int = None load_dotenv() if str(_REPO_SRC) not in sys.path: sys.path.insert(0, str(_REPO_SRC)) - from meshic_pipeline.persistence.db import get_db_engine + from suhail_pipeline.persistence.db import get_db_engine engine = get_db_engine() diff --git a/scripts/util/generate_tile_list.py b/scripts/util/generate_tile_list.py index 5fc75b8..3f4c036 100644 --- a/scripts/util/generate_tile_list.py +++ b/scripts/util/generate_tile_list.py @@ -6,7 +6,7 @@ from shapely.ops import unary_union from tqdm import tqdm -from meshic_pipeline.decoder.mvt_decoder import MVTDecoder +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder # --- Configuration --- LOG_LEVEL = logging.INFO diff --git a/scripts/util/inspect_tiles.py b/scripts/util/inspect_tiles.py index 05b7887..481806d 100644 --- a/scripts/util/inspect_tiles.py +++ b/scripts/util/inspect_tiles.py @@ -1,7 +1,7 @@ import os import glob import re -from meshic_pipeline.decoder.mvt_decoder import MVTDecoder +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder from collections import defaultdict diff --git a/scripts/util/intelligent_province_discovery.py b/scripts/util/intelligent_province_discovery.py index 83b9ce7..e3c885c 100644 --- a/scripts/util/intelligent_province_discovery.py +++ b/scripts/util/intelligent_province_discovery.py @@ -14,7 +14,7 @@ from shapely.geometry import box import typer -from meshic_pipeline.decoder.mvt_decoder import MVTDecoder +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder # --- Enhanced Configuration --- LOG_LEVEL = logging.INFO diff --git a/scripts/util/market_analysis.py b/scripts/util/market_analysis.py index 3a432da..5547e37 100644 --- a/scripts/util/market_analysis.py +++ b/scripts/util/market_analysis.py @@ -16,7 +16,7 @@ # Add the src directory to the path sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) -from meshic_pipeline.persistence.db import get_db_engine +from suhail_pipeline.persistence.db import get_db_engine from sqlalchemy import text diff --git a/scripts/util/sync_provinces.py b/scripts/util/sync_provinces.py index 087446b..ba38c1a 100644 --- a/scripts/util/sync_provinces.py +++ b/scripts/util/sync_provinces.py @@ -8,7 +8,7 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') # Database connection string (read from environment or default) -DATABASE_URL = os.getenv('DATABASE_URL', 'postgresql://localhost/meshic') +DATABASE_URL = os.getenv('DATABASE_URL', 'postgresql://localhost/suhail') def fetch_provinces(): url = 'https://api2.suhail.ai/regions' diff --git a/scripts/util/validate_data_types.py b/scripts/util/validate_data_types.py index c58a1ae..f492ccd 100644 --- a/scripts/util/validate_data_types.py +++ b/scripts/util/validate_data_types.py @@ -14,7 +14,7 @@ # Add src to path for imports sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -from meshic_pipeline.logging_utils import get_logger +from suhail_pipeline.logging_utils import get_logger logger = get_logger(__name__) diff --git a/scripts/util/validate_foreign_keys.py b/scripts/util/validate_foreign_keys.py index 65382f3..d612e00 100644 --- a/scripts/util/validate_foreign_keys.py +++ b/scripts/util/validate_foreign_keys.py @@ -3,7 +3,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import sqlalchemy -from src.meshic_pipeline.config import settings +from src.suhail_pipeline.config import settings def main(): """ diff --git a/scripts/util/validate_tile_data.py b/scripts/util/validate_tile_data.py index 014f707..5a38d4d 100644 --- a/scripts/util/validate_tile_data.py +++ b/scripts/util/validate_tile_data.py @@ -12,7 +12,7 @@ import glob import argparse import json -from meshic_pipeline.decoder.mvt_decoder import MVTDecoder +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder from shapely.geometry import mapping ID_FIELDS = { diff --git a/src/meshic_pipeline/__main__.py b/src/meshic_pipeline/__main__.py deleted file mode 100644 index c523732..0000000 --- a/src/meshic_pipeline/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Allow: python -m meshic_pipeline db-geometric ... (uses active env's installed package).""" - -from meshic_pipeline.cli import app - -if __name__ == "__main__": - app() diff --git a/src/meshic_pipeline/__init__.py b/src/suhail_pipeline/__init__.py similarity index 100% rename from src/meshic_pipeline/__init__.py rename to src/suhail_pipeline/__init__.py diff --git a/src/suhail_pipeline/__main__.py b/src/suhail_pipeline/__main__.py new file mode 100644 index 0000000..c52b4ad --- /dev/null +++ b/src/suhail_pipeline/__main__.py @@ -0,0 +1,6 @@ +"""Allow: python -m suhail_pipeline db-geometric ... (uses active env's installed package).""" + +from suhail_pipeline.cli import app + +if __name__ == "__main__": + app() diff --git a/src/meshic_pipeline/cli.py b/src/suhail_pipeline/cli.py similarity index 98% rename from src/meshic_pipeline/cli.py rename to src/suhail_pipeline/cli.py index 33573bf..e796116 100644 --- a/src/meshic_pipeline/cli.py +++ b/src/suhail_pipeline/cli.py @@ -3,7 +3,7 @@ import sys from typing import List, Optional, Tuple from pathlib import Path -from meshic_pipeline import run_enrichment_pipeline +from suhail_pipeline import run_enrichment_pipeline # Determine the directory containing this CLI file SCRIPT_DIR = Path(__file__).parent @@ -182,12 +182,12 @@ def seed_tiles( Use --limit and/or --stride to seed a small pilot sample. """ - from meshic_pipeline.config import settings - from meshic_pipeline.utils.tile_list_generator import tiles_from_bbox_z - from meshic_pipeline.persistence.db import get_db_engine + from suhail_pipeline.config import settings + from suhail_pipeline.utils.tile_list_generator import tiles_from_bbox_z + from suhail_pipeline.persistence.db import get_db_engine from sqlalchemy.orm import Session from sqlalchemy.dialects.postgresql import insert as pg_insert - from meshic_pipeline.persistence.models import TileURL + from suhail_pipeline.persistence.models import TileURL engine = get_db_engine(str(settings.database_url)) session = Session(engine) diff --git a/src/meshic_pipeline/config.py b/src/suhail_pipeline/config.py similarity index 81% rename from src/meshic_pipeline/config.py rename to src/suhail_pipeline/config.py index d2c31e6..03444f8 100644 --- a/src/meshic_pipeline/config.py +++ b/src/suhail_pipeline/config.py @@ -8,7 +8,7 @@ from pydantic import Field, PostgresDsn, PrivateAttr, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict -from meshic_pipeline.persistence.db import load_provinces_from_db +from suhail_pipeline.persistence.db import load_provinces_from_db # Define project root independently # Resolves to the parent directory of 'src', which is the project's root. @@ -30,9 +30,32 @@ "building_rule_aname": "building_rule_ar", "description_aname": "description_ar", "name_aname": "name_ar", + # Municipality name on the parcels layer (source field: municipality_aname). + # Without this entry municipality_ar was always NULL. + "municipalityaname": "municipality_ar", + "municipality_aname": "municipality_ar", + # Subdivision Arabic name now emitted on the subdivisions layer. + "subdivision_name_ar": "subdivision_name_ar", # Add any other discovered or future Arabic columns here } +# Time-series market fields now emitted inline on parcels/neighborhoods/subdivisions +# tile layers. Mapped to 'first' so they survive the PostGIS dissolve/stitch step. +_MARKET_TIMESERIES_FIRST = { + field: "first" + for field in ( + "transaction_price_1w", "transaction_price_1m", + "transaction_price_6m", "transaction_price_12m", + "price_of_meter_1w", "price_of_meter_1m", + "price_of_meter_6m", "price_of_meter_12m", + "transactions_count_1w", "transactions_count_1m", + "transactions_count_6m", "transactions_count_12m", + "transaction_date_1w", "transaction_date_1m", + "transaction_date_6m", "transaction_date_12m", + ) +} + + class Environment(str, Enum): """Environment types for configuration management.""" DEVELOPMENT = "development" @@ -184,11 +207,16 @@ class Settings(BaseSettings): "subdivisions": "subdivision_id", "neighborhoods": "neighborhood_id", "neighborhoods-centroids": "id", - "dimensions": "parcel_objectid", + # NB: `dimensions` is intentionally NOT keyed here — it has many rows per + # parcel (one per edge), so keying on parcel_objectid would collapse it to + # one edge. It uses the tile-scoped write (postgis_persister.TILE_SCOPED_LAYERS). "metro_stations": "station_code", "riyadh_bus_stations": "station_code", "qi_population_metrics": "grid_id", "qi_stripes": "strip_id", + "non_saudi_ownership_zones": "id", + # Synthetic deterministic key (postgis_persister.SYNTHETIC_PK_CONFIG). + "building_detection": "bd_id", }, description="Column to use as the unique ID for dissolving geometries, per layer. Only tables with unique constraints should be listed here.", ) @@ -198,37 +226,56 @@ class Settings(BaseSettings): "parcels": { "shape_area": "first", "transaction_price": "first", - "price_of_meter": "first", + "price_of_meter": "first", "landuseagroup": "first", + "landuseadetailed": "first", "zoning_id": "first", + "zoning_group": "first", "subdivision_id": "first", + "neighborhood_id": "first", "neighborhood_ar": "first", + "municipality_ar": "first", + "province_id": "first", + "block_no": "first", + "parcel_id": "first", "parcel_no": "first", "subdivision_no": "first", - "zoning_color": "first" + "zoning_color": "first", + "ruleid": "first", + **_MARKET_TIMESERIES_FIRST, }, "parcels-centroids": { "transaction_date": "first", - "transaction_price": "first", - "price_of_meter": "first" + "transaction_price": "first", + "price_of_meter": "first", + "transactions_count": "first", + **_MARKET_TIMESERIES_FIRST, }, "neighborhoods": { "shape_area": "first", "region_id": "first", - "province_id": "first", + "province_id": "first", "zoning_id": "first", "zoning_color": "first", + "zoning_group": "first", "neighborhood_ar": "first", + "neighborhood_name": "first", "transaction_price": "first", - "price_of_meter": "first" + "price_of_meter": "first", + **_MARKET_TIMESERIES_FIRST, }, "subdivisions": { "shape_area": "first", "subdivision_no": "first", + "subdivision_name_ar": "first", + "neighborhood_id": "first", + "region_id": "first", + "province_id": "first", "zoning_id": "first", "zoning_color": "first", "transaction_price": "first", - "price_of_meter": "first" + "price_of_meter": "first", + **_MARKET_TIMESERIES_FIRST, }, }, description="Attribute aggregation rules for GeoPandas dissolve, per layer.", @@ -247,6 +294,11 @@ class Settings(BaseSettings): "riyadh_bus_stations": "riyadh_bus_stations", "qi_population_metrics": "qi_population_metrics", "qi_stripes": "qi_stripes", + "non_saudi_ownership_zones": "non_saudi_ownership_zones", + # Documented-but-not-yet-enabled new layers map to their own tables: + "dimensions": "dimensions", + "streets": "streets", + "building_detection": "building_detection", }, description="Mapping from layer name to the desired PostGIS table name.", ) @@ -268,6 +320,9 @@ class Settings(BaseSettings): "riyadh_bus_stations", "qi_population_metrics", "qi_stripes", + "non_saudi_ownership_zones", + "dimensions", + "building_detection", ], description="A list of all layer names to be processed by the pipeline.", ) diff --git a/src/meshic_pipeline/decoder/mvt_decoder.py b/src/suhail_pipeline/decoder/mvt_decoder.py similarity index 94% rename from src/meshic_pipeline/decoder/mvt_decoder.py rename to src/suhail_pipeline/decoder/mvt_decoder.py index 1cc85f5..68f7ccc 100644 --- a/src/meshic_pipeline/decoder/mvt_decoder.py +++ b/src/suhail_pipeline/decoder/mvt_decoder.py @@ -143,7 +143,17 @@ def decode_bytes( """ if not tile_data: return {} - + + # Guard against non-MVT payloads: the tile server can return an HTML error + # or maintenance page (or other text) instead of a vector tile. MVT is + # binary protobuf and never starts with '<', so a leading angle bracket + # means this isn't a tile — skip cleanly instead of raising a DecodeError. + if tile_data.lstrip()[:1] == b"<": + logger.warning( + "Tile %s/%s/%s payload looks like HTML/non-MVT, skipping decode.", z, x, y + ) + return {} + decoded_tile = mapbox_vector_tile.decode(tile_data) output_layers: Dict[str, List[Dict[str, Any]]] = {} diff --git a/src/meshic_pipeline/discovery/tile_discovery.py b/src/suhail_pipeline/discovery/tile_discovery.py similarity index 98% rename from src/meshic_pipeline/discovery/tile_discovery.py rename to src/suhail_pipeline/discovery/tile_discovery.py index e8fd71e..193c969 100644 --- a/src/meshic_pipeline/discovery/tile_discovery.py +++ b/src/suhail_pipeline/discovery/tile_discovery.py @@ -14,8 +14,8 @@ import re from sqlalchemy.orm import Session from sqlalchemy.dialects.postgresql import insert as pg_insert -from meshic_pipeline.persistence.db import get_db_engine -from meshic_pipeline.persistence.models import TileURL, Province +from suhail_pipeline.persistence.db import get_db_engine +from suhail_pipeline.persistence.models import TileURL, Province from shapely import wkb Z15_OUTPUT_PATH = 'data_raw/validation_reports/Z15_tiles.json' diff --git a/src/meshic_pipeline/downloader/async_tile_downloader.py b/src/suhail_pipeline/downloader/async_tile_downloader.py similarity index 100% rename from src/meshic_pipeline/downloader/async_tile_downloader.py rename to src/suhail_pipeline/downloader/async_tile_downloader.py diff --git a/src/meshic_pipeline/enrichment/__init__.py b/src/suhail_pipeline/enrichment/__init__.py similarity index 100% rename from src/meshic_pipeline/enrichment/__init__.py rename to src/suhail_pipeline/enrichment/__init__.py diff --git a/src/meshic_pipeline/enrichment/api_client.py b/src/suhail_pipeline/enrichment/api_client.py similarity index 53% rename from src/meshic_pipeline/enrichment/api_client.py rename to src/suhail_pipeline/enrichment/api_client.py index 0952a3f..16c0f5e 100644 --- a/src/meshic_pipeline/enrichment/api_client.py +++ b/src/suhail_pipeline/enrichment/api_client.py @@ -2,15 +2,15 @@ import asyncio from typing import List, Dict, Any -from meshic_pipeline.config import settings, ARABIC_COLUMN_MAP -from meshic_pipeline.exceptions import ( +from suhail_pipeline.config import settings, ARABIC_COLUMN_MAP +from suhail_pipeline.exceptions import ( ExternalAPIException, with_retry, RetryConfig, async_error_context, ) -from meshic_pipeline.logging_utils import get_logger, log_performance -from meshic_pipeline.persistence.models import ( +from suhail_pipeline.logging_utils import get_logger, log_performance +from suhail_pipeline.persistence.models import ( Transaction, BuildingRule, ParcelPriceMetric, @@ -20,6 +20,141 @@ logger = get_logger(__name__) +def _safe_int(value: Any) -> int | None: + """Best-effort int coercion for API fields that arrive as int/float/str.""" + if value is None or value == "": + return None + try: + return int(float(value)) + except (TypeError, ValueError): + return None + + +def _safe_float(value: Any) -> float | None: + """Best-effort float coercion for API fields that arrive as int/float/str.""" + if value is None or value == "": + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _parse_iso_date(value: Any): + """Parse an ISO date/datetime string, tolerating a trailing Z; None on failure.""" + if not value: + return None + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + + +# --- Pure payload parsers (transport-free, unit-testable against fixtures) --- + +def parse_transactions_payload(data: Dict[str, Any], parcel_objectid) -> List[Transaction]: + """Parse a `/transactions?parcelObjectId=` payload into Transaction rows. + + Promotes materially-useful attributes out of raw_data into queryable columns + while preserving the full original object in ``raw_data``. + """ + if ( + not data + or not data.get("status") + or "data" not in data + or not data["data"].get("transactions") + ): + return [] + + out: List[Transaction] = [] + for tx_data in data["data"]["transactions"]: + out.append( + Transaction( + transaction_id=tx_data.get("transactionNumber"), + parcel_objectid=parcel_objectid, + transaction_price=tx_data.get("transactionPrice"), + price_of_meter=tx_data.get("priceOfMeter"), + transaction_date=_parse_iso_date(tx_data.get("transactionDate")), + area=tx_data.get("area"), + transaction_type=tx_data.get("type"), + property_type=tx_data.get("propertyType"), + metrics_type=tx_data.get("metricsType"), + land_use_group=tx_data.get("landUseGroup") + or tx_data.get("landUsageGroup"), + land_use_detailed=tx_data.get("landUseaDetailed"), + selling_type=tx_data.get("sellingType"), + transaction_source=tx_data.get("transactionSource"), + total_area=_safe_float(tx_data.get("totalArea")), + subdivision_id=_safe_int(tx_data.get("subdivisionId")), + neighborhood_id=_safe_int(tx_data.get("neighborhoodId")), + is_low_value_transaction=tx_data.get("isLowValueTransaction"), + raw_data=tx_data, + ) + ) + return out + + +def parse_building_rules_payload(data: Dict[str, Any], parcel_objectid) -> List[BuildingRule]: + """Parse a `/parcel/buildingRules?parcelObjectId=` payload into BuildingRule rows.""" + if not data or not data.get("status") or "data" not in data or not data["data"]: + return [] + + out: List[BuildingRule] = [] + for rule_data in data["data"]: + out.append( + BuildingRule( + parcel_objectid=parcel_objectid, + building_rule_id=rule_data.get("id"), + zoning_id=rule_data.get("zoningId"), + zoning_color=rule_data.get("zoningColor"), + zoning_group=rule_data.get("zoningGroup"), + landuse=rule_data.get("landuse"), + description=rule_data.get("description"), + name=rule_data.get("name"), + coloring=rule_data.get("coloring"), + coloring_description=rule_data.get("coloringDescription"), + max_building_coefficient=rule_data.get("maxBuildingCoefficient"), + max_building_height=rule_data.get("maxBuildingHeight"), + max_parcel_coverage=rule_data.get("maxParcelCoverage"), + max_rule_depth=rule_data.get("maxRuleDepth"), + main_streets_setback=rule_data.get("mainStreetsSetback"), + secondary_streets_setback=rule_data.get("secondaryStreetsSetback"), + side_rear_setback=rule_data.get("sideRearSetback"), + raw_data=rule_data, + ) + ) + return out + + +def parse_price_metrics_payload(data: Dict[str, Any]) -> List[ParcelPriceMetric]: + """Parse an `api/parcel/metrics/priceOfMeter` payload into ParcelPriceMetric rows. + + Captures ``neighborhoodId`` (previously always dropped) from the metric row, + falling back to the parcel-level id. + """ + if not data or not data.get("status") or "data" not in data: + return [] + + out: List[ParcelPriceMetric] = [] + for parcel_data in data["data"]: + pid = parcel_data.get("parcelObjId") + parcel_nbhd_id = _safe_int(parcel_data.get("neighborhoodId")) + for source in ("parcelMetrics", "neighborhoodMetrics"): + for metric_data in parcel_data.get(source, []): + out.append( + ParcelPriceMetric( + parcel_objectid=pid, + month=metric_data.get("month"), + year=metric_data.get("year"), + metrics_type=metric_data.get("metricsType"), + average_price_of_meter=metric_data.get("avaragePriceOfMeter"), + neighborhood_id=_safe_int(metric_data.get("neighborhoodId")) + or parcel_nbhd_id, + ) + ) + return out + + class SuhailAPIClient: def __init__(self, session: aiohttp.ClientSession): self.session = session @@ -86,33 +221,7 @@ async def fetch_transactions(self, parcel_objectid: str) -> List[Transaction]: logger.debug(f"No transactions found for parcel {parcel_objectid}") return [] - new_transactions = [] - for tx_data in data["data"]["transactions"]: - transaction_date = None - if tx_data.get("transactionDate"): - try: - transaction_date = datetime.fromisoformat( - tx_data.get("transactionDate").replace("Z", "+00:00") - ) - except ValueError as e: - logger.warning( - f"Failed to parse transaction_date: {tx_data.get('transactionDate')}", - context={ - "parcel_objectid": parcel_objectid, - "error": str(e), - }, - ) - - new_tx = Transaction( - transaction_id=tx_data.get("transactionNumber"), - parcel_objectid=parcel_objectid, - transaction_price=tx_data.get("transactionPrice"), - price_of_meter=tx_data.get("priceOfMeter"), - transaction_date=transaction_date, - area=tx_data.get("area"), - raw_data=tx_data, - ) - new_transactions.append(new_tx) + new_transactions = parse_transactions_payload(data, parcel_objectid) logger.debug( f"Fetched {len(new_transactions)} transactions for parcel {parcel_objectid}", @@ -156,46 +265,7 @@ async def fetch_building_rules(self, parcel_objectid: str) -> List[BuildingRule] response.raise_for_status() data = await response.json() - if ( - not data - or not data.get("status") - or "data" not in data - or not data["data"] - ): - return [] - - rules = [] - for rule_data in data["data"]: - new_rule = BuildingRule( - parcel_objectid=parcel_objectid, - building_rule_id=rule_data.get("id"), - zoning_id=rule_data.get("zoningId"), - zoning_color=rule_data.get("zoningColor"), - zoning_group=rule_data.get("zoningGroup"), - landuse=rule_data.get("landuse"), - description=rule_data.get("description"), - name=rule_data.get("name"), - coloring=rule_data.get("coloring"), - coloring_description=rule_data.get( - "coloringDescription" - ), - max_building_coefficient=rule_data.get( - "maxBuildingCoefficient" - ), - max_building_height=rule_data.get("maxBuildingHeight"), - max_parcel_coverage=rule_data.get("maxParcelCoverage"), - max_rule_depth=rule_data.get("maxRuleDepth"), - main_streets_setback=rule_data.get( - "mainStreetsSetback" - ), - secondary_streets_setback=rule_data.get( - "secondaryStreetsSetback" - ), - side_rear_setback=rule_data.get("sideRearSetback"), - raw_data=rule_data, - ) - rules.append(new_rule) - return rules + return parse_building_rules_payload(data, parcel_objectid) except aiohttp.ClientError as e: logger.warning( @@ -232,39 +302,7 @@ async def fetch_price_metrics( response.raise_for_status() data = await response.json() - if not data or not data.get("status") or "data" not in data: - return [] - - metrics = [] - for parcel_data in data["data"]: - pid = parcel_data.get("parcelObjId") - - # Process parcel-specific metrics (if available) - for metric_data in parcel_data.get("parcelMetrics", []): - new_metric = ParcelPriceMetric( - parcel_objectid=pid, - month=metric_data.get("month"), - year=metric_data.get("year"), - metrics_type=metric_data.get("metricsType"), - average_price_of_meter=metric_data.get( - "avaragePriceOfMeter" - ), - ) - metrics.append(new_metric) - - # Process neighborhood-based metrics (main data source) - for metric_data in parcel_data.get("neighborhoodMetrics", []): - new_metric = ParcelPriceMetric( - parcel_objectid=pid, - month=metric_data.get("month"), - year=metric_data.get("year"), - metrics_type=metric_data.get("metricsType"), - average_price_of_meter=metric_data.get( - "avaragePriceOfMeter" - ), - ) - metrics.append(new_metric) - return metrics + return parse_price_metrics_payload(data) except aiohttp.ClientError as e: logger.warning( diff --git a/src/meshic_pipeline/enrichment/metrics_only_processor.py b/src/suhail_pipeline/enrichment/metrics_only_processor.py similarity index 91% rename from src/meshic_pipeline/enrichment/metrics_only_processor.py rename to src/suhail_pipeline/enrichment/metrics_only_processor.py index ebf0c34..0b2489c 100644 --- a/src/meshic_pipeline/enrichment/metrics_only_processor.py +++ b/src/suhail_pipeline/enrichment/metrics_only_processor.py @@ -3,9 +3,9 @@ """ from typing import AsyncGenerator, List, Tuple -from meshic_pipeline.enrichment.api_client import SuhailAPIClient -from meshic_pipeline.persistence.models import Transaction, BuildingRule, ParcelPriceMetric -from meshic_pipeline.logging_utils import get_logger +from suhail_pipeline.enrichment.api_client import SuhailAPIClient +from suhail_pipeline.persistence.models import Transaction, BuildingRule, ParcelPriceMetric +from suhail_pipeline.logging_utils import get_logger import asyncio logger = get_logger(__name__) diff --git a/src/meshic_pipeline/enrichment/processor.py b/src/suhail_pipeline/enrichment/processor.py similarity index 91% rename from src/meshic_pipeline/enrichment/processor.py rename to src/suhail_pipeline/enrichment/processor.py index 675486d..318320a 100644 --- a/src/meshic_pipeline/enrichment/processor.py +++ b/src/suhail_pipeline/enrichment/processor.py @@ -2,14 +2,14 @@ import aiohttp from typing import List, AsyncGenerator, Tuple -from meshic_pipeline.enrichment.api_client import SuhailAPIClient -from meshic_pipeline.persistence.models import ( +from suhail_pipeline.enrichment.api_client import SuhailAPIClient +from suhail_pipeline.persistence.models import ( Transaction, BuildingRule, ParcelPriceMetric, ) -from meshic_pipeline.logging_utils import get_logger -from meshic_pipeline.persistence.db import get_async_db_engine +from suhail_pipeline.logging_utils import get_logger +from suhail_pipeline.persistence.db import get_async_db_engine from sqlalchemy.ext.asyncio import async_sessionmaker logger = get_logger(__name__) diff --git a/src/meshic_pipeline/enrichment/strategies.py b/src/suhail_pipeline/enrichment/strategies.py similarity index 99% rename from src/meshic_pipeline/enrichment/strategies.py rename to src/suhail_pipeline/enrichment/strategies.py index 416da16..4e57de9 100644 --- a/src/meshic_pipeline/enrichment/strategies.py +++ b/src/suhail_pipeline/enrichment/strategies.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta import re -from meshic_pipeline.logging_utils import get_logger +from suhail_pipeline.logging_utils import get_logger logger = get_logger(__name__) diff --git a/src/meshic_pipeline/exceptions.py b/src/suhail_pipeline/exceptions.py similarity index 100% rename from src/meshic_pipeline/exceptions.py rename to src/suhail_pipeline/exceptions.py diff --git a/src/meshic_pipeline/geometry/stitcher.py b/src/suhail_pipeline/geometry/stitcher.py similarity index 100% rename from src/meshic_pipeline/geometry/stitcher.py rename to src/suhail_pipeline/geometry/stitcher.py diff --git a/src/meshic_pipeline/geometry/validator.py b/src/suhail_pipeline/geometry/validator.py similarity index 100% rename from src/meshic_pipeline/geometry/validator.py rename to src/suhail_pipeline/geometry/validator.py diff --git a/src/meshic_pipeline/logging_utils.py b/src/suhail_pipeline/logging_utils.py similarity index 99% rename from src/meshic_pipeline/logging_utils.py rename to src/suhail_pipeline/logging_utils.py index a9046a2..16f9e1c 100644 --- a/src/meshic_pipeline/logging_utils.py +++ b/src/suhail_pipeline/logging_utils.py @@ -382,7 +382,7 @@ def setup_logging(force_reconfigure: bool = False) -> None: _setup_log_sampling() # Log the configuration setup (but only once) - logger = get_logger("meshic_pipeline.logging") + logger = get_logger("suhail_pipeline.logging") logger.info( "Optimized logging configured", context={ diff --git a/src/meshic_pipeline/memory_utils.py b/src/suhail_pipeline/memory_utils.py similarity index 100% rename from src/meshic_pipeline/memory_utils.py rename to src/suhail_pipeline/memory_utils.py diff --git a/src/meshic_pipeline/persistence/db.py b/src/suhail_pipeline/persistence/db.py similarity index 100% rename from src/meshic_pipeline/persistence/db.py rename to src/suhail_pipeline/persistence/db.py diff --git a/src/meshic_pipeline/persistence/enrichment_persister.py b/src/suhail_pipeline/persistence/enrichment_persister.py similarity index 71% rename from src/meshic_pipeline/persistence/enrichment_persister.py rename to src/suhail_pipeline/persistence/enrichment_persister.py index f51681a..e3bc2b5 100644 --- a/src/meshic_pipeline/persistence/enrichment_persister.py +++ b/src/suhail_pipeline/persistence/enrichment_persister.py @@ -5,7 +5,7 @@ from math import ceil from .models import Transaction, BuildingRule, ParcelPriceMetric -from meshic_pipeline.logging_utils import get_logger +from suhail_pipeline.logging_utils import get_logger logger = get_logger(__name__) @@ -34,6 +34,17 @@ def get_batch_size(num_columns: int) -> int: "price_of_meter": tx.price_of_meter, "transaction_date": tx.transaction_date, "area": tx.area, + "transaction_type": tx.transaction_type, + "property_type": tx.property_type, + "metrics_type": tx.metrics_type, + "land_use_group": tx.land_use_group, + "land_use_detailed": tx.land_use_detailed, + "selling_type": tx.selling_type, + "transaction_source": tx.transaction_source, + "total_area": tx.total_area, + "subdivision_id": tx.subdivision_id, + "neighborhood_id": tx.neighborhood_id, + "is_low_value_transaction": tx.is_low_value_transaction, "raw_data": tx.raw_data, } for tx in transactions @@ -47,10 +58,14 @@ def get_batch_size(num_columns: int) -> int: result = await async_session.execute(stmt) tx_count += result.rowcount or 0 - # Bulk insert rules (deduplicated) + # Bulk insert rules (deduplicated on the full composite PK, not just the parcel). + # A parcel can legitimately have multiple building rules; keying only on + # parcel_objectid previously discarded every rule but the last. rules_count = 0 if rules: - unique_rules = {rule.parcel_objectid: rule for rule in rules} + unique_rules = { + (rule.parcel_objectid, rule.building_rule_id): rule for rule in rules + } rules_values = [ { "parcel_objectid": int(r.parcel_objectid), @@ -83,6 +98,28 @@ def get_batch_size(num_columns: int) -> int: result = await async_session.execute(stmt) rules_count += result.rowcount or 0 + # Ensure neighborhood stubs exist before writing FK-bearing rows. + # Both transactions.neighborhood_id and parcel_price_metrics.neighborhood_id + # reference neighborhoods.neighborhood_id; enrichment can run on parcels whose + # neighborhood row is not present yet, and a single FK violation would abort the + # whole batch. Insert lightweight stubs (id only) mirroring the geometric path. + nbhd_ids = set() + for m in metrics: + if getattr(m, "neighborhood_id", None) is not None: + nbhd_ids.add(int(m.neighborhood_id)) + for tx in transactions: + if getattr(tx, "neighborhood_id", None) is not None: + nbhd_ids.add(int(tx.neighborhood_id)) + if nbhd_ids: + await async_session.execute( + text( + "INSERT INTO public.neighborhoods (neighborhood_id) " + "SELECT unnest(CAST(:ids AS bigint[])) " + "ON CONFLICT (neighborhood_id) DO NOTHING" + ), + {"ids": list(nbhd_ids)}, + ) + # Bulk insert metrics metrics_count = 0 if metrics: @@ -93,6 +130,7 @@ def get_batch_size(num_columns: int) -> int: "year": m.year, "metrics_type": m.metrics_type, "average_price_of_meter": m.average_price_of_meter, + "neighborhood_id": m.neighborhood_id, } for m in metrics ] diff --git a/src/meshic_pipeline/persistence/models.py b/src/suhail_pipeline/persistence/models.py similarity index 67% rename from src/meshic_pipeline/persistence/models.py rename to src/suhail_pipeline/persistence/models.py index d8c4cb6..c36be74 100644 --- a/src/meshic_pipeline/persistence/models.py +++ b/src/suhail_pipeline/persistence/models.py @@ -44,6 +44,25 @@ class Parcel(Base): municipality_ar = Column(String) parcel_id = Column(BigInteger) parcel_no = Column(String) + zoning_group = Column(String) + + # Inline market time-series now emitted on the parcels MVT layer (2026-07). + transaction_price_1w = Column(Float) + transaction_price_1m = Column(Float) + transaction_price_6m = Column(Float) + transaction_price_12m = Column(Float) + price_of_meter_1w = Column(Float) + price_of_meter_1m = Column(Float) + price_of_meter_6m = Column(Float) + price_of_meter_12m = Column(Float) + transactions_count_1w = Column(BigInteger) + transactions_count_1m = Column(BigInteger) + transactions_count_6m = Column(BigInteger) + transactions_count_12m = Column(BigInteger) + transaction_date_1w = Column(DateTime) + transaction_date_1m = Column(DateTime) + transaction_date_6m = Column(DateTime) + transaction_date_12m = Column(DateTime) # From DB Report (audit columns) created_at = Column(DateTime, nullable=False, server_default=func.now()) @@ -69,8 +88,20 @@ class Transaction(Base): price_of_meter = Column(Float) transaction_date = Column(DateTime) area = Column(Float) + # Materially-useful attributes now promoted out of raw_data into queryable columns. + transaction_type = Column(String) + property_type = Column(String) + metrics_type = Column(String) + land_use_group = Column(String) + land_use_detailed = Column(String) + selling_type = Column(String) + transaction_source = Column(String) + total_area = Column(Float) + subdivision_id = Column(BigInteger) + neighborhood_id = Column(BigInteger) + is_low_value_transaction = Column(Boolean) raw_data = Column(JSON) - + parcel = relationship("Parcel", back_populates="transactions") __table_args__ = ( @@ -138,8 +169,27 @@ class Neighborhood(Base): transaction_price = Column(Float) zoning_id = Column(BigInteger) zoning_color = Column(String) + zoning_group = Column(String) geometry_hash = Column(String, nullable=True) + # Inline market time-series now emitted on the neighborhoods MVT layer (2026-07). + transaction_price_1w = Column(Float) + transaction_price_1m = Column(Float) + transaction_price_6m = Column(Float) + transaction_price_12m = Column(Float) + price_of_meter_1w = Column(Float) + price_of_meter_1m = Column(Float) + price_of_meter_6m = Column(Float) + price_of_meter_12m = Column(Float) + transactions_count_1w = Column(BigInteger) + transactions_count_1m = Column(BigInteger) + transactions_count_6m = Column(BigInteger) + transactions_count_12m = Column(BigInteger) + transaction_date_1w = Column(DateTime) + transaction_date_1m = Column(DateTime) + transaction_date_6m = Column(DateTime) + transaction_date_12m = Column(DateTime) + parcels = relationship("Parcel", back_populates="neighborhood") price_metrics = relationship("ParcelPriceMetric", back_populates="neighborhood") province = relationship("Province", back_populates="neighborhoods") @@ -181,6 +231,9 @@ class Subdivision(Base): subdivision_id = Column(BigInteger, primary_key=True) geometry = Column(Geometry('GEOMETRY', srid=4326)) subdivision_no = Column(String) + subdivision_name_ar = Column(String) + neighborhood_id = Column(BigInteger) + region_id = Column(BigInteger) shape_area = Column(Float) transaction_price = Column(Float) price_of_meter = Column(Float) @@ -188,6 +241,24 @@ class Subdivision(Base): zoning_color = Column(String) province_id = Column(BigInteger, ForeignKey('provinces.province_id'), nullable=True) + # Inline market time-series now emitted on the subdivisions MVT layer (2026-07). + transaction_price_1w = Column(Float) + transaction_price_1m = Column(Float) + transaction_price_6m = Column(Float) + transaction_price_12m = Column(Float) + price_of_meter_1w = Column(Float) + price_of_meter_1m = Column(Float) + price_of_meter_6m = Column(Float) + price_of_meter_12m = Column(Float) + transactions_count_1w = Column(BigInteger) + transactions_count_1m = Column(BigInteger) + transactions_count_6m = Column(BigInteger) + transactions_count_12m = Column(BigInteger) + transaction_date_1w = Column(DateTime) + transaction_date_1m = Column(DateTime) + transaction_date_6m = Column(DateTime) + transaction_date_12m = Column(DateTime) + province = relationship("Province", back_populates="subdivisions") class ParcelsBase(Base): @@ -229,14 +300,34 @@ class ParcelsCentroids(Base): transaction_date = Column(DateTime) transaction_price = Column(Float) price_of_meter = Column(Float) + transactions_count = Column(BigInteger) + transaction_price_1w = Column(Float) + transaction_price_1m = Column(Float) + transaction_price_6m = Column(Float) + transaction_price_12m = Column(Float) + price_of_meter_1w = Column(Float) + price_of_meter_1m = Column(Float) + price_of_meter_6m = Column(Float) + price_of_meter_12m = Column(Float) + transactions_count_1w = Column(BigInteger) + transactions_count_1m = Column(BigInteger) + transactions_count_6m = Column(BigInteger) + transactions_count_12m = Column(BigInteger) + transaction_date_1w = Column(DateTime) + transaction_date_1m = Column(DateTime) + transaction_date_6m = Column(DateTime) + transaction_date_12m = Column(DateTime) class BusLines(Base): __tablename__ = 'bus_lines' - id = Column(Integer, primary_key=True, autoincrement=False) + # No stable unique key in the tile; written in replace mode. Native tile fields. + id = Column(Integer, primary_key=True, autoincrement=True) geometry = Column(Geometry('MULTILINESTRING', srid=4326)) busroute = Column(String) - route_name = Column(String) - route_type = Column(String) + color = Column(String) + type = Column(String) + origin = Column(String) + originar = Column(String) class MetroStations(Base): __tablename__ = 'metro_stations' @@ -244,6 +335,8 @@ class MetroStations(Base): geometry = Column(Geometry('POINT', srid=4326)) station_name = Column(String) line = Column(String) + station_long = Column(Float) + station_lat = Column(Float) class RiyadhBusStations(Base): __tablename__ = 'riyadh_bus_stations' @@ -251,18 +344,65 @@ class RiyadhBusStations(Base): geometry = Column(Geometry('POINT', srid=4326)) station_name = Column(String) route = Column(String) + station_long = Column(Float) + station_lat = Column(Float) class QIPopulationMetrics(Base): __tablename__ = 'qi_population_metrics' grid_id = Column(String, primary_key=True) population = Column(Integer) geometry = Column(Geometry('POLYGON', srid=4326)) + region_id = Column(BigInteger) + # Values are colour-coded metric strings (hex) in the current tile schema. + population_density = Column(String) + rent_apartment = Column(String) + rent_villa = Column(String) + rent_shop = Column(String) + rent_office = Column(String) + purchasing_power = Column(String) + weighted_median_income_monthly = Column(String) + poi_count = Column(String) class QIStripes(Base): __tablename__ = 'qi_stripes' strip_id = Column(String, primary_key=True) geometry = Column(Geometry('POLYGON', srid=4326)) value = Column(Float) + centroid_longitude = Column(Float) + centroid_latitude = Column(Float) + +class NonSaudiOwnershipZones(Base): + __tablename__ = 'non_saudi_ownership_zones' + id = Column(BigInteger, primary_key=True, autoincrement=False) + geometry = Column(Geometry('GEOMETRY', srid=4326)) + name_ar = Column(String) + name_en = Column(String) + is_show = Column(Boolean) + region_id = Column(BigInteger) + province_id = Column(BigInteger) + +class Dimensions(Base): + __tablename__ = 'dimensions' + # Per-parcel edge measurements: many rows per parcel (one point per boundary edge), + # no natural unique key -> persisted via tile-scoped delete+append keyed on source_tile. + row_id = Column(BigInteger, primary_key=True, autoincrement=True) + parcel_objectid = Column(BigInteger) + geometry = Column(Geometry('POINT', srid=4326)) + length_m = Column(Float) + azimuth = Column(Float) + province_id = Column(BigInteger) + source_tile = Column(String) + +class BuildingDetection(Base): + __tablename__ = 'building_detection' + # AI-detected building footprints/classification, year-stamped. Keyless in the + # source -> deterministic synthetic bd_id (see SYNTHETIC_PK_CONFIG) for upsert. + bd_id = Column(BigInteger, primary_key=True, autoincrement=False) + geometry = Column(Geometry('GEOMETRY', srid=4326)) + class_pred = Column(String) + prediction_year = Column(BigInteger) + region_id = Column(BigInteger) + source_tile = Column(String) class TileURL(Base): __tablename__ = 'tile_urls' diff --git a/src/meshic_pipeline/persistence/postgis_persister.py b/src/suhail_pipeline/persistence/postgis_persister.py similarity index 74% rename from src/meshic_pipeline/persistence/postgis_persister.py rename to src/suhail_pipeline/persistence/postgis_persister.py index bc73a6e..14413c9 100644 --- a/src/meshic_pipeline/persistence/postgis_persister.py +++ b/src/suhail_pipeline/persistence/postgis_persister.py @@ -23,6 +23,28 @@ def _quote_identifier(name: str) -> str: raise ValueError(f"Unsafe identifier: {name}") return str(quoted_name(name, quote=True)) +# Time-series market fields now emitted inline on the parcels / neighborhoods / +# subdivisions MVT layers (source of truth: live tile decode, 2026-07). Kept as a +# helper so the four windows (1w/1m/6m/12m) stay in sync across layers. +_MARKET_TIMESERIES = { + 'transaction_price_1w': 'float64', + 'transaction_price_1m': 'float64', + 'transaction_price_6m': 'float64', + 'transaction_price_12m': 'float64', + 'price_of_meter_1w': 'float64', + 'price_of_meter_1m': 'float64', + 'price_of_meter_6m': 'float64', + 'price_of_meter_12m': 'float64', + 'transactions_count_1w': 'int64', + 'transactions_count_1m': 'int64', + 'transactions_count_6m': 'int64', + 'transactions_count_12m': 'int64', + 'transaction_date_1w': 'datetime64[ns]', + 'transaction_date_1m': 'datetime64[ns]', + 'transaction_date_6m': 'datetime64[ns]', + 'transaction_date_12m': 'datetime64[ns]', +} + # Add a canonical schema map for all layers SCHEMA_MAP = { 'parcels': { @@ -33,7 +55,9 @@ def _quote_identifier(name: str) -> str: 'subdivision_no': 'string', 'transaction_price': 'float64', 'zoning_id': 'int64', + 'zoning_group': 'string', 'neighborhood_id': 'int64', + 'neighborhood_ar': 'string', 'block_no': 'string', 'subdivision_id': 'int64', 'price_of_meter': 'float64', @@ -44,6 +68,7 @@ def _quote_identifier(name: str) -> str: 'municipality_ar': 'string', 'parcel_id': 'int64', 'parcel_no': 'string', + **_MARKET_TIMESERIES, 'created_at': 'datetime64[ns]', 'updated_at': 'datetime64[ns]', 'is_active': 'bool', @@ -56,11 +81,14 @@ def _quote_identifier(name: str) -> str: 'transaction_date': 'datetime64[ns]', 'transaction_price': 'float64', 'price_of_meter': 'float64', + 'transactions_count': 'int64', + **_MARKET_TIMESERIES, }, 'neighborhoods': { 'neighborhood_id': 'int64', 'geometry': 'geometry', 'neighborhood_name': 'string', + 'neighborhood_ar': 'string', 'region_id': 'int64', 'province_id': 'int64', 'price_of_meter': 'float64', @@ -68,6 +96,8 @@ def _quote_identifier(name: str) -> str: 'transaction_price': 'float64', 'zoning_id': 'int64', 'zoning_color': 'string', + 'zoning_group': 'string', + **_MARKET_TIMESERIES, 'geometry_hash': 'string', }, 'neighborhoods-centroids': { @@ -82,12 +112,38 @@ def _quote_identifier(name: str) -> str: 'subdivision_id': 'int64', 'geometry': 'geometry', 'subdivision_no': 'string', + 'subdivision_name_ar': 'string', + 'neighborhood_id': 'int64', + 'region_id': 'int64', 'shape_area': 'float64', 'transaction_price': 'float64', 'price_of_meter': 'float64', 'zoning_id': 'int64', 'zoning_color': 'string', 'province_id': 'int64', + **_MARKET_TIMESERIES, + }, + # `dimensions` (per-parcel edge measurements) is multi-row-per-parcel with no + # natural unique key. It is persisted with a tile-scoped delete+append write + # (see TILE_SCOPED_LAYERS / write_tile_scoped): `source_tile` records the + # originating tile so a reprocessed tile replaces exactly its own rows. + 'dimensions': { + 'parcel_objectid': 'int64', + 'geometry': 'geometry', + 'length_m': 'float64', + 'azimuth': 'float64', + 'province_id': 'int64', + 'source_tile': 'string', + }, + # `streets` remains keyless / not yet enabled (needs the same accumulate + # semantics as dimensions before it can be persisted across a province run). + # The tile `provinces` layer is intentionally omitted: it collides with the + # `provinces` metadata table (populated from the regions API) and has no unique key. + 'streets': { + 'geometry': 'geometry', + 'name_ar': 'string', + 'name_en': 'string', + 'width': 'float64', }, 'metro_lines': { 'id': 'int64', @@ -97,41 +153,107 @@ def _quote_identifier(name: str) -> str: 'track_name': 'string', }, 'bus_lines': { - 'id': 'int64', + # Native live-tile field names (2026-07): color/type/busroute/origin/originar. 'geometry': 'geometry', - 'route_name': 'string', - 'route_color': 'string', - 'route_length': 'float64', - 'route_type': 'string', - 'route_id': 'string', + 'busroute': 'string', + 'color': 'string', + 'type': 'string', + 'origin': 'string', + 'originar': 'string', }, 'metro_stations': { 'station_code': 'string', 'geometry': 'geometry', 'station_name': 'string', - 'line_id': 'int64', - 'location': 'string', + 'station_long': 'float64', + 'station_lat': 'float64', }, 'riyadh_bus_stations': { 'station_code': 'string', 'geometry': 'geometry', 'station_name': 'string', - 'route_id': 'string', - 'location': 'string', + 'station_long': 'float64', + 'station_lat': 'float64', }, 'qi_population_metrics': { 'grid_id': 'string', - 'population': 'int64', 'geometry': 'geometry', + 'region_id': 'int64', + 'population_density': 'string', + 'rent_apartment': 'string', + 'rent_villa': 'string', + 'rent_shop': 'string', + 'rent_office': 'string', + 'purchasing_power': 'string', + 'weighted_median_income_monthly': 'string', + 'poi_count': 'string', }, 'qi_stripes': { 'strip_id': 'string', 'geometry': 'geometry', - 'stripe_value': 'float64', - 'stripe_type': 'string', + 'centroid_longitude': 'float64', + 'centroid_latitude': 'float64', + }, + # `building_detection` is keyless in the source; we synthesize a stable bd_id + # (see SYNTHETIC_PK_CONFIG) from region + class + prediction_year + geometry so + # the existing upsert path accumulates it and versions it by prediction year. + 'building_detection': { + 'bd_id': 'int64', + 'geometry': 'geometry', + 'class_pred': 'string', + 'prediction_year': 'int64', + 'region_id': 'int64', + 'source_tile': 'string', + }, + 'non_saudi_ownership_zones': { + 'id': 'int64', + 'geometry': 'geometry', + 'name_ar': 'string', + 'name_en': 'string', + 'is_show': 'bool', + 'region_id': 'int64', + 'province_id': 'int64', }, } +# Layers persisted with a tile-scoped delete+append (keyless, multi-row-per-key). +# Each batch deletes the rows for the tiles it re-decoded (via ``source_tile``) and +# appends fresh ones, so reruns are idempotent and multi-tile runs accumulate. +TILE_SCOPED_LAYERS = {"dimensions"} + +# Layers with no natural unique key that get a deterministic synthetic primary key +# derived from stable content + geometry, so the normal upsert path accumulates them. +# Format: layer -> (id_column, [key_columns]). +SYNTHETIC_PK_CONFIG = { + "building_detection": ("bd_id", ["region_id", "class_pred", "prediction_year"]), +} + + +def compute_synthetic_pk(gdf: gpd.GeoDataFrame, layer_name: str) -> gpd.GeoDataFrame: + """Add a deterministic BIGINT primary key for keyless layers (SYNTHETIC_PK_CONFIG). + + The key is a hash of the configured content columns plus the geometry, so the + same feature always hashes to the same id (idempotent upsert, cross-tile-overlap + dedup) while distinct features stay distinct. + """ + cfg = SYNTHETIC_PK_CONFIG.get(layer_name) + if cfg is None or gdf.empty: + return gdf + id_col, key_cols = cfg + geom_col = gdf.geometry.name + + def _key(row) -> int: + parts = [str(row.get(c)) for c in key_cols] + geom = row[geom_col] + parts.append(geom.wkb_hex if geom is not None and not geom.is_empty else "") + digest = hashlib.sha1("|".join(parts).encode("utf-8")).hexdigest() + # 60 bits keeps it comfortably inside a signed BIGINT. + return int(digest[:15], 16) + + out = gdf.copy() + out[id_col] = out.apply(_key, axis=1) + return out + def ensure_neighborhood_centroids_primary_key(gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame: """Ensure neighborhoods-centroids rows have integer `id` for upsert (matches DB PK).""" @@ -438,6 +560,72 @@ def _upsert(self, gdf: gpd.GeoDataFrame, table_name: str, id_column: str, schema self.drop_table(temp_table_name, schema) logger.debug("Dropped temporary upsert table: %s", temp_table_name) + def write_tile_scoped( + self, + gdf: gpd.GeoDataFrame, + layer_name: str, + table: str, + scope_col: str = "source_tile", + schema: str = "public", + chunksize: int = 5000, + geometry_type: str | None = None, + ) -> None: + """Idempotent accumulate write for keyless / multi-row-per-key layers. + + Deletes the existing rows for exactly the tiles present in ``gdf`` (via + ``scope_col``) and appends the new rows. Reprocessing a tile replaces only + that tile's rows; different tiles accumulate. Used for TILE_SCOPED_LAYERS. + """ + if gdf.empty: + return + if scope_col not in gdf.columns: + raise ValueError( + f"write_tile_scoped requires a '{scope_col}' column on layer '{layer_name}'" + ) + + validated_gdf = self._validate_and_cast_types(gdf, layer_name=layer_name) + tile_keys = sorted( + str(v) for v in validated_gdf[scope_col].dropna().unique() + ) + + inspector = inspect(self.engine) + if not inspector.has_table(table, schema=schema): + self.create_table_from_gdf( + validated_gdf.iloc[0:0], + table, + schema=schema, + known_columns=list(validated_gdf.columns), + geometry_type=geometry_type or "GEOMETRY", + ) + + schema_q = _quote_identifier(schema) + table_q = _quote_identifier(table) + scope_q = _quote_identifier(scope_col) + if tile_keys: + with self.engine.begin() as conn: + conn.execute( + text( + f'DELETE FROM {schema_q}.{table_q} ' + f'WHERE {scope_q} = ANY(:keys)' + ), + {"keys": tile_keys}, + ) + validated_gdf.to_postgis( + table, + self.engine, + schema=schema, + if_exists="append", + index=False, + chunksize=chunksize, + ) + logger.info( + "Tile-scoped write: %d rows into %s.%s across %d tile(s)", + len(validated_gdf), + schema, + table, + len(tile_keys), + ) + def write( self, gdf: gpd.GeoDataFrame, diff --git a/src/meshic_pipeline/persistence/table_management.py b/src/suhail_pipeline/persistence/table_management.py similarity index 100% rename from src/meshic_pipeline/persistence/table_management.py rename to src/suhail_pipeline/persistence/table_management.py diff --git a/src/meshic_pipeline/pipeline_orchestrator.py b/src/suhail_pipeline/pipeline_orchestrator.py similarity index 97% rename from src/meshic_pipeline/pipeline_orchestrator.py rename to src/suhail_pipeline/pipeline_orchestrator.py index 4308a19..924bac1 100644 --- a/src/meshic_pipeline/pipeline_orchestrator.py +++ b/src/suhail_pipeline/pipeline_orchestrator.py @@ -21,10 +21,12 @@ from .persistence.postgis_persister import ( PostGISPersister, SCHEMA_MAP, + TILE_SCOPED_LAYERS, + compute_synthetic_pk, ensure_neighborhood_centroids_primary_key, ) from sqlalchemy import text -from meshic_pipeline.persistence.table_management import reset_temp_table +from suhail_pipeline.persistence.table_management import reset_temp_table import mercantile @@ -162,6 +164,12 @@ def decode_and_validate_tile( gdf = validate_geometries(gdf) if not gdf.empty: + # Provenance: stamp the source tile for layers whose schema records it + # (drives the tile-scoped delete+append write for keyless layers). + if 'source_tile' in SCHEMA_MAP.get(layer_name, {}): + gdf['source_tile'] = f"{z}/{x}/{y}" + # Deterministic synthetic PK for keyless layers (e.g. building_detection). + gdf = compute_synthetic_pk(gdf, layer_name) validated_gdfs.append((layer_name, gdf)) stats = monitor.get_memory_stats() diff --git a/src/meshic_pipeline/run_db_geometric.py b/src/suhail_pipeline/run_db_geometric.py similarity index 94% rename from src/meshic_pipeline/run_db_geometric.py rename to src/suhail_pipeline/run_db_geometric.py index 2f0863c..4f43e03 100644 --- a/src/meshic_pipeline/run_db_geometric.py +++ b/src/suhail_pipeline/run_db_geometric.py @@ -11,20 +11,21 @@ from sqlalchemy.orm import Session from sqlalchemy import inspect -from meshic_pipeline.config import settings -from meshic_pipeline.persistence.db import get_db_engine -from meshic_pipeline.persistence.models import TileURL -from meshic_pipeline.decoder.mvt_decoder import MVTDecoder -from meshic_pipeline.geometry.validator import validate_geometries -from meshic_pipeline.persistence.postgis_persister import ( +from suhail_pipeline.config import settings +from suhail_pipeline.persistence.db import get_db_engine +from suhail_pipeline.persistence.models import TileURL +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder +from suhail_pipeline.geometry.validator import validate_geometries +from suhail_pipeline.persistence.postgis_persister import ( PostGISPersister, SCHEMA_MAP, + TILE_SCOPED_LAYERS, ensure_neighborhood_centroids_primary_key, ensure_neighborhood_stubs_for_parcels, ensure_subdivision_stubs_for_parcels, ) -from meshic_pipeline.persistence.table_management import reset_temp_table -from meshic_pipeline.pipeline_orchestrator import decode_and_validate_tile +from suhail_pipeline.persistence.table_management import reset_temp_table +from suhail_pipeline.pipeline_orchestrator import decode_and_validate_tile logger = logging.getLogger(__name__) @@ -194,6 +195,15 @@ def persist_layers() -> None: gdf = gpd.GeoDataFrame(pd.concat(gdfs, ignore_index=True)) + # Keyless multi-row layers (e.g. dimensions): idempotent tile-scoped + # accumulate instead of the per-batch replace that would keep only the + # last batch. Deletes this batch's tiles' rows, then appends. + if layer in TILE_SCOPED_LAYERS: + persister.write_tile_scoped( + gdf, layer, table_name, chunksize=settings.db_chunk_size + ) + continue + pk_col = settings.id_column_per_layer.get(layer) if pk_col and pk_col in gdf.columns: before = len(gdf) diff --git a/src/meshic_pipeline/run_db_geometric_fixed.py b/src/suhail_pipeline/run_db_geometric_fixed.py similarity index 97% rename from src/meshic_pipeline/run_db_geometric_fixed.py rename to src/suhail_pipeline/run_db_geometric_fixed.py index 9b13c1e..827bf08 100644 --- a/src/meshic_pipeline/run_db_geometric_fixed.py +++ b/src/suhail_pipeline/run_db_geometric_fixed.py @@ -11,20 +11,20 @@ from sqlalchemy.orm import Session from sqlalchemy import inspect -from meshic_pipeline.config import settings -from meshic_pipeline.persistence.db import get_db_engine -from meshic_pipeline.persistence.models import TileURL -from meshic_pipeline.decoder.mvt_decoder import MVTDecoder -from meshic_pipeline.geometry.validator import validate_geometries -from meshic_pipeline.persistence.postgis_persister import ( +from suhail_pipeline.config import settings +from suhail_pipeline.persistence.db import get_db_engine +from suhail_pipeline.persistence.models import TileURL +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder +from suhail_pipeline.geometry.validator import validate_geometries +from suhail_pipeline.persistence.postgis_persister import ( PostGISPersister, SCHEMA_MAP, ensure_neighborhood_centroids_primary_key, ensure_neighborhood_stubs_for_parcels, ensure_subdivision_stubs_for_parcels, ) -from meshic_pipeline.persistence.table_management import reset_temp_table -from meshic_pipeline.pipeline_orchestrator import decode_and_validate_tile +from suhail_pipeline.persistence.table_management import reset_temp_table +from suhail_pipeline.pipeline_orchestrator import decode_and_validate_tile logger = logging.getLogger(__name__) diff --git a/src/meshic_pipeline/run_enrichment_pipeline.py b/src/suhail_pipeline/run_enrichment_pipeline.py similarity index 95% rename from src/meshic_pipeline/run_enrichment_pipeline.py rename to src/suhail_pipeline/run_enrichment_pipeline.py index c7ef007..3dc0316 100644 --- a/src/meshic_pipeline/run_enrichment_pipeline.py +++ b/src/suhail_pipeline/run_enrichment_pipeline.py @@ -20,20 +20,20 @@ project_root = Path(__file__).resolve().parents[1] sys.path.insert(0, str(project_root)) -from meshic_pipeline.persistence.models import ( +from suhail_pipeline.persistence.models import ( Base, Transaction, ParcelPriceMetric, BuildingRule, ) -from meshic_pipeline.config import settings +from suhail_pipeline.config import settings import logging from sqlalchemy import BigInteger, Column, DateTime, Float, Integer, String, JSON from sqlalchemy.orm import sessionmaker from sqlalchemy import ForeignKey -from meshic_pipeline.exceptions import ValidationException -from meshic_pipeline.logging_utils import get_logger -from meshic_pipeline.enrichment.strategies import ( +from suhail_pipeline.exceptions import ValidationException +from suhail_pipeline.logging_utils import get_logger +from suhail_pipeline.enrichment.strategies import ( get_unprocessed_parcel_ids, get_stale_parcel_ids, get_delta_parcel_ids, @@ -41,16 +41,16 @@ get_all_parcel_ids_for_metrics, get_delta_parcel_ids_with_details, ) -from meshic_pipeline.enrichment.processor import fast_worker -from meshic_pipeline.enrichment.metrics_only_processor import metrics_only_worker -from meshic_pipeline.persistence.db import ( +from suhail_pipeline.enrichment.processor import fast_worker +from suhail_pipeline.enrichment.metrics_only_processor import metrics_only_worker +from suhail_pipeline.persistence.db import ( get_async_db_engine, setup_database_async, ) -from meshic_pipeline.persistence.enrichment_persister import ( +from suhail_pipeline.persistence.enrichment_persister import ( fast_store_batch_data, ) -from meshic_pipeline.enrichment.api_client import SuhailAPIClient +from suhail_pipeline.enrichment.api_client import SuhailAPIClient from rich import print as rprint logger = get_logger(__name__) @@ -358,7 +358,7 @@ async def run_delta_enrichment(): if auto_created_table: typer.echo("🧹 Cleaning up auto-generated temporary table...") try: - from meshic_pipeline.persistence.postgis_persister import PostGISPersister + from suhail_pipeline.persistence.postgis_persister import PostGISPersister persister = PostGISPersister(str(settings.database_url)) persister.drop_table(fresh_mvt_table) typer.echo(f"🧹 Cleaned up temporary table: {fresh_mvt_table}") @@ -388,7 +388,7 @@ def smart_pipeline_enrich( if geometric_first: print("\n🗺️ STAGE 1: Running geometric pipeline...") - from meshic_pipeline.run_geometric_pipeline import main as run_geometric_pipeline + from suhail_pipeline.run_geometric_pipeline import main as run_geometric_pipeline import sys # Prepare args for geometric pipeline diff --git a/src/meshic_pipeline/run_geometric_pipeline.py b/src/suhail_pipeline/run_geometric_pipeline.py similarity index 97% rename from src/meshic_pipeline/run_geometric_pipeline.py rename to src/suhail_pipeline/run_geometric_pipeline.py index cad6232..c6022e6 100644 --- a/src/meshic_pipeline/run_geometric_pipeline.py +++ b/src/suhail_pipeline/run_geometric_pipeline.py @@ -6,8 +6,8 @@ import typer -from meshic_pipeline.pipeline_orchestrator import run_pipeline -from meshic_pipeline.config import settings +from suhail_pipeline.pipeline_orchestrator import run_pipeline +from suhail_pipeline.config import settings # Setup logging logger = logging.getLogger(__name__) diff --git a/src/meshic_pipeline/run_monitoring.py b/src/suhail_pipeline/run_monitoring.py similarity index 94% rename from src/meshic_pipeline/run_monitoring.py rename to src/suhail_pipeline/run_monitoring.py index d88466f..d3d4a7d 100644 --- a/src/meshic_pipeline/run_monitoring.py +++ b/src/suhail_pipeline/run_monitoring.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Monitoring utilities for Meshic pipeline. +Monitoring utilities for Suhail pipeline. Provides: - status: queue + enrichment overview - recommend: next action suggestions @@ -16,7 +16,7 @@ import time import statistics -from meshic_pipeline.config import settings +from suhail_pipeline.config import settings app = typer.Typer() @@ -171,23 +171,23 @@ def recommend(): if new_parcels > 10000: print(f"🚀 HIGHEST PRIORITY: {new_parcels:,} unenriched parcels with transaction data") print(f" 💡 Stage 1 revealed parcels with transaction_price > 0.") - print(f" Recommended: meshic-pipeline fast-enrich --batch-size 400") + print(f" Recommended: suhail-pipeline fast-enrich --batch-size 400") print(" → Leverages transaction_price field from MVT tiles for maximum efficiency") print() elif new_parcels > 1000: print(f"🆕 HIGH PRIORITY: {new_parcels:,} unenriched parcels found") - print(f" Recommended: meshic-pipeline fast-enrich --batch-size 300") + print(f" Recommended: suhail-pipeline fast-enrich --batch-size 300") print() # Priority 2: Incremental updates for already enriched parcels if stale_7d > 5000: print(f"⚠️ MEDIUM PRIORITY: {stale_7d:,} stale parcels (7+ days)") print(f" 💡 May have new transactions not yet captured") - print(f" Recommended: meshic-pipeline incremental-enrich --days-old 7") + print(f" Recommended: suhail-pipeline incremental-enrich --days-old 7") print() elif stale_30d > 1000: print(f"⏰ LOW PRIORITY: {stale_30d:,} stale parcels (30+ days)") - print(f" Recommended: meshic-pipeline incremental-enrich --days-old 30") + print(f" Recommended: suhail-pipeline incremental-enrich --days-old 30") print() # System status @@ -206,7 +206,7 @@ def recommend(): print(" 💡 The most efficient method is 'delta-enrich', which only processes parcels with detected changes.") print() print(" 🆕 RECOMMENDED WORKFLOW: DELTA ENRICHMENT (Maximum Precision):") - print(" meshic-pipeline delta-enrich --auto-geometric") + print(" suhail-pipeline delta-enrich --auto-geometric") print(" → MVT-based change detection: Only enrich parcels with ACTUAL price changes") print(" → Detects: New parcels, price changes, market updates") print(" → Ultimate efficiency: Only processes parcels with proven changes") @@ -226,28 +226,28 @@ def schedule_info(): print("🔄 RECOMMENDED SCHEDULE:") print() print("1️⃣ DAILY / WEEKLY:") - print(" meshic-pipeline delta-enrich --auto-geometric") + print(" suhail-pipeline delta-enrich --auto-geometric") print(" → The most efficient method. Automatically finds and processes only parcels") print(" → with genuine changes, ensuring your data is always current.") print() print("2️⃣ MONTHLY (or as needed for a broader refresh):") - print(" meshic-pipeline incremental-enrich --days-old 30") + print(" suhail-pipeline incremental-enrich --days-old 30") print(" → Catches any parcels that might have been missed by delta detection.") print() print("3️⃣ QUARTERLY (for data integrity checks):") - print(" meshic-pipeline full-refresh") + print(" suhail-pipeline full-refresh") print(" → Complete refresh to guarantee data completeness.") print() print("💡 TIPS:") - print(" • Monitor with: meshic-pipeline monitor status") - print(" • Get recommendations: meshic-pipeline monitor recommend") + print(" • Monitor with: suhail-pipeline monitor status") + print(" • Get recommendations: suhail-pipeline monitor recommend") @app.command(name="reset-stale") def reset_stale(stale_minutes: int = typer.Option(60, "--stale-minutes", help="Minutes after which in_progress tiles are considered stale")): """Reset stale in_progress tiles to failed so they can be retried.""" from sqlalchemy.orm import Session - from meshic_pipeline.persistence.db import get_db_engine - from meshic_pipeline.persistence.models import TileURL + from suhail_pipeline.persistence.db import get_db_engine + from suhail_pipeline.persistence.models import TileURL engine = get_db_engine(str(settings.database_url)) session = Session(engine) diff --git a/src/meshic_pipeline/run_tile_pipeline.py b/src/suhail_pipeline/run_tile_pipeline.py similarity index 95% rename from src/meshic_pipeline/run_tile_pipeline.py rename to src/suhail_pipeline/run_tile_pipeline.py index 99df3b5..d0a1d2d 100644 --- a/src/meshic_pipeline/run_tile_pipeline.py +++ b/src/suhail_pipeline/run_tile_pipeline.py @@ -2,8 +2,8 @@ import asyncio import aiohttp from sqlalchemy.orm import Session -from meshic_pipeline.persistence.db import get_db_engine -from meshic_pipeline.persistence.models import TileURL +from suhail_pipeline.persistence.db import get_db_engine +from suhail_pipeline.persistence.models import TileURL BATCH_SIZE = 1000 MAX_RETRIES = 5 diff --git a/src/meshic_pipeline/show_discovery_summary.py b/src/suhail_pipeline/show_discovery_summary.py similarity index 86% rename from src/meshic_pipeline/show_discovery_summary.py rename to src/suhail_pipeline/show_discovery_summary.py index 7e6de91..d7c34fe 100644 --- a/src/meshic_pipeline/show_discovery_summary.py +++ b/src/suhail_pipeline/show_discovery_summary.py @@ -9,8 +9,8 @@ # Add the src directory to the path so we can import our modules sys.path.append(str(Path(__file__).parent.parent / "src")) -from meshic_pipeline.config import settings -from meshic_pipeline.utils.tile_list_generator import tiles_from_bbox_z +from suhail_pipeline.config import settings +from suhail_pipeline.utils.tile_list_generator import tiles_from_bbox_z def main(): """Display enhanced province discovery summary.""" @@ -46,13 +46,13 @@ def main(): print(f"\n🚀 Integration Examples:") print(f" # Single province") - print(f" python -m meshic_pipeline.cli province-geometric riyadh") + print(f" python -m suhail_pipeline.cli province-geometric riyadh") print(f" ") print(f" # All Saudi Arabia") - print(f" python -m meshic_pipeline.cli saudi-arabia-geometric") + print(f" python -m suhail_pipeline.cli saudi-arabia-geometric") print(f" ") print(f" # Efficient strategy for large areas") - print(f" python -m meshic_pipeline.cli saudi-arabia-geometric --strategy efficient") + print(f" python -m suhail_pipeline.cli saudi-arabia-geometric --strategy efficient") print(f"\n✅ System Status: Ready for production use!") diff --git a/src/meshic_pipeline/utils/tile_list_generator.py b/src/suhail_pipeline/utils/tile_list_generator.py similarity index 100% rename from src/meshic_pipeline/utils/tile_list_generator.py rename to src/suhail_pipeline/utils/tile_list_generator.py diff --git a/tests/fixtures/suhail_live_2026_07/README.md b/tests/fixtures/suhail_live_2026_07/README.md new file mode 100644 index 0000000..ff6dc18 --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/README.md @@ -0,0 +1,27 @@ +# Suhail live-source fixtures (captured 2026-07-15) + +Real payloads captured from the live Suhail platform, used as regression / contract-test +inputs and as evidence for [`docs/SUHAIL_SOURCE_AUDIT_2026-07.md`](../../../docs/SUHAIL_SOURCE_AUDIT_2026-07.md). + +## `tiles/` — MVT vector tiles (gzip-compressed, as served) +- `riyadh_15_20636_14069.vector.pbf.gz` — downtown Riyadh; contains all 17 tile layers + (parcels, parcels-base, parcels-centroids, neighborhoods(+centroids), subdivisions, + provinces, dimensions, streets, metro_lines, bus_lines, metro_stations, + riyadh_bus_stations, qi_population_metrics, qi_stripes, building_detection, + non_saudi_ownership_zones). +- `riyadh_15_20640_14060.vector.pbf.gz` — residential Riyadh; parcels with recent transactions. + +Tiles are served with `Content-Encoding: gzip` (always on). `aiohttp` auto-decompresses in +the pipeline; test helpers `gzip.decompress()` first. + +## `api/` — api2.suhail.ai JSON payloads +- `regions.json`, `settings_app.json`, `tiles_modes.json`, `gl_style_ksa.sources_and_layers.json` + — platform/config surface. +- `transactions__9941681.json` — 6 real transaction records (39 fields each). +- `buildingRules__9858274.json`, `priceOfMeter__9858274.json` — enrichment payloads. +- `parcel_detail__9858274.json` — consolidated `parcel/{id}` endpoint (new). +- `landMetrics_list__region10.json`, `landMetrics__neighborhood1002969.json`, + `landZoningGroups.json` — new neighbourhood market-intelligence endpoints (not yet ingested). + +Consumed by `tests/unit/test_live_tile_schema.py` and +`tests/unit/test_enrichment_parsers_contract.py`. diff --git a/tests/fixtures/suhail_live_2026_07/api/buildingRules__9858274.json b/tests/fixtures/suhail_live_2026_07/api/buildingRules__9858274.json new file mode 100644 index 0000000..d27dfbf --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/buildingRules__9858274.json @@ -0,0 +1 @@ +{"data":[{"id":"ع/م 111","zoningId":5,"zoningColor":"e74545","zoningGroup":null,"landuse":"سكني / تجاري / مكتبي","description":"ع/م 111","name":"منطقة التقسيم ع/م 111","coloring":"MU","coloringDescription":"م1 - استعمال مختلط - دورين - 60% تغطية - 1.2 معامل بناء","maxBuildingCoefficient":"2","maxBuildingHeight":"ارضي + اول + 50% ملاحق علوية","maxParcelCoverage":"100%","maxRuleDepth":"يجب مراجعة امانة منطقة الرياض لتحديد عمق الإستخدام التجاري المسموح به","mainStreetsSetback":"بدون","secondaryStreetsSetback":"بدون","sideRearSetback":"بدون"}],"message":null,"messageTemplate":null,"status":true,"meta":null} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/api/gl_style_ksa.sources_and_layers.json b/tests/fixtures/suhail_live_2026_07/api/gl_style_ksa.sources_and_layers.json new file mode 100644 index 0000000..0d08755 --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/gl_style_ksa.sources_and_layers.json @@ -0,0 +1,922 @@ +{ + "sources": { + "mapbox://mapbox.satellite": { + "url": "mapbox://mapbox.satellite", + "type": "raster", + "tileSize": 256 + }, + "composite": { + "url": "mapbox://mapbox.mapbox-streets-v8,suhail-app.c4y35hil,mapbox.mapbox-terrain-v2", + "type": "vector" + }, + "province-61000-src": { + "type": "geojson", + "data": { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 42.50618302642124, + 18.216490450501226 + ] + } + } + }, + "ksa": { + "type": "vector", + "maxzoom": 15, + "tiles": [ + "https://tiles.suhail.ai/maps/ksa/{z}/{x}/{y}.vector.pbf?" + ] + }, + "riyadh": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 41.686026, + 19.208334, + 48.1838, + 27.7021 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/riyadh/{z}/{x}/{y}.vector.pbf?" + ] + }, + "al_madenieh": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 37.084, + 22.51971, + 42.1566, + 27.46665 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/madinah/{z}/{x}/{y}.vector.pbf?" + ] + }, + "al_qassim": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 41.307313, + 24.4836, + 44.989467, + 28.2508 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/qassim/{z}/{x}/{y}.vector.pbf?" + ] + }, + "asir_region": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 41.214223, + 17.366667, + 44.440789, + 20.80266 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/asir/{z}/{x}/{y}.vector.pbf?" + ] + }, + "eastern_region": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 44.670664, + 19.000517, + 55.666667, + 29.141386 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/eastern/{z}/{x}/{y}.vector.pbf?" + ] + }, + "makkah_region": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 38.61693, + 18.564691, + 43.83016, + 23.6952 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/makkah/{z}/{x}/{y}.vector.pbf?" + ] + }, + "bahah": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 40.829164, + 19.44581, + 42.129716, + 20.82938 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/bahah/{z}/{x}/{y}.vector.pbf?" + ] + }, + "hail": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 39.11418, + 25.25439, + 44.367661, + 29.207876 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/hail/{z}/{x}/{y}.vector.pbf?" + ] + }, + "jawf": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 37.005875, + 28.36794, + 41.99684, + 31.749873 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/jawf/{z}/{x}/{y}.vector.pbf?" + ] + }, + "jazan": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 41.38948, + 16.08843, + 43.3365, + 18.060398 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/jazan/{z}/{x}/{y}.vector.pbf?" + ] + }, + "najran": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 43.59569, + 16.95, + 52.001753, + 19.448889 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/najran/{z}/{x}/{y}.vector.pbf?" + ] + }, + "northern_borders": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 37.955253, + 27.831761, + 45.464141, + 32.153962 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/northern_borders/{z}/{x}/{y}.vector.pbf?" + ] + }, + "tabuk": { + "type": "vector", + "maxzoom": 15, + "bounds": [ + 34.571234, + 24.560854, + 39.923088, + 29.976269 + ], + "tiles": [ + "https://tiles.suhail.ai/maps/tabuk/{z}/{x}/{y}.vector.pbf?" + ] + }, + "base-raster-riyadh": { + "type": "raster", + "scheme": "tms", + "minzoom": 8, + "maxzoom": 18, + "bounds": [ + 41.686026, + 19.208334, + 48.1838, + 27.7021 + ], + "tiles": [ + "https://qeye-tiles-prod.obs.me-east-1.myhuaweicloud.com/1/2023-08-01/{z}/{x}/{y}.png" + ] + }, + "base-raster-al_madenieh": { + "type": "raster", + "scheme": "tms", + "minzoom": 8, + "maxzoom": 18, + "bounds": [ + 37.084, + 22.51971, + 42.1566, + 27.46665 + ], + "tiles": [ + "https://qeye-tiles-prod.obs.me-east-1.myhuaweicloud.com/3/2023-08-01/{z}/{x}/{y}.png" + ] + }, + "base-raster-eastern_region": { + "type": "raster", + "scheme": "tms", + "minzoom": 8, + "maxzoom": 18, + "bounds": [ + 44.670664, + 19.000517, + 55.666667, + 29.141386 + ], + "tiles": [ + "https://qeye-tiles-prod.obs.me-east-1.myhuaweicloud.com/4/2023-08-01/{z}/{x}/{y}.png" + ] + }, + "base-raster-makkah_region": { + "type": "raster", + "scheme": "tms", + "minzoom": 8, + "maxzoom": 18, + "bounds": [ + 38.61693, + 18.564691, + 43.83016, + 23.6952 + ], + "tiles": [ + "https://qeye-tiles-prod.obs.me-east-1.myhuaweicloud.com/2/2023-08-01/{z}/{x}/{y}.png" + ] + }, + "watheer_1_project_id_61": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 46.745422050576735, + 24.51596594131675, + 46.76191981407351, + 24.525095488456188 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/watheer_1_project_id_61/{z}/{x}/{y}.png" + ] + }, + "watheer_3_project_id_62": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 46.73750468401522, + 24.50762925878948, + 46.75129332030497, + 24.516613545746253 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/watheer_3_project_id_62/{z}/{x}/{y}.png" + ] + }, + "sma_rosan": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 46.93681872420916, + 24.908261154289747, + 46.95851464507655, + 24.92992699957884 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/SMA_ROSAN/{z}/{x}/{y}.png" + ] + }, + "rakiz": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 46.62224044087486, + 24.49139443675746, + 46.62784482347615, + 24.495102396718096 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/rakiz/{z}/{x}/{y}.png" + ] + }, + "osus": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 46.85392731555123, + 24.907714868029046, + 46.865581535916135, + 24.918589500320408 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/osus/{z}/{x}/{y}.png" + ] + }, + "thenode": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 46.96335085311898, + 24.83532185522587, + 46.97267736723461, + 24.843671694476598 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/thenode/{z}/{x}/{y}.png" + ] + }, + "Alzumuruda": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 43.83640477793267, + 26.437120622829795, + 43.859762425786926, + 26.44498395270118 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/Alzumuruda/{z}/{x}/{y}.png" + ] + }, + "albsateen": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 39.6699056520946, + 24.503588462386404, + 39.681804798599956, + 24.508643396766473 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/albsateen/{z}/{x}/{y}.png" + ] + }, + "Nobla": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 39.65688411431529, + 24.37602165533194, + 39.67086681578439, + 24.380175760861277 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/Nobla/{z}/{x}/{y}.png" + ] + }, + "akhial": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 49.92420628407717, + 26.353124564993635, + 49.955896386042355, + 26.413738134738054 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/akhial/{z}/{x}/{y}.png" + ] + }, + "ajdal": { + "type": "raster", + "scheme": "tms", + "minzoom": 14, + "maxzoom": 19, + "bounds": [ + 39.26981593177834, + 21.328977740201424, + 39.28501100816402, + 21.35141759744951 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/ajdal/{z}/{x}/{y}.png" + ] + }, + "ajdal-otherSubdivisions": { + "type": "raster", + "scheme": "tms", + "minzoom": 12, + "maxzoom": 19, + "bounds": [ + 39.267628097, + 21.336002009, + 39.309957819, + 21.363523021 + ], + "tiles": [ + "https://tiles.suhail.ai/tiles/ajdal-otherSubdivisions/{z}/{x}/{y}.png" + ] + }, + "ًProject-Wareef": { + "type": "geojson", + "data": { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 39.288763311466596, + 21.34695631947318 + ] + } + } + }, + "Project-Saden": { + "type": "geojson", + "data": { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 39.25020519297186, + 21.386766172776674 + ] + } + } + }, + "regions": { + "type": "geojson", + "data": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "id": "12", + "ArabicName": "الباحة", + "EnglishName": "Al Bahah Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 41.46680824895095, + 20.15428414166781 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "15", + "ArabicName": "الجوف", + "EnglishName": "Al Jawf Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 39.54109227509982, + 29.979723799811712 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "9", + "ArabicName": "الحدود الشماليه", + "EnglishName": "Northern Borders Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 41.892302988053004, + 30.22633714333982 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "8", + "ArabicName": "حائل", + "EnglishName": "Hail Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 41.43400160571424, + 27.42673497763816 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "6", + "ArabicName": "عسير", + "EnglishName": "Asir Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 42.943519352286216, + 19.15201338425933 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "5", + "ArabicName": "الشرقية", + "EnglishName": "Eastern Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 50.16873341480063, + 23.268309751138542 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "2", + "ArabicName": "مكة المكرمة", + "EnglishName": "Makkah Al Mukarramah Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 41.26142010756434, + 21.634563628812057 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "11", + "ArabicName": "نجران", + "EnglishName": "Najran Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 46.870331850227686, + 18.449846942629176 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "13", + "ArabicName": "المدينة المنورة", + "EnglishName": "Al Madinah Al Munawwarah Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 39.580045332006314, + 24.83919847332798 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "10", + "ArabicName": "الرياض", + "EnglishName": "Riyadh" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 45.585933615839345, + 23.11687235271373 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "4", + "ArabicName": "القصيم", + "EnglishName": "Al Qassim Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 43.33063952265524, + 26.191745453894846 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "7", + "ArabicName": "تبوك", + "EnglishName": "Tabuk Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 37.241569546881514, + 27.873669253033608 + ] + } + }, + { + "type": "Feature", + "properties": { + "id": "14", + "ArabicName": "جازان", + "EnglishName": "Jazan Region" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 42.65388886783398, + 17.204688900565603 + ] + } + } + ] + } + } + }, + "source_layers": { + "landcover": { + "style_layer_count": 1, + "examples": [ + "landcover" + ] + }, + "landuse_overlay": { + "style_layer_count": 1, + "examples": [ + "national-park" + ] + }, + "landuse": { + "style_layer_count": 1, + "examples": [ + "landuse" + ] + }, + "waterway": { + "style_layer_count": 2, + "examples": [ + "waterway-shadow", + "waterway" + ] + }, + "water": { + "style_layer_count": 2, + "examples": [ + "water-shadow", + "water" + ] + }, + "structure": { + "style_layer_count": 2, + "examples": [ + "land-structure-polygon", + "land-structure-line" + ] + }, + "building": { + "style_layer_count": 2, + "examples": [ + "building-outline", + "building" + ] + }, + "road": { + "style_layer_count": 45, + "examples": [ + "tunnel-street-minor-low", + "tunnel-street-minor-case", + "tunnel-primary-secondary-tertiary-case" + ] + }, + "provinces": { + "style_layer_count": 26, + "examples": [ + "provinces_outline@riyadh", + "provinces@riyadh", + "provinces_outline@al_madenieh" + ] + }, + "neighborhoods": { + "style_layer_count": 26, + "examples": [ + "neighborhoods@riyadh", + "neighborhoods-outline@riyadh", + "neighborhoods@al_madenieh" + ] + }, + "subdivisions": { + "style_layer_count": 26, + "examples": [ + "subdivisions-fill@riyadh", + "subdivisions-outline@riyadh", + "subdivisions-fill@al_madenieh" + ] + }, + "parcels": { + "style_layer_count": 39, + "examples": [ + "parcels@riyadh", + "parcels-base@riyadh", + "parcels outline@riyadh" + ] + }, + "building_detection": { + "style_layer_count": 13, + "examples": [ + "building_detection@riyadh", + "building_detection@al_madenieh", + "building_detection@al_qassim" + ] + }, + "provinces-centroids": { + "style_layer_count": 13, + "examples": [ + "provinces-labels@riyadh", + "provinces-labels@al_madenieh", + "provinces-labels@al_qassim" + ] + }, + "neighborhoods-centroids": { + "style_layer_count": 13, + "examples": [ + "neighborhoods-labels@riyadh", + "neighborhoods-labels@al_madenieh", + "neighborhoods-labels@al_qassim" + ] + }, + "streets": { + "style_layer_count": 26, + "examples": [ + "streets-lines@riyadh", + "streets-width@riyadh", + "streets-lines@al_madenieh" + ] + }, + "dimensions": { + "style_layer_count": 13, + "examples": [ + "dimensions@riyadh", + "dimensions@al_madenieh", + "dimensions@al_qassim" + ] + }, + "parcels-centroids": { + "style_layer_count": 26, + "examples": [ + "parcel-labels-dots@riyadh", + "parcel-labels-text@riyadh", + "parcel-labels-dots@al_madenieh" + ] + }, + "sb_shape": { + "style_layer_count": 1, + "examples": [ + "sb_shape@riyadh" + ] + }, + "sb_area": { + "style_layer_count": 1, + "examples": [ + "sb_area@riyadh" + ] + }, + "qi_population_metrics": { + "style_layer_count": 26, + "examples": [ + "qi_population_metrics@riyadh", + "qi_population_metrics_outline@riyadh", + "qi_population_metrics@al_madenieh" + ] + }, + "metro_lines": { + "style_layer_count": 1, + "examples": [ + "metro_lines@riyadh" + ] + }, + "bus_lines": { + "style_layer_count": 1, + "examples": [ + "bus_lines@riyadh" + ] + }, + "metro_stations": { + "style_layer_count": 1, + "examples": [ + "metro_stations@riyadh" + ] + }, + "riyadh_bus_stations": { + "style_layer_count": 1, + "examples": [ + "riyadh_bus_stations@riyadh" + ] + }, + "non_saudi_ownership_zones": { + "style_layer_count": 26, + "examples": [ + "non_saudi_ownership_zones_fill@riyadh", + "non_saudi_ownership_zones_outline@riyadh", + "non_saudi_ownership_zones_fill@al_madenieh" + ] + }, + "mega_projects": { + "style_layer_count": 1, + "examples": [ + "mega_projects@riyadh" + ] + }, + "aeroway": { + "style_layer_count": 1, + "examples": [ + "aeroway-polygon" + ] + }, + "admin": { + "style_layer_count": 3, + "examples": [ + "admin-1-boundary", + "admin-0-boundary", + "admin-0-boundary-disputed" + ] + }, + "regions": { + "style_layer_count": 14, + "examples": [ + "regions", + "regions@riyadh", + "regions@al_madenieh" + ] + }, + "natural_label": { + "style_layer_count": 5, + "examples": [ + "waterway-label", + "natural-line-label", + "natural-point-label" + ] + }, + "poi_label": { + "style_layer_count": 1, + "examples": [ + "poi-label copy" + ] + }, + "airport_label": { + "style_layer_count": 1, + "examples": [ + "airport-label" + ] + }, + "Cities_data-11gwc6": { + "style_layer_count": 3, + "examples": [ + "settlement-minor-label _", + "settlement-major-labels", + "capital-label" + ] + }, + "place_label": { + "style_layer_count": 2, + "examples": [ + "state-label original", + "country-label_" + ] + } + } +} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/api/landMetrics__neighborhood1002969.json b/tests/fixtures/suhail_live_2026_07/api/landMetrics__neighborhood1002969.json new file mode 100644 index 0000000..617b073 --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/landMetrics__neighborhood1002969.json @@ -0,0 +1 @@ +{"data":{"totalMetricData":{"landUseGroup":"الكل","totalCount":0.0,"totalPrice":0.0,"totalCountGrowthIndicator":"DOWN","totalPriceGrowthIndicator":"DOWN","totalPriceGrowthValue":100.0,"totalCountGrowthValue":100.0,"median":8585.57142857143},"neighborhoodId":1002969,"neighborhoodName":"المرقب","provinceName":"الرياض","provinceId":101000,"landUseGroup":[{"landUseGroup":"أرض سكني","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":null,"priceOfMeter":null},"median":1929.0},{"landUseGroup":"فلل","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":null,"priceOfMeter":null},"median":3245.0},{"landUseGroup":"شقق","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":null,"priceOfMeter":null},"median":1650.0},{"landUseGroup":"أرض تجاري","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":null,"priceOfMeter":null},"median":3782.0},{"landUseGroup":"مبنى تجاري","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":null,"priceOfMeter":null},"median":7328.0},{"landUseGroup":"مبنى تجاري سكني","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":null,"priceOfMeter":null},"median":2534.0},{"landUseGroup":"محل تجزئة","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":null,"priceOfMeter":null},"median":39631.0}],"presentDate":"2026-06-30T00:00:00","pastDate":"2026-06-30T00:00:00","orgPastDate":"2025-06-30T00:00:00"},"message":null,"messageTemplate":null,"status":true,"meta":null} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/api/landMetrics_list__region10.json b/tests/fixtures/suhail_live_2026_07/api/landMetrics_list__region10.json new file mode 100644 index 0000000..8ffc8a9 --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/landMetrics_list__region10.json @@ -0,0 +1 @@ +{"data":{"items":[{"totalMetricData":{"landUseGroup":"الكل","totalCount":3.0,"totalPrice":860000.0,"totalCountGrowthIndicator":"DOWN","totalPriceGrowthIndicator":"DOWN","totalPriceGrowthValue":84.36,"totalCountGrowthValue":25.0,"median":4694.083333333333},"neighborhoodId":1003343,"neighborhoodName":"غرناطة","provinceName":"الرياض","provinceId":101000,"landUseGroup":[{"landUseGroup":"أرض سكني","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2025-10-12T00:00:00","priceOfMeter":3098.0},"median":4000.0},{"landUseGroup":"فلل","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-06-03T00:00:00","priceOfMeter":5864.0},"median":6759.0},{"landUseGroup":"شقق","totalCount":3,"totalPrice":860000.0,"lastExecutionPrice":{"transactionDate":"2026-06-02T00:00:00","priceOfMeter":6215.0},"median":5345.0},{"landUseGroup":"دور","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2025-07-10T00:00:00","priceOfMeter":6445.0},"median":0.0},{"landUseGroup":"أرض تجاري","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2024-12-29T00:00:00","priceOfMeter":7000.0},"median":3187.0},{"landUseGroup":"مبنى تجاري","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-05-06T00:00:00","priceOfMeter":6545.0},"median":8485.0},{"landUseGroup":"أرض تجاري سكني","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2025-12-21T00:00:00","priceOfMeter":6000.0},"median":6000.0},{"landUseGroup":"مبنى تجاري سكني","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-05-06T00:00:00","priceOfMeter":6545.0},"median":7939.0},{"landUseGroup":"مرافق","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2018-10-15T00:00:00","priceOfMeter":357.0},"median":357.0},{"landUseGroup":"أرض زراعي","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2021-11-08T00:00:00","priceOfMeter":2708.0},"median":2708.0},{"landUseGroup":"محل تجزئة","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2025-02-18T00:00:00","priceOfMeter":6952.0},"median":6556.0},{"landUseGroup":"أخرى","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":null,"priceOfMeter":null},"median":4993.0}],"presentDate":"2026-06-30T00:00:00","pastDate":"2026-06-30T00:00:00","orgPastDate":"2025-06-30T00:00:00"},{"totalMetricData":{"landUseGroup":"الكل","totalCount":0.0,"totalPrice":0.0,"totalCountGrowthIndicator":"DOWN","totalPriceGrowthIndicator":"UP","totalPriceGrowthValue":0.0,"totalCountGrowthValue":100.0,"median":793.2},"neighborhoodId":1004775,"neighborhoodName":"الصفراء","provinceName":"شقراء","provinceId":101009,"landUseGroup":[{"landUseGroup":"أرض سكني","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-06-14T00:00:00","priceOfMeter":265.0},"median":275.0},{"landUseGroup":"فلل","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-03-11T00:00:00","priceOfMeter":822.0},"median":822.0},{"landUseGroup":"شقق","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-06-16T00:00:00","priceOfMeter":1179.0},"median":1529.0},{"landUseGroup":"أرض تجاري","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2018-02-11T00:00:00","priceOfMeter":1311.0},"median":1311.0},{"landUseGroup":"أرض زراعي","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":null,"priceOfMeter":null},"median":29.0}],"presentDate":"2026-06-30T00:00:00","pastDate":"2026-06-30T00:00:00","orgPastDate":"2025-06-30T00:00:00"},{"totalMetricData":{"landUseGroup":"الكل","totalCount":0.0,"totalPrice":0.0,"totalCountGrowthIndicator":"DOWN","totalPriceGrowthIndicator":"DOWN","totalPriceGrowthValue":100.0,"totalCountGrowthValue":100.0,"median":9428.083333333334},"neighborhoodId":1005347,"neighborhoodName":"التعاون","provinceName":"الرياض","provinceId":101000,"landUseGroup":[{"landUseGroup":"أرض سكني","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2025-12-18T00:00:00","priceOfMeter":0.0},"median":3899.0},{"landUseGroup":"فلل","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-06-15T00:00:00","priceOfMeter":17946.0},"median":10634.0},{"landUseGroup":"شقق","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-03-09T00:00:00","priceOfMeter":9769.0},"median":6334.0},{"landUseGroup":"دور","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-03-16T00:00:00","priceOfMeter":18327.0},"median":7173.0},{"landUseGroup":"مبنى تجاري","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2025-02-05T00:00:00","priceOfMeter":14000.0},"median":14000.0},{"landUseGroup":"أرض تجاري سكني","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2025-12-24T00:00:00","priceOfMeter":0.0},"median":9300.0},{"landUseGroup":"مبنى تجاري سكني","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-06-23T00:00:00","priceOfMeter":10420.0},"median":8853.0},{"landUseGroup":"مرافق","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":null,"priceOfMeter":null},"median":11053.0},{"landUseGroup":"مبنى استراحات","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-03-09T00:00:00","priceOfMeter":19469.0},"median":19469.0},{"landUseGroup":"محل تجزئة","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2024-11-20T00:00:00","priceOfMeter":5208.0},"median":5208.0},{"landUseGroup":"أخرى","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2024-09-24T00:00:00","priceOfMeter":10366.0},"median":10357.0},{"landUseGroup":"أرض تجاري","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2025-11-09T00:00:00","priceOfMeter":0.0},"median":6857.0}],"presentDate":"2026-06-30T00:00:00","pastDate":"2026-06-30T00:00:00","orgPastDate":"2025-06-30T00:00:00"},{"totalMetricData":{"landUseGroup":"الكل","totalCount":0.0,"totalPrice":0.0,"totalCountGrowthIndicator":"UP","totalPriceGrowthIndicator":"UP","totalPriceGrowthValue":0.0,"totalCountGrowthValue":0.0,"median":726.6666666666666},"neighborhoodId":1003529,"neighborhoodName":"البدع","provinceName":"الخرج","provinceId":101002,"landUseGroup":[{"landUseGroup":"أرض سكني","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2025-10-28T00:00:00","priceOfMeter":214.0},"median":211.0},{"landUseGroup":"فلل","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2021-09-07T00:00:00","priceOfMeter":914.0},"median":914.0},{"landUseGroup":"شقق","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2014-03-23T00:00:00","priceOfMeter":1055.0},"median":1055.0}],"presentDate":"2026-06-30T00:00:00","pastDate":"2026-06-30T00:00:00","orgPastDate":"2025-06-30T00:00:00"},{"totalMetricData":{"landUseGroup":"الكل","totalCount":1.0,"totalPrice":0.0,"totalCountGrowthIndicator":"UP","totalPriceGrowthIndicator":"UP","totalPriceGrowthValue":0.0,"totalCountGrowthValue":0.0,"median":1020.0},"neighborhoodId":1004855,"neighborhoodName":"العقيق","provinceName":"وادى الدواسر","provinceId":101006,"landUseGroup":[{"landUseGroup":"أرض سكني","totalCount":1,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2026-06-30T00:00:00","priceOfMeter":1600.0},"median":120.0},{"landUseGroup":"فلل","totalCount":0,"totalPrice":0.0,"lastExecutionPrice":{"transactionDate":"2025-10-29T00:00:00","priceOfMeter":940.0},"median":1920.0}],"presentDate":"2026-06-30T00:00:00","pastDate":"2026-06-30T00:00:00","orgPastDate":"2025-06-30T00:00:00"}],"count":5,"totalCount":764,"offset":0,"limit":5},"message":null,"messageTemplate":null,"status":true,"meta":null} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/api/landZoningGroups.json b/tests/fixtures/suhail_live_2026_07/api/landZoningGroups.json new file mode 100644 index 0000000..a2982c5 --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/landZoningGroups.json @@ -0,0 +1 @@ +{"data":[{"id":3,"group":"سكني","color":"f1ce63","usageCount":4263782},{"id":2,"group":"شقق 3.5 دور/ تجاري","color":"fa6d6d","usageCount":719427},{"id":5,"group":"تجاري","color":"e74545","usageCount":334009},{"id":9,"group":"مرافق","color":"bab0ac","usageCount":325100},{"id":1,"group":"شقق 2.5 دور","color":"f28e2b","usageCount":297994},{"id":16,"group":"مناطق مفتوحة","color":"a0cbe8","usageCount":211139},{"id":14,"group":"زراعي","color":"8cd17d","usageCount":87713},{"id":8,"group":"صناعي","color":"fabfd2","usageCount":73134},{"id":10,"group":"حكومي","color":"305d8a","usageCount":56812},{"id":6,"group":"مستودعات","color":"b07aa1","usageCount":32993},{"id":12,"group":"مشاريع","color":"9d7660","usageCount":22086},{"id":7,"group":"تعليمي","color":"027b8e","usageCount":20661},{"id":11,"group":"استراحات","color":"d7b5a6","usageCount":16598},{"id":15,"group":"أودية","color":"59a14f","usageCount":14135},{"id":4,"group":"أعصاب تجارية","color":"ae123a","usageCount":6327},{"id":13,"group":"مواصلات","color":"a0cbe8","usageCount":3158},{"id":19,"group":"SB-T4.1","color":"BF7FFF","usageCount":1199},{"id":17,"group":"SB-T3.1","color":"FFFF7F","usageCount":1022},{"id":24,"group":"SB-T5.3","color":"FF0000","usageCount":768},{"id":18,"group":"SB-T3.2","color":"FFBF00","usageCount":600},{"id":30,"group":"SB-غير محدد","color":"F5F4F2","usageCount":477},{"id":23,"group":"SB-T5.2","color":"FF7F00","usageCount":369},{"id":20,"group":"SB-T4.2","color":"6418AB","usageCount":299},{"id":27,"group":"SB-Services","color":"7FBFFF","usageCount":196},{"id":21,"group":"SB-T5.1","color":"FF4DD8","usageCount":129},{"id":25,"group":"SB-T5.3-TOD","color":"970000","usageCount":74},{"id":28,"group":"SB-Community Residential","color":"00A884","usageCount":66},{"id":22,"group":"SB-T5.1-TOD","color":"B2008B","usageCount":21},{"id":26,"group":"SB-T6","color":"3C0000","usageCount":15},{"id":29,"group":"SB-Subject to Subdivision","color":"FCBCB6","usageCount":3}],"message":null,"messageTemplate":null,"status":true,"meta":null} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/api/parcel_detail__9858274.json b/tests/fixtures/suhail_live_2026_07/api/parcel_detail__9858274.json new file mode 100644 index 0000000..6783250 --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/parcel_detail__9858274.json @@ -0,0 +1 @@ +{"data":{"geometry":{"type":"Polygon","coordinates":[[[46.722032207,24.636894502],[46.722121462,24.636876956],[46.722113432,24.636831632],[46.722106602,24.636801828],[46.7220187,24.63683877],[46.722032207,24.636894502]]]},"properties":{"count":1,"parcelObjectId":9858274,"neighborhoodId":"1002969","neighborhoodName":"المرقب","municipalityCodeId":"1012","municipalityName":"الرياض","blockNo":"0","landUseDetailed":null,"landUseGroup":null,"parcelId":"9858274","parcelNo":null,"shapeArea":69.72,"subdivisionId":"1045983","subdivisionNo":"بدون","shapeLen":null,"regionId":10,"provinceId":101000,"landUseGroupCode":null,"dimensionsCount":0,"centroid":{"x":46.72207203992842,"y":24.63685211778096},"regionName":"الرياض","zoningColor":"e74545"}},"message":null,"messageTemplate":null,"status":true,"meta":null} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/api/priceOfMeter__9858274.json b/tests/fixtures/suhail_live_2026_07/api/priceOfMeter__9858274.json new file mode 100644 index 0000000..1cff983 --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/priceOfMeter__9858274.json @@ -0,0 +1 @@ +{"data":[{"parcelObjId":9858274,"neighborhoodId":1002969,"neighborhoodMetrics":[{"neighborhoodId":1002969,"month":1,"year":2026,"metricsType":"أرض سكني","avaragePriceOfMeter":1796.0},{"neighborhoodId":1002969,"month":1,"year":2026,"metricsType":"مبنى تجاري سكني","avaragePriceOfMeter":1911.0},{"neighborhoodId":1002969,"month":1,"year":2026,"metricsType":"محل تجزئة","avaragePriceOfMeter":35676.0},{"neighborhoodId":1002969,"month":1,"year":2026,"metricsType":"أرض تجاري","avaragePriceOfMeter":3782.0},{"neighborhoodId":1002969,"month":1,"year":2026,"metricsType":"شقق","avaragePriceOfMeter":1650.0},{"neighborhoodId":1002969,"month":1,"year":2026,"metricsType":"فلل","avaragePriceOfMeter":4375.0},{"neighborhoodId":1002969,"month":1,"year":2026,"metricsType":"مبنى تجاري","avaragePriceOfMeter":7328.0},{"neighborhoodId":1002969,"month":2,"year":2026,"metricsType":"فلل","avaragePriceOfMeter":4337.0},{"neighborhoodId":1002969,"month":2,"year":2026,"metricsType":"أرض تجاري","avaragePriceOfMeter":3782.0},{"neighborhoodId":1002969,"month":2,"year":2026,"metricsType":"مبنى تجاري","avaragePriceOfMeter":7328.0},{"neighborhoodId":1002969,"month":2,"year":2026,"metricsType":"مبنى تجاري سكني","avaragePriceOfMeter":1911.0},{"neighborhoodId":1002969,"month":2,"year":2026,"metricsType":"محل تجزئة","avaragePriceOfMeter":39631.0},{"neighborhoodId":1002969,"month":2,"year":2026,"metricsType":"شقق","avaragePriceOfMeter":1650.0},{"neighborhoodId":1002969,"month":2,"year":2026,"metricsType":"أرض سكني","avaragePriceOfMeter":1554.0},{"neighborhoodId":1002969,"month":3,"year":2026,"metricsType":"أرض تجاري","avaragePriceOfMeter":3782.0},{"neighborhoodId":1002969,"month":3,"year":2026,"metricsType":"أرض سكني","avaragePriceOfMeter":1271.0},{"neighborhoodId":1002969,"month":3,"year":2026,"metricsType":"مبنى تجاري","avaragePriceOfMeter":7328.0},{"neighborhoodId":1002969,"month":3,"year":2026,"metricsType":"محل تجزئة","avaragePriceOfMeter":39631.0},{"neighborhoodId":1002969,"month":3,"year":2026,"metricsType":"فلل","avaragePriceOfMeter":4006.0},{"neighborhoodId":1002969,"month":3,"year":2026,"metricsType":"شقق","avaragePriceOfMeter":1650.0},{"neighborhoodId":1002969,"month":3,"year":2026,"metricsType":"مبنى تجاري سكني","avaragePriceOfMeter":2067.0},{"neighborhoodId":1002969,"month":4,"year":2026,"metricsType":"مبنى تجاري","avaragePriceOfMeter":7328.0},{"neighborhoodId":1002969,"month":4,"year":2026,"metricsType":"أرض سكني","avaragePriceOfMeter":1430.0},{"neighborhoodId":1002969,"month":4,"year":2026,"metricsType":"أرض تجاري","avaragePriceOfMeter":3782.0},{"neighborhoodId":1002969,"month":4,"year":2026,"metricsType":"شقق","avaragePriceOfMeter":1650.0},{"neighborhoodId":1002969,"month":4,"year":2026,"metricsType":"محل تجزئة","avaragePriceOfMeter":39631.0},{"neighborhoodId":1002969,"month":4,"year":2026,"metricsType":"مبنى تجاري سكني","avaragePriceOfMeter":2222.0},{"neighborhoodId":1002969,"month":4,"year":2026,"metricsType":"فلل","avaragePriceOfMeter":3159.0},{"neighborhoodId":1002969,"month":5,"year":2026,"metricsType":"محل تجزئة","avaragePriceOfMeter":39631.0},{"neighborhoodId":1002969,"month":5,"year":2026,"metricsType":"شقق","avaragePriceOfMeter":1650.0},{"neighborhoodId":1002969,"month":5,"year":2026,"metricsType":"فلل","avaragePriceOfMeter":3181.0},{"neighborhoodId":1002969,"month":5,"year":2026,"metricsType":"أرض تجاري","avaragePriceOfMeter":3782.0},{"neighborhoodId":1002969,"month":5,"year":2026,"metricsType":"أرض سكني","avaragePriceOfMeter":1687.0},{"neighborhoodId":1002969,"month":5,"year":2026,"metricsType":"مبنى تجاري","avaragePriceOfMeter":7328.0},{"neighborhoodId":1002969,"month":5,"year":2026,"metricsType":"مبنى تجاري سكني","avaragePriceOfMeter":2378.0},{"neighborhoodId":1002969,"month":6,"year":2026,"metricsType":"أرض سكني","avaragePriceOfMeter":1929.0},{"neighborhoodId":1002969,"month":6,"year":2026,"metricsType":"شقق","avaragePriceOfMeter":1650.0},{"neighborhoodId":1002969,"month":6,"year":2026,"metricsType":"مبنى تجاري","avaragePriceOfMeter":7328.0},{"neighborhoodId":1002969,"month":6,"year":2026,"metricsType":"فلل","avaragePriceOfMeter":3245.0},{"neighborhoodId":1002969,"month":6,"year":2026,"metricsType":"أرض تجاري","avaragePriceOfMeter":3782.0},{"neighborhoodId":1002969,"month":6,"year":2026,"metricsType":"مبنى تجاري سكني","avaragePriceOfMeter":2534.0},{"neighborhoodId":1002969,"month":6,"year":2026,"metricsType":"محل تجزئة","avaragePriceOfMeter":39631.0}],"parcelMetrics":[],"from":"2026-01-15T00:00:00+00:00","to":"2026-07-15T23:59:59.9999999+00:00","groupingType":0}],"message":null,"messageTemplate":null,"status":true,"meta":null} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/api/regions.json b/tests/fixtures/suhail_live_2026_07/api/regions.json new file mode 100644 index 0000000..5ef43c7 --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/regions.json @@ -0,0 +1 @@ +{"data":[{"orderIndex":1,"id":10,"key":"Riyadh","name":"الرياض","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/Riyadh/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":46.6893526894509,"y":24.6787559510409},"restrictBoundaryBox":{"southwest":{"x":41.686026,"y":19.208334},"northeast":{"x":48.1838,"y":27.7021}},"provinces":[{"id":101007,"name":"الأفلاج","centroid":{"x":46.715444267115785,"y":22.295073722206904}},{"id":101018,"name":"الحريق","centroid":{"x":46.502570981041295,"y":23.636735246847998}},{"id":101002,"name":"الخرج","centroid":{"x":47.34136889971158,"y":23.995710471370597}},{"id":101001,"name":"الدرعية","centroid":{"x":46.56910567089585,"y":24.74693834073349}},{"id":101021,"name":"الدلم","centroid":{"x":47.112904782275336,"y":24.01763887722338}},{"id":101003,"name":"الدوادمي","centroid":{"x":44.397866882388435,"y":24.538975871157785}},{"id":101000,"name":"الرياض","centroid":{"x":46.688852973932114,"y":24.67893932829408}},{"id":101022,"name":"الرين","centroid":{"x":45.511691182826006,"y":23.54122232345389}},{"id":101008,"name":"الزلفي","centroid":{"x":44.814681416938626,"y":26.313379655035813}},{"id":101012,"name":"السليل","centroid":{"x":45.59169373166689,"y":20.45582409087714}},{"id":101019,"name":"الغاط","centroid":{"x":44.937938596442095,"y":26.0301475625097}},{"id":101005,"name":"القويعية","centroid":{"x":45.2874711392253,"y":24.082317945487265}},{"id":101004,"name":"المجمعة","centroid":{"x":45.37596291402336,"y":25.90337465924469}},{"id":101014,"name":"المزاحمية","centroid":{"x":46.27978313722945,"y":24.46778261162726}},{"id":101016,"name":"ثادق","centroid":{"x":45.86409457237059,"y":25.276020650951917}},{"id":101017,"name":"حريملاء","centroid":{"x":46.19546673984374,"y":25.1036160175749}},{"id":101010,"name":"حوطة بني تميم","centroid":{"x":46.862398717238506,"y":23.458690676328878}},{"id":101015,"name":"رماح","centroid":{"x":47.17701555395518,"y":25.520706416692203}},{"id":101009,"name":"شقراء","centroid":{"x":45.24554681390492,"y":25.20184909171238}},{"id":101013,"name":"ضرما","centroid":{"x":46.12354756720016,"y":24.595259960213333}},{"id":101011,"name":"عفيف","centroid":{"x":42.902788904750615,"y":23.893567294444367}},{"id":101020,"name":"مرات","centroid":{"x":45.44699751302853,"y":25.0727180843136}},{"id":101006,"name":"وادي الدواسر","centroid":{"x":44.77088060208422,"y":20.4668318509133}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_riyadh.png","mapKey":"riyadh"},{"orderIndex":2,"id":13,"key":"Madinah","name":"المدينة المنورة","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/Madinah/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":39.609449138409516,"y":24.4802585936543},"restrictBoundaryBox":{"southwest":{"x":37.084,"y":22.51971},"northeast":{"x":42.1566,"y":27.46665}},"provinces":[{"id":131006,"name":"الحناكية","centroid":{"x":40.51657831063046,"y":24.8642071449309}},{"id":131002,"name":"العلا","centroid":{"x":37.94356087702908,"y":26.5861347459765}},{"id":131008,"name":"العيص","centroid":{"x":38.131109968316636,"y":25.06525120683267}},{"id":131000,"name":"المدينة المنورة","centroid":{"x":39.609449138409516,"y":24.4802585936543}},{"id":131003,"name":"المهد","centroid":{"x":40.889122019456664,"y":23.49353891879711}},{"id":131004,"name":"بدر","centroid":{"x":38.779041415508075,"y":23.787342410568133}},{"id":131005,"name":"خيبر","centroid":{"x":39.290025992756604,"y":25.659775349708003}},{"id":131007,"name":"وادي الفرع","centroid":{"x":39.851506676633186,"y":23.485539728709643}},{"id":131001,"name":"ينبع","centroid":{"x":38.05797682625832,"y":24.079794141492023}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_al_madenieh.png","mapKey":"al_madenieh"},{"orderIndex":3,"id":5,"key":"Eastern","name":"الشرقية","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/Eastern/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":50.0826958589995,"y":26.3891891271042},"restrictBoundaryBox":{"southwest":{"x":44.670664,"y":19.000517},"northeast":{"x":55.666667,"y":29.141386}},"provinces":[{"id":51001,"name":"الاحساء","centroid":{"x":49.59577107516184,"y":25.383500231099365}},{"id":51003,"name":"الجبيل","centroid":{"x":49.65606510275657,"y":26.982316570147656}},{"id":51005,"name":"الخبر","centroid":{"x":50.211115516864645,"y":26.30970252185347}},{"id":51006,"name":"الخفجي","centroid":{"x":48.48192522796914,"y":28.463416934809995}},{"id":51000,"name":"الدمام","centroid":{"x":50.10442741114076,"y":26.429026459390794}},{"id":51011,"name":"العديد","centroid":{"x":50.75181025047487,"y":24.749502958660766}},{"id":51004,"name":"القطيف","centroid":{"x":49.99630637603599,"y":26.565806082113713}},{"id":51009,"name":"النعيرية","centroid":{"x":48.4829118711081,"y":27.477683589204393}},{"id":51008,"name":"بقيق","centroid":{"x":49.4284287303845,"y":25.938832776755888}},{"id":51002,"name":"حفر الباطن","centroid":{"x":45.97727327000438,"y":28.420220242112197}},{"id":51007,"name":"راس تنورة","centroid":{"x":50.060378872778955,"y":26.705245012675014}},{"id":51010,"name":"قرية العليا","centroid":{"x":47.69984164367187,"y":27.562538732841663}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_eastern.png","mapKey":"eastern_region"},{"orderIndex":4,"id":2,"key":"Makkah","name":"مكة المكرمة","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/Makkah/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":39.85532681763745,"y":21.416090070987288},"restrictBoundaryBox":{"southwest":{"x":38.61693,"y":18.564691},"northeast":{"x":43.83016,"y":23.6952}},"provinces":[{"id":21016,"name":"اضم","centroid":{"x":40.8573604331748,"y":20.405927014059944}},{"id":21006,"name":"الجموم","centroid":{"x":39.26189264188951,"y":21.53022290317747}},{"id":21009,"name":"الخرمة","centroid":{"x":42.02270023593635,"y":21.91973368982235}},{"id":21002,"name":"الطائف","centroid":{"x":40.42346015605826,"y":21.296458731062103}},{"id":21015,"name":"العرضيات","centroid":{"x":41.833888013401214,"y":19.44621834918492}},{"id":21003,"name":"القنفذة","centroid":{"x":41.07147201876349,"y":19.148772315311973}},{"id":21008,"name":"الكامل","centroid":{"x":39.79048612455639,"y":22.259311778939438}},{"id":21004,"name":"الليث","centroid":{"x":40.27025241468968,"y":20.1411886486247}},{"id":61011,"name":"المجاردة","centroid":{"x":41.865993579176184,"y":19.121714104804187}},{"id":21013,"name":"المويه","centroid":{"x":41.76826356431981,"y":22.439923604335213}},{"id":21012,"name":"بحرة","centroid":{"x":39.1724994666939,"y":21.307587867592186}},{"id":21011,"name":"تربة","centroid":{"x":41.596702748629845,"y":21.192246328487087}},{"id":21001,"name":"جدة","centroid":{"x":39.183720837472364,"y":21.571800029722173}},{"id":21007,"name":"خليص","centroid":{"x":39.28891822888672,"y":22.187204718011646}},{"id":21005,"name":"رابغ","centroid":{"x":39.05236175347281,"y":22.777604071985326}},{"id":21010,"name":"رنية","centroid":{"x":42.852642292507795,"y":21.223368906820063}},{"id":21000,"name":"مكة المكرمة","centroid":{"x":39.85532681763745,"y":21.416090070987288}},{"id":21014,"name":"ميسان","centroid":{"x":41.04610880762626,"y":20.598715729130458}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_makkah.png","mapKey":"makkah_region"},{"orderIndex":5,"id":4,"key":"Al_Qassim","name":"القصيم","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/Al_Qassim/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":43.957183957099915,"y":26.364278487136083},"restrictBoundaryBox":{"southwest":{"x":40.16052246093751,"y":24.647017162630366},"northeast":{"x":47.83447265625001,"y":28.420391085674304}},"provinces":[{"id":41006,"name":"الاسياح","centroid":{"x":44.19411596528751,"y":26.796570778782762}},{"id":41005,"name":"البدائع","centroid":{"x":43.73896131460193,"y":26.012161858095755}},{"id":41004,"name":"البكيرية","centroid":{"x":43.267442642223976,"y":26.446151972145547}},{"id":41002,"name":"الرس","centroid":{"x":43.510488338771516,"y":25.828780295055964}},{"id":41010,"name":"الشماسية","centroid":{"x":44.235965710446244,"y":26.324440867307686}},{"id":41003,"name":"المذنب","centroid":{"x":44.19926481864793,"y":25.833157147038733}},{"id":41007,"name":"النبهانية","centroid":{"x":43.08124616814241,"y":25.86463411301878}},{"id":41000,"name":"بريدة","centroid":{"x":43.921643932903464,"y":26.34492808551673}},{"id":41009,"name":"رياض الخبراء","centroid":{"x":43.57888091087288,"y":26.058804533476547}},{"id":41012,"name":"ضرية","centroid":{"x":42.91905836426816,"y":24.706010474581884}},{"id":41011,"name":"عقلة الصقور","centroid":{"x":42.18389411280304,"y":25.831244484930647}},{"id":41001,"name":"عنيزة","centroid":{"x":43.971238464222424,"y":26.097600227672075}},{"id":41008,"name":"عيون الجواء","centroid":{"x":43.4611711451123,"y":26.793640979998035}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/al_qassim.png","mapKey":"al_qassim"},{"orderIndex":6,"id":6,"key":"Asir","name":"عسير","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/Asir/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":42.751242220401764,"y":18.312010011422437},"restrictBoundaryBox":{"southwest":{"x":41.214223,"y":17.366667},"northeast":{"x":44.440789,"y":20.80266}},"provinces":[{"id":61000,"name":"أبها","centroid":{"x":42.549664742783314,"y":18.307064008976994}},{"id":61008,"name":"احد رفيدة","centroid":{"x":42.82098314283127,"y":18.177092982976824}},{"id":61017,"name":"الأمواه","centroid":{"x":43.87514642512652,"y":18.740495129997793}},{"id":61013,"name":"البرك","centroid":{"x":41.536570923666375,"y":18.213246458034526}},{"id":61016,"name":"الحرجة","centroid":{"x":43.50889141359282,"y":18.020403160695604}},{"id":61003,"name":"النماص","centroid":{"x":42.11433535166822,"y":19.03914373344304}},{"id":61014,"name":"بارق","centroid":{"x":41.9988836819683,"y":18.86004591072215}},{"id":61010,"name":"بلقرن","centroid":{"x":41.91800746530642,"y":19.75900691740014}},{"id":61002,"name":"بيشة","centroid":{"x":42.602690648252754,"y":19.995273313792758}},{"id":61006,"name":"تثليث","centroid":{"x":43.54107035892252,"y":19.525144076905562}},{"id":61015,"name":"تنومة","centroid":{"x":42.22089826469791,"y":18.94781809757498}},{"id":61001,"name":"خميس مشيط","centroid":{"x":42.6975358196979,"y":18.291703604360137}},{"id":61007,"name":"رجال المع","centroid":{"x":42.21967966192891,"y":18.08920475088153}},{"id":61005,"name":"سراة عبيدة","centroid":{"x":43.13902343975601,"y":18.0760491458154}},{"id":61012,"name":"طريب","centroid":{"x":43.278227244781945,"y":18.8262092658174}},{"id":61009,"name":"ظهران الجنوب","centroid":{"x":43.51838292745236,"y":17.6977735812593}},{"id":61004,"name":"محايل","centroid":{"x":41.68044981211372,"y":18.00469926392769}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_asir.png","mapKey":"asir_region"},{"orderIndex":7,"id":7,"key":"Tabuk","name":"تبوك","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/Tabuk/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":36.511958188017736,"y":28.42458682342848},"restrictBoundaryBox":{"southwest":{"x":34.571234,"y":24.560854},"northeast":{"x":39.923088,"y":29.976269}},"provinces":[{"id":71006,"name":"البدع","centroid":{"x":35.007850585097515,"y":28.410698964086606}},{"id":71001,"name":"الوجه","centroid":{"x":36.46032229508487,"y":26.250400062144085}},{"id":71004,"name":"املج","centroid":{"x":37.261712825672056,"y":25.1730070755383}},{"id":71000,"name":"تبوك","centroid":{"x":36.53308641826973,"y":28.449591716298567}},{"id":71003,"name":"تيماء","centroid":{"x":38.535876904844095,"y":27.630984950384846}},{"id":71005,"name":"حقل","centroid":{"x":34.9462924874948,"y":29.285138949073385}},{"id":71002,"name":"ضباء","centroid":{"x":35.70859108962764,"y":27.34335564622256}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_asir.png","mapKey":"tabuk"},{"orderIndex":8,"id":8,"key":"Hail","name":"حائل","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/Hail/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":41.689419447347575,"y":27.491964102762882},"restrictBoundaryBox":{"southwest":{"x":39.11418,"y":25.25439},"northeast":{"x":44.367661,"y":29.207876}},"provinces":[{"id":81007,"name":"الحائط","centroid":{"x":40.46578242082372,"y":25.98703710916808}},{"id":81008,"name":"السليمي","centroid":{"x":41.34244654660869,"y":26.29308308387064}},{"id":81004,"name":"الشملي","centroid":{"x":40.33697319222971,"y":26.862396498652146}},{"id":81003,"name":"الشنان","centroid":{"x":43.013260630692415,"y":27.059496932904498}},{"id":81002,"name":"الغزالة","centroid":{"x":41.3265875,"y":26.835137}},{"id":81001,"name":"بقعاء","centroid":{"x":42.38492247535399,"y":27.900037626621263}},{"id":81000,"name":"حائل","centroid":{"x":41.673729167023005,"y":27.521521476888704}},{"id":81006,"name":"سميراء","centroid":{"x":42.12425968543399,"y":26.492506484338392}},{"id":81005,"name":"موقق","centroid":{"x":41.19453402172998,"y":27.37549978942258}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_asir.png","mapKey":"hail"},{"orderIndex":9,"id":9,"key":"Northern_Borders","name":"الحدود الشماليه","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/northern_borders/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":41.027417444038136,"y":30.97156581394864},"restrictBoundaryBox":{"southwest":{"x":37.955253,"y":27.831761},"northeast":{"x":45.464141,"y":32.153962}},"provinces":[{"id":91003,"name":"العويقيلة","centroid":{"x":42.25704811586929,"y":30.357165313035562}},{"id":91001,"name":"رفحاء","centroid":{"x":43.5293616629359,"y":29.644792096538175}},{"id":91002,"name":"طريف","centroid":{"x":38.65499453290945,"y":31.664196214020752}},{"id":91000,"name":"عرعر","centroid":{"x":40.9678093410463,"y":30.98960922384898}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_asir.png","mapKey":"northern_borders"},{"orderIndex":10,"id":11,"key":"Najran","name":"نجران","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/najran/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":44.3440073133832,"y":17.551953704242568},"restrictBoundaryBox":{"southwest":{"x":43.59569,"y":16.95},"northeast":{"x":52.001753,"y":19.448889}},"provinces":[{"id":111003,"name":"بدر الجنوب","centroid":{"x":43.7160905,"y":17.920384}},{"id":111005,"name":"ثار","centroid":{"x":44.10606544834801,"y":17.97590597058463}},{"id":111002,"name":"حبونا","centroid":{"x":44.03995496657999,"y":17.853508774330376}},{"id":111006,"name":"خباش","centroid":{"x":45.76316739398078,"y":18.043890107398482}},{"id":111001,"name":"شرورة","centroid":{"x":47.110595319765615,"y":17.486947637153918}},{"id":111000,"name":"نجران","centroid":{"x":44.19306708534653,"y":17.5244436577356}},{"id":111004,"name":"يدمه","centroid":{"x":44.22198058430397,"y":18.581691532585744}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_asir.png","mapKey":"najran"},{"orderIndex":11,"id":12,"key":"Al_Bahah","name":"الباحة","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/bahah/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":41.464232148061186,"y":20.02593462771641},"restrictBoundaryBox":{"southwest":{"x":40.829164,"y":19.44581},"northeast":{"x":42.129716,"y":20.82938}},"provinces":[{"id":121000,"name":"الباحة","centroid":{"x":41.464232148061186,"y":20.02593462771641}},{"id":121009,"name":"الحجرة","centroid":{"x":41.07889422235841,"y":20.203198335521904}},{"id":121004,"name":"العقيق","centroid":{"x":41.646822460764284,"y":20.257462945219757}},{"id":121006,"name":"القرى","centroid":{"x":41.36807751961936,"y":20.273898326750476}},{"id":121003,"name":"المخواة","centroid":{"x":41.4215861725123,"y":19.76801799923247}},{"id":121002,"name":"المندق","centroid":{"x":41.302872770021345,"y":20.157605998565842}},{"id":121001,"name":"بلجرشي","centroid":{"x":41.583743988060505,"y":19.88923812003347}},{"id":121007,"name":"بني حسن","centroid":{"x":41.36295240312482,"y":20.072541496848913}},{"id":121008,"name":"غامد الزناد","centroid":{"x":41.55353073568331,"y":19.67415984632521}},{"id":121005,"name":"قلوة","centroid":{"x":41.238302558558864,"y":19.942758872627127}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_asir.png","mapKey":"bahah"},{"orderIndex":12,"id":14,"key":"Jazan","name":"جازان","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/jazan/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":42.58500358308977,"y":16.897199694154573},"restrictBoundaryBox":{"southwest":{"x":41.38948,"y":16.08843},"northeast":{"x":43.3365,"y":18.060398}},"provinces":[{"id":141002,"name":"ابو عريش","centroid":{"x":42.81415083449986,"y":16.965814097274002}},{"id":141010,"name":"احد المسارحة","centroid":{"x":42.96521968961733,"y":16.703645038957486}},{"id":141004,"name":"الحرث","centroid":{"x":43.141599637627436,"y":16.812003280506644}},{"id":141009,"name":"الدائر","centroid":{"x":43.13836719047619,"y":17.332664542857145}},{"id":141013,"name":"الدرب","centroid":{"x":42.216780052949744,"y":17.725161436784678}},{"id":141006,"name":"الريث","centroid":{"x":42.707976,"y":17.663274}},{"id":141014,"name":"الطوال","centroid":{"x":42.894743000000005,"y":16.4548305}},{"id":141012,"name":"العارضة","centroid":{"x":43.114194499999996,"y":16.937821499999995}},{"id":141011,"name":"العيدابي","centroid":{"x":43.025211500000005,"y":17.249993000000003}},{"id":141007,"name":"بيش","centroid":{"x":42.49249708442299,"y":17.377022481142557}},{"id":141000,"name":"جازان","centroid":{"x":42.58500358308977,"y":16.897199694154573}},{"id":141003,"name":"صامطة","centroid":{"x":42.95180465241114,"y":16.62018126613816}},{"id":141001,"name":"صبيا","centroid":{"x":42.75116137352931,"y":17.15508399778257}},{"id":141005,"name":"ضمد","centroid":{"x":42.74983642491161,"y":17.11062934737523}},{"id":141008,"name":"فرسان","centroid":{"x":42.12070190012882,"y":16.696307743297094}},{"id":141016,"name":"فيفاء","centroid":{"x":43.07931106749507,"y":17.313763009884216}},{"id":141015,"name":"هروب","centroid":{"x":42.88615891838771,"y":17.362284283847153}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_asir.png","mapKey":"jazan"},{"orderIndex":13,"id":15,"key":"Al_Jawf","name":"الجوف","mapStyleUrl":"https://tiles.suhail.ai/gl-styles/ksa.json","mapZoomLevel":10.0,"metricsUrl":"/jawf/mobile-metrics","defaultTransactionsDateRange":"-1/y","centroid":{"x":40.203656830089905,"y":29.949845524050016},"restrictBoundaryBox":{"southwest":{"x":37.005875,"y":28.36794},"northeast":{"x":41.99684,"y":31.749873}},"provinces":[{"id":151001,"name":"القريات","centroid":{"x":37.33710356706453,"y":31.31739936265569}},{"id":151002,"name":"دومة الجندل","centroid":{"x":39.87360533717283,"y":29.814014711960713}},{"id":151000,"name":"سكاكا","centroid":{"x":40.23630832398995,"y":29.988216049720492}},{"id":151003,"name":"طبرجل","centroid":{"x":38.190265696126765,"y":30.494012618844177}}],"image":"https://suhailapp.obs.me-east-1.myhuaweicloud.com/static-files/region_asir.png","mapKey":"jawf"}],"message":null,"messageTemplate":null,"status":true,"meta":null} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/api/settings_app.json b/tests/fixtures/suhail_live_2026_07/api/settings_app.json new file mode 100644 index 0000000..ac60c2f --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/settings_app.json @@ -0,0 +1 @@ +{"AnonUserRestrictionsEnabled":true,"AnonUserSessionTimeSec":900,"AnonUserParcelClicks":15,"TransactionsFilterTypes":[{"Display":"الكل","Value":null},{"Display":"أرضسكني","Value":"أرضسكني"},{"Display":"فلل","Value":"فلل"},{"Display":"أرضتجاري","Value":"أرضتجاري"},{"Display":"شقق","Value":"شقق"}],"AuthVerificationCodeTimeoutSeconds":300,"Reports":{"ParcelReportUrl":"https://reports.suhail.ai/api/report/Parcel","OrderReportUrl":"https://reports.suhail.ai/api/report/Order"},"EnforceOfferVerification":false,"EnableNafath":true,"VerifyAccountDismissHoursTimeout":24,"Map":{"MaxZoom":20,"MinZoom":9.4,"MaxBounds":[[46.136283976881685,24.285527825347828],[47.48678856414921,25.191409011097292]]},"IsUserTrackingEnabled":true,"SegmentWriteKeyAndroid":"UfVKIEDDpKxJ37CXxuFpSsOybierN7HW","SegmentWriteKeyIOS":"wToqB5JFsnJLNQ1znbFc9xqdq55uAHIQ","SegmentWriteKeyWeb":"","NewsFeedFilterMinDate":"2020-08-01","UnitStatusColors":[{"Status":"available","Color":"#4EA96F"},{"Status":"unavailable","Color":"#556E7A"},{"Status":"booked","Color":"#DC9B36"},{"Status":"confirmedbooking","Color":"#DC9B36"},{"Status":"sold","Color":"#e63946"},{"Status":"default","Color":"#556E7A"},null],"realEstateTaxPercentage":5.0} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/api/tiles_modes.json b/tests/fixtures/suhail_live_2026_07/api/tiles_modes.json new file mode 100644 index 0000000..99dbe01 --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/tiles_modes.json @@ -0,0 +1,399 @@ +{ + "zoning_color": { + "label": "استخدام الأراضي ", + "label_en": "Zoning codes", + "fill-color": [ + "case", + [ + "==", + [ + "get", + "zoning_id" + ], + 1 + ], + "#f28e2b", + [ + "==", + [ + "get", + "zoning_id" + ], + 2 + ], + "#fa6d6d", + [ + "==", + [ + "get", + "zoning_id" + ], + 3 + ], + "#f1ce63", + [ + "==", + [ + "get", + "zoning_id" + ], + 4 + ], + "#ae123a", + [ + "==", + [ + "get", + "zoning_id" + ], + 5 + ], + "#e74545", + [ + "==", + [ + "get", + "zoning_id" + ], + 6 + ], + "#b07aa1", + [ + "==", + [ + "get", + "zoning_id" + ], + 7 + ], + "#027b8e", + [ + "==", + [ + "get", + "zoning_id" + ], + 8 + ], + "#fabfd2", + [ + "==", + [ + "get", + "zoning_id" + ], + 9 + ], + "#bab0ac", + [ + "==", + [ + "get", + "zoning_id" + ], + 10 + ], + "#305d8a", + [ + "==", + [ + "get", + "zoning_id" + ], + 11 + ], + "#d7b5a6", + [ + "==", + [ + "get", + "zoning_id" + ], + 12 + ], + "#9d7660", + [ + "==", + [ + "get", + "zoning_id" + ], + 13 + ], + "#a0cbe8", + [ + "==", + [ + "get", + "zoning_id" + ], + 14 + ], + "#8cd17d", + [ + "==", + [ + "get", + "zoning_id" + ], + 15 + ], + "#59a14f", + [ + "==", + [ + "get", + "zoning_id" + ], + 16 + ], + "#a0cbe8", + [ + "==", + [ + "get", + "zoning_id" + ], + 17 + ], + "#FFFF7F", + [ + "==", + [ + "get", + "zoning_id" + ], + 18 + ], + "#FFBF00", + [ + "==", + [ + "get", + "zoning_id" + ], + 19 + ], + "#BF7FFF", + [ + "==", + [ + "get", + "zoning_id" + ], + 20 + ], + "#6418AB", + [ + "==", + [ + "get", + "zoning_id" + ], + 21 + ], + "#FF4DD8", + [ + "==", + [ + "get", + "zoning_id" + ], + 22 + ], + "#B2008B", + [ + "==", + [ + "get", + "zoning_id" + ], + 23 + ], + "#FF7F00", + [ + "==", + [ + "get", + "zoning_id" + ], + 24 + ], + "#FF0000", + [ + "==", + [ + "get", + "zoning_id" + ], + 25 + ], + "#970000", + [ + "==", + [ + "get", + "zoning_id" + ], + 26 + ], + "#3C0000", + [ + "==", + [ + "get", + "zoning_id" + ], + 27 + ], + "#7FBFFF", + [ + "==", + [ + "get", + "zoning_id" + ], + 28 + ], + "#00A884", + [ + "==", + [ + "get", + "zoning_id" + ], + 29 + ], + "#FCBCB6", + [ + "==", + [ + "get", + "zoning_id" + ], + 30 + ], + "#F5F4F2", + [ + "==", + [ + "get", + "zoning_id" + ], + 31 + ], + "#169404", + + "#cccccc" + ] + }, + "shape_area": { + "label": "المساحة", + "label_en": "Parcel Area", + "fill-color": [ + "interpolate", + [ + "linear" + ], + [ + "get", + "shape_area" + ], + 0, + "hsl(86, 83%, 65%)", + 300, + "hsl(64, 63%, 53%)", + 400, + "hsl(42, 80%, 55%)", + 500, + "hsl(25, 100%, 61%)", + 600, + "hsl(8, 100%, 66%)", + 700, + "hsl(348, 100%, 66%)", + 800, + "hsl(331, 83%, 60%)", + 900, + "hsl(312, 56%, 51%)", + 1200, + "hsl(288, 49%, 47%)", + 16992097, + "hsl(266, 45%, 46%)" + ] + }, + "price_of_meter": { + "label": "سعر المتر", + "label_en": "Price of meter", + "fill-color": [ + "interpolate", + [ + "linear" + ], + [ + "get", + "price_of_meter" + ], + 0, + "hsl(0, 0%, 80%)", + 109, + "hsl(64, 63%, 53%)", + 180, + "hsl(42, 80%, 55%)", + 287, + "hsl(25, 100%, 61%)", + 478, + "hsl(8, 100%, 66%)", + 750, + "hsl(348, 100%, 66%)", + 1150, + "hsl(331, 83%, 60%)", + 1900, + "hsl(312, 56%, 51%)", + 3150, + "hsl(288, 49%, 47%)", + 20954060, + "hsl(266, 45%, 46%)" + ] + }, + "transaction_price": { + "label": "قيمة الصفقة", + "label_en": "Transaction Price", + "fill-color": [ + "interpolate", + [ + "linear" + ], + [ + "get", + "transaction_price" + ], + 0, + "hsl(0, 0%, 80%)", + 4167, + "hsl(86, 83%, 65%)", + 70000, + "hsl(64, 63%, 53%)", + 100000, + "hsl(42, 80%, 55%)", + 175000, + "hsl(25, 100%, 61%)", + 270000, + "hsl(8, 100%, 66%)", + 400000, + "hsl(348, 100%, 66%)", + 570000, + "hsl(331, 83%, 60%)", + 850000, + "hsl(312, 56%, 51%)", + 1390000, + "hsl(288, 49%, 47%)", + 4897964892, + "hsl(266, 45%, 46%)" + ] + }, + "plain": { + "label": "", + "label_en": "", + "fill-color": "rgba(0,0,0,0.1)" + } +} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/api/transactions__9941681.json b/tests/fixtures/suhail_live_2026_07/api/transactions__9941681.json new file mode 100644 index 0000000..909b8fa --- /dev/null +++ b/tests/fixtures/suhail_live_2026_07/api/transactions__9941681.json @@ -0,0 +1 @@ +{"data":{"lastExecutionDate":"2026-05-19T00:00:00","lastExecutionPrice":6393.0,"transactions":[{"_priceOfMeter":6393.0,"transactionPrice":1200000.0,"priceOfMeter":6393.0,"transactionNumber":-148623,"transactionDate":"2026-05-19T00:00:00","type":"مبنى سكني","subdivisionNo":"1082","subdivisionId":"1023186","polygonData":"{\"type\":\"Polygon\",\"coordinates\":[[[46.762974723,24.724893689],[46.762786858,24.724776766],[46.762647714,24.724963278],[46.76283558,24.725080202],[46.762974723,24.724893689]]]}","noOfProperties":null,"zoningId":3,"neighborhood":"الروضة","neighborhoodId":1000417,"region":"الرياض","parcelId":"9941681","parcelObjectId":9941681,"parcelNo":"52","blockNo":"0","area":187.7,"centroidX":46.76281121860331,"centroidY":24.72492848404151,"metricsType":"فلل","provinceId":101000,"provinceName":"الرياض","regionId":10,"geometry":null,"parcelImageURL":"","projectName":"","landUsageGroup":"سكني","sellingType":"فردي","landUseaDetailed":"مباني سكنية","centroid":{"x":46.76281121860331,"y":24.72492848404151},"propertyType":"مبنى سكني","totalArea":187.7,"details":"المساحة الإجمالية للصفقة 188م²\nعدد القطع 0","landUseGroup":"سكني","orignalTransactionNum":-148623,"propertyNumber":"52","transactionSource":"RER","isLowValueTransaction":false},{"_priceOfMeter":6260.0,"transactionPrice":1175000.0,"priceOfMeter":6260.0,"transactionNumber":-130632,"transactionDate":"2026-04-14T00:00:00","type":"مبنى سكني","subdivisionNo":"1082","subdivisionId":"1023186","polygonData":"{\"type\":\"Polygon\",\"coordinates\":[[[46.762974723,24.724893689],[46.762786858,24.724776766],[46.762647714,24.724963278],[46.76283558,24.725080202],[46.762974723,24.724893689]]]}","noOfProperties":null,"zoningId":3,"neighborhood":"الروضة","neighborhoodId":1000417,"region":"الرياض","parcelId":"9941681","parcelObjectId":9941681,"parcelNo":"52","blockNo":"0","area":187.7,"centroidX":46.76281121860331,"centroidY":24.72492848404151,"metricsType":"فلل","provinceId":101000,"provinceName":"الرياض","regionId":10,"geometry":null,"parcelImageURL":"","projectName":"","landUsageGroup":"سكني","sellingType":"فردي","landUseaDetailed":"مباني سكنية","centroid":{"x":46.76281121860331,"y":24.72492848404151},"propertyType":"مبنى سكني","totalArea":187.7,"details":"المساحة الإجمالية للصفقة 188م²\nعدد القطع 0","landUseGroup":"سكني","orignalTransactionNum":-130632,"propertyNumber":"52","transactionSource":"RER","isLowValueTransaction":false},{"_priceOfMeter":7956.0,"transactionPrice":1372000.0,"priceOfMeter":7956.0,"transactionNumber":-28480,"transactionDate":"2026-01-19T00:00:00","type":"مبنى سكني","subdivisionNo":"1082","subdivisionId":"1023186","polygonData":"{\"type\":\"Polygon\",\"coordinates\":[[[46.762974723,24.724893689],[46.762786858,24.724776766],[46.762647714,24.724963278],[46.76283558,24.725080202],[46.762974723,24.724893689]]]}","noOfProperties":null,"zoningId":3,"neighborhood":"الروضة","neighborhoodId":1000417,"region":"الرياض","parcelId":"9941681","parcelObjectId":9941681,"parcelNo":"52","blockNo":"0","area":172.4,"centroidX":46.76281121860331,"centroidY":24.72492848404151,"metricsType":"فلل","provinceId":101000,"provinceName":"الرياض","regionId":10,"geometry":null,"parcelImageURL":"","projectName":"","landUsageGroup":"سكني","sellingType":"فردي","landUseaDetailed":"مباني سكنية","centroid":{"x":46.76281121860331,"y":24.72492848404151},"propertyType":"مبنى سكني","totalArea":172.4,"details":"المساحة الإجمالية للصفقة 172م²\nعدد القطع 0","landUseGroup":"سكني","orignalTransactionNum":-28480,"propertyNumber":"52","transactionSource":"RER","isLowValueTransaction":false},{"_priceOfMeter":8318.0,"transactionPrice":1100000.0,"priceOfMeter":8318.0,"transactionNumber":-12877,"transactionDate":"2025-09-07T00:00:00","type":"مبنى سكني","subdivisionNo":"1082","subdivisionId":"1023186","polygonData":"{\"type\":\"Polygon\",\"coordinates\":[[[46.762974723,24.724893689],[46.762786858,24.724776766],[46.762647714,24.724963278],[46.76283558,24.725080202],[46.762974723,24.724893689]]]}","noOfProperties":null,"zoningId":3,"neighborhood":"الروضة","neighborhoodId":1000417,"region":"الرياض","parcelId":"9941681","parcelObjectId":9941681,"parcelNo":"52","blockNo":"0","area":132.2,"centroidX":46.76281121860331,"centroidY":24.72492848404151,"metricsType":"فلل","provinceId":101000,"provinceName":"الرياض","regionId":10,"geometry":null,"parcelImageURL":"","projectName":"","landUsageGroup":"سكني","sellingType":"فردي","landUseaDetailed":"مباني سكنية","centroid":{"x":46.76281121860331,"y":24.72492848404151},"propertyType":"مبنى سكني","totalArea":132.2,"details":"المساحة الإجمالية للصفقة 132م²\nعدد القطع 0","landUseGroup":"سكني","orignalTransactionNum":-12877,"propertyNumber":"52","transactionSource":"RER","isLowValueTransaction":false},{"_priceOfMeter":6959.0,"transactionPrice":1200000.0,"priceOfMeter":6959.0,"transactionNumber":-13419,"transactionDate":"2025-09-04T00:00:00","type":"مبنى سكني","subdivisionNo":"1082","subdivisionId":"1023186","polygonData":"{\"type\":\"Polygon\",\"coordinates\":[[[46.762974723,24.724893689],[46.762786858,24.724776766],[46.762647714,24.724963278],[46.76283558,24.725080202],[46.762974723,24.724893689]]]}","noOfProperties":null,"zoningId":3,"neighborhood":"الروضة","neighborhoodId":1000417,"region":"الرياض","parcelId":"9941681","parcelObjectId":9941681,"parcelNo":"52","blockNo":"0","area":172.4,"centroidX":46.76281121860331,"centroidY":24.72492848404151,"metricsType":"فلل","provinceId":101000,"provinceName":"الرياض","regionId":10,"geometry":null,"parcelImageURL":"","projectName":"","landUsageGroup":"سكني","sellingType":"فردي","landUseaDetailed":"مباني سكنية","centroid":{"x":46.76281121860331,"y":24.72492848404151},"propertyType":"مبنى سكني","totalArea":172.4,"details":"المساحة الإجمالية للصفقة 172م²\nعدد القطع 0","landUseGroup":"سكني","orignalTransactionNum":-13419,"propertyNumber":"52","transactionSource":"RER","isLowValueTransaction":false},{"_priceOfMeter":8582.0,"transactionPrice":1135000.0,"priceOfMeter":8582.0,"transactionNumber":-14129,"transactionDate":"2025-09-02T00:00:00","type":"مبنى سكني","subdivisionNo":"1082","subdivisionId":"1023186","polygonData":"{\"type\":\"Polygon\",\"coordinates\":[[[46.762974723,24.724893689],[46.762786858,24.724776766],[46.762647714,24.724963278],[46.76283558,24.725080202],[46.762974723,24.724893689]]]}","noOfProperties":null,"zoningId":3,"neighborhood":"الروضة","neighborhoodId":1000417,"region":"الرياض","parcelId":"9941681","parcelObjectId":9941681,"parcelNo":"52","blockNo":"0","area":132.2,"centroidX":46.76281121860331,"centroidY":24.72492848404151,"metricsType":"فلل","provinceId":101000,"provinceName":"الرياض","regionId":10,"geometry":null,"parcelImageURL":"","projectName":"","landUsageGroup":"سكني","sellingType":"فردي","landUseaDetailed":"مباني سكنية","centroid":{"x":46.76281121860331,"y":24.72492848404151},"propertyType":"مبنى سكني","totalArea":132.2,"details":"المساحة الإجمالية للصفقة 132م²\nعدد القطع 0","landUseGroup":"سكني","orignalTransactionNum":-14129,"propertyNumber":"52","transactionSource":"RER","isLowValueTransaction":false}]},"message":null,"messageTemplate":null,"status":true,"meta":null} \ No newline at end of file diff --git a/tests/fixtures/suhail_live_2026_07/tiles/riyadh_15_20636_14069.vector.pbf.gz b/tests/fixtures/suhail_live_2026_07/tiles/riyadh_15_20636_14069.vector.pbf.gz new file mode 100644 index 0000000..9ffd334 Binary files /dev/null and b/tests/fixtures/suhail_live_2026_07/tiles/riyadh_15_20636_14069.vector.pbf.gz differ diff --git a/tests/fixtures/suhail_live_2026_07/tiles/riyadh_15_20640_14060.vector.pbf.gz b/tests/fixtures/suhail_live_2026_07/tiles/riyadh_15_20640_14060.vector.pbf.gz new file mode 100644 index 0000000..63dd23c Binary files /dev/null and b/tests/fixtures/suhail_live_2026_07/tiles/riyadh_15_20640_14060.vector.pbf.gz differ diff --git a/tests/integration/test_pipeline_integration.py b/tests/integration/test_pipeline_integration.py index 78068f3..62464c6 100644 --- a/tests/integration/test_pipeline_integration.py +++ b/tests/integration/test_pipeline_integration.py @@ -5,10 +5,10 @@ import mapbox_vector_tile from sqlalchemy import create_engine -from meshic_pipeline.config import settings -import meshic_pipeline.persistence.table_management as table_management -import meshic_pipeline.pipeline_orchestrator as orchestrator -from meshic_pipeline.pipeline_orchestrator import run_pipeline +from suhail_pipeline.config import settings +import suhail_pipeline.persistence.table_management as table_management +import suhail_pipeline.pipeline_orchestrator as orchestrator +from suhail_pipeline.pipeline_orchestrator import run_pipeline # Integration test that exercises the pipeline orchestration with mocked # downloader and database persistence. @@ -21,12 +21,12 @@ class DummyInspector: def has_table(self, table_name, schema=None): return True monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.inspect", + "suhail_pipeline.pipeline_orchestrator.inspect", lambda engine: DummyInspector(), ) # Patch reset_temp_table to a no-op in both orchestrator and table_management monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.reset_temp_table", lambda *a, **kw: None + "suhail_pipeline.pipeline_orchestrator.reset_temp_table", lambda *a, **kw: None ) monkeypatch.setattr(table_management, "reset_temp_table", lambda *a, **kw: None) # sample tile with a single parcel feature @@ -48,7 +48,7 @@ def has_table(self, table_name, schema=None): # discovery returns exactly one tile coordinate monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.get_tile_coordinates_for_bounds", + "suhail_pipeline.pipeline_orchestrator.get_tile_coordinates_for_bounds", lambda bbox, zoom: [(15, 0, 0)], ) @@ -67,7 +67,7 @@ async def download_many(self, tiles): return {tiles[0]: tile_bytes} monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.AsyncTileDownloader", + "suhail_pipeline.pipeline_orchestrator.AsyncTileDownloader", DummyDownloader, ) @@ -113,7 +113,7 @@ def drop_table(self, table, schema="public"): temp_tables.pop(table, None) monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.PostGISPersister", + "suhail_pipeline.pipeline_orchestrator.PostGISPersister", DummyPersister, ) @@ -144,7 +144,7 @@ def dummy_stitch(self, table_name, layer_name, id_column, agg_rules, known_colum return gpd.GeoDataFrame(df, geometry="geometry", crs=settings.default_crs) monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.GeometryStitcher.stitch_from_table", + "suhail_pipeline.pipeline_orchestrator.GeometryStitcher.stitch_from_table", dummy_stitch, ) @@ -180,12 +180,12 @@ class DummyInspector: def has_table(self, table_name, schema=None): return True monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.inspect", + "suhail_pipeline.pipeline_orchestrator.inspect", lambda engine: DummyInspector(), ) # Patch reset_temp_table to a no-op in both orchestrator and table_management monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.reset_temp_table", lambda *a, **kw: None + "suhail_pipeline.pipeline_orchestrator.reset_temp_table", lambda *a, **kw: None ) monkeypatch.setattr(table_management, "reset_temp_table", lambda *a, **kw: None) tile_bytes = mapbox_vector_tile.encode( @@ -205,7 +205,7 @@ def has_table(self, table_name, schema=None): ) monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.get_tile_coordinates_for_bounds", + "suhail_pipeline.pipeline_orchestrator.get_tile_coordinates_for_bounds", lambda bbox, zoom: [(15, 0, 0)], ) @@ -223,7 +223,7 @@ async def download_many(self, tiles): return {tiles[0]: tile_bytes} monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.AsyncTileDownloader", + "suhail_pipeline.pipeline_orchestrator.AsyncTileDownloader", DummyDownloader, ) @@ -264,7 +264,7 @@ def drop_table(self, table, schema="public"): pass monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.PostGISPersister", + "suhail_pipeline.pipeline_orchestrator.PostGISPersister", DummyPersister, ) @@ -294,7 +294,7 @@ def dummy_stitch(self, table_name, layer_name, id_column, agg_rules, known_colum ) monkeypatch.setattr( - "meshic_pipeline.pipeline_orchestrator.GeometryStitcher.stitch_from_table", + "suhail_pipeline.pipeline_orchestrator.GeometryStitcher.stitch_from_table", dummy_stitch, ) diff --git a/tests/test_async_tile_downloader.py b/tests/test_async_tile_downloader.py index db892be..3359775 100644 --- a/tests/test_async_tile_downloader.py +++ b/tests/test_async_tile_downloader.py @@ -2,7 +2,7 @@ import pytest -from meshic_pipeline.downloader.async_tile_downloader import AsyncTileDownloader +from suhail_pipeline.downloader.async_tile_downloader import AsyncTileDownloader class FakeResponse: def __init__(self, status=200, data=b"data"): diff --git a/tests/unit/test_async_tile_downloader.py b/tests/unit/test_async_tile_downloader.py index e16d766..c93732e 100644 --- a/tests/unit/test_async_tile_downloader.py +++ b/tests/unit/test_async_tile_downloader.py @@ -1,9 +1,9 @@ import asyncio import pytest -from meshic_pipeline.config import settings +from suhail_pipeline.config import settings -from meshic_pipeline.downloader.async_tile_downloader import AsyncTileDownloader +from suhail_pipeline.downloader.async_tile_downloader import AsyncTileDownloader class FakeResponse: def __init__(self, status=200, data=b"data"): diff --git a/tests/unit/test_cli_commands.py b/tests/unit/test_cli_commands.py index d420e44..7baf3fb 100644 --- a/tests/unit/test_cli_commands.py +++ b/tests/unit/test_cli_commands.py @@ -1,6 +1,6 @@ import pytest from typer.testing import CliRunner -from meshic_pipeline.cli import app +from suhail_pipeline.cli import app import asyncio from unittest.mock import AsyncMock @@ -42,7 +42,7 @@ def test_geometric_command_runs(monkeypatch, args, expected): (["--limit", "10"], "delta enrichment"), ]) def test_delta_enrich_command_runs(monkeypatch, args, expected): - import meshic_pipeline.run_enrichment_pipeline as rep + import suhail_pipeline.run_enrichment_pipeline as rep monkeypatch.setattr(rep, "get_async_db_engine", lambda: None) monkeypatch.setattr(rep, "_table_exists", AsyncMock(return_value=True)) monkeypatch.setattr(rep, "get_delta_parcel_ids_with_details", AsyncMock(return_value=([], {}))) @@ -95,7 +95,7 @@ def test_monitor_command(monkeypatch, action, should_succeed): ]) def test_fast_enrich_command(monkeypatch, args): # Mock enrichment function - import meshic_pipeline.run_enrichment_pipeline as rep + import suhail_pipeline.run_enrichment_pipeline as rep monkeypatch.setattr(rep, "fast_enrich", lambda *a, **k: None) result = runner.invoke(app, ["fast-enrich"] + args) assert result.exit_code == 0 @@ -107,7 +107,7 @@ def test_fast_enrich_command(monkeypatch, args): ["--limit", "5"], ]) def test_incremental_enrich_command(monkeypatch, args): - import meshic_pipeline.run_enrichment_pipeline as rep + import suhail_pipeline.run_enrichment_pipeline as rep monkeypatch.setattr(rep, "incremental_enrich", lambda *a, **k: None) result = runner.invoke(app, ["incremental-enrich"] + args) assert result.exit_code == 0 @@ -118,7 +118,7 @@ def test_incremental_enrich_command(monkeypatch, args): ["--limit", "2"], ]) def test_full_refresh_command(monkeypatch, args): - import meshic_pipeline.run_enrichment_pipeline as rep + import suhail_pipeline.run_enrichment_pipeline as rep monkeypatch.setattr(rep, "full_refresh", lambda *a, **k: None) result = runner.invoke(app, ["full-refresh"] + args) assert result.exit_code == 0 @@ -188,7 +188,7 @@ def test_geometric_help_output(): # Enhance delta-enrich output assertion def test_delta_enrich_auto_geometric_output(monkeypatch): - import meshic_pipeline.run_enrichment_pipeline as rep + import suhail_pipeline.run_enrichment_pipeline as rep monkeypatch.setattr(rep, "get_async_db_engine", lambda: None) monkeypatch.setattr(rep, "_table_exists", AsyncMock(return_value=True)) monkeypatch.setattr(rep, "get_delta_parcel_ids_with_details", AsyncMock(return_value=([], {}))) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 517e7b6..211eb1d 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,4 +1,4 @@ -from meshic_pipeline.config import Settings, Environment, ApiConfig +from suhail_pipeline.config import Settings, Environment, ApiConfig def test_environment_override(): diff --git a/tests/unit/test_data_validation.py b/tests/unit/test_data_validation.py index f35fe45..7a985b8 100644 --- a/tests/unit/test_data_validation.py +++ b/tests/unit/test_data_validation.py @@ -2,7 +2,7 @@ import geopandas as gpd from shapely.geometry import Point -from meshic_pipeline.persistence.postgis_persister import PostGISPersister +from suhail_pipeline.persistence.postgis_persister import PostGISPersister class MockPersister(PostGISPersister): def __init__(self): diff --git a/tests/unit/test_decoder.py b/tests/unit/test_decoder.py index ca95997..f8f8860 100644 --- a/tests/unit/test_decoder.py +++ b/tests/unit/test_decoder.py @@ -1,4 +1,4 @@ -from meshic_pipeline.decoder.mvt_decoder import MVTDecoder +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder import geopandas as gpd import shapely.geometry diff --git a/tests/unit/test_decoder_html_guard.py b/tests/unit/test_decoder_html_guard.py new file mode 100644 index 0000000..fa85266 --- /dev/null +++ b/tests/unit/test_decoder_html_guard.py @@ -0,0 +1,21 @@ +"""The decoder must skip non-MVT payloads (e.g. an HTML error page the tile server +returns instead of a vector tile) cleanly, rather than raising a protobuf DecodeError. +Salvaged idea from the abandoned fix-mvt-decoder-str-bug branch, cleaned up. +""" +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder + + +def test_html_payload_is_skipped_not_decoded(): + dec = MVTDecoder() + html = b"503 Service Unavailable" + assert dec.decode_bytes(html, 15, 20636, 14069) == {} + + +def test_leading_whitespace_html_is_skipped(): + dec = MVTDecoder() + assert dec.decode_bytes(b"\n error", 15, 1, 1) == {} + + +def test_empty_payload_is_skipped(): + dec = MVTDecoder() + assert dec.decode_bytes(b"", 15, 1, 1) == {} diff --git a/tests/unit/test_delta_enrichment_cli.py b/tests/unit/test_delta_enrichment_cli.py index 35873de..7ac7d55 100644 --- a/tests/unit/test_delta_enrichment_cli.py +++ b/tests/unit/test_delta_enrichment_cli.py @@ -1,7 +1,7 @@ from typer.testing import CliRunner -from meshic_pipeline.run_enrichment_pipeline import app -import meshic_pipeline.run_enrichment_pipeline as rep +from suhail_pipeline.run_enrichment_pipeline import app +import suhail_pipeline.run_enrichment_pipeline as rep runner = CliRunner() diff --git a/tests/unit/test_discovery.py b/tests/unit/test_discovery.py index a2b6cb2..07a250e 100644 --- a/tests/unit/test_discovery.py +++ b/tests/unit/test_discovery.py @@ -1,6 +1,6 @@ import mercantile -from meshic_pipeline.pipeline_orchestrator import get_tile_coordinates_for_grid +from suhail_pipeline.pipeline_orchestrator import get_tile_coordinates_for_grid def test_get_tile_coordinates_for_bounds_round_trip(): diff --git a/tests/unit/test_enrichment_parsers_contract.py b/tests/unit/test_enrichment_parsers_contract.py new file mode 100644 index 0000000..4e9218b --- /dev/null +++ b/tests/unit/test_enrichment_parsers_contract.py @@ -0,0 +1,95 @@ +"""Contract tests: parse real captured Suhail API payloads through the enrichment +parsers and assert the fields we now promise to capture actually survive. + +Fixtures were captured live on 2026-07-15 (see docs/SUHAIL_SOURCE_AUDIT_2026-07.md) +and live under tests/fixtures/suhail_live_2026_07/api/. +""" +import json +from pathlib import Path + +import pytest + +from suhail_pipeline.enrichment.api_client import ( + parse_transactions_payload, + parse_building_rules_payload, + parse_price_metrics_payload, +) +from suhail_pipeline.persistence.enrichment_persister import fast_store_batch_data # noqa: F401 (import guard) + +FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "suhail_live_2026_07" / "api" + + +def _load(name): + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def test_transactions_payload_promotes_market_attributes(): + data = _load("transactions__9941681.json") + txs = parse_transactions_payload(data, 9941681) + assert txs, "fixture parcel 9941681 should have transactions" + + tx = txs[0] + # Legacy fields still captured. + assert tx.transaction_id is not None + assert tx.transaction_price is not None + assert tx.price_of_meter is not None + assert tx.transaction_date is not None + # Newly-promoted attributes (previously only in raw_data). + assert tx.transaction_type is not None # "type" + assert tx.property_type is not None # "propertyType" + assert tx.metrics_type is not None # "metricsType" + assert tx.land_use_group is not None # landUseGroup / landUsageGroup + assert tx.transaction_source is not None # "transactionSource" + assert isinstance(tx.subdivision_id, int) # string in source, coerced to int + assert isinstance(tx.neighborhood_id, int) + assert tx.is_low_value_transaction is not None + # raw_data preserved in full. + assert tx.raw_data.get("transactionNumber") == tx.transaction_id + + +def test_transactions_empty_payload_is_safe(): + empty = {"data": {"transactions": []}, "status": True} + assert parse_transactions_payload(empty, 1) == [] + assert parse_transactions_payload({}, 1) == [] + + +def test_building_rules_payload_captures_all_setback_fields(): + data = _load("buildingRules__9858274.json") + rules = parse_building_rules_payload(data, 9858274) + assert len(rules) >= 1 + r = rules[0] + assert r.building_rule_id is not None + assert r.max_building_coefficient is not None + assert r.max_building_height is not None + assert r.max_parcel_coverage is not None + # setback fields must survive + assert r.main_streets_setback is not None + assert r.secondary_streets_setback is not None + assert r.side_rear_setback is not None + + +def test_building_rules_multiple_rules_are_not_collapsed(): + """The persister must key on (parcel_objectid, building_rule_id), not parcel alone.""" + data = { + "status": True, + "data": [ + {"id": "RULE-A", "zoningId": 1}, + {"id": "RULE-B", "zoningId": 2}, + ], + } + rules = parse_building_rules_payload(data, 555) + assert len(rules) == 2 + dedup = {(r.parcel_objectid, r.building_rule_id): r for r in rules} + assert len(dedup) == 2, "two distinct rules for one parcel must both survive dedup" + + +def test_price_metrics_payload_sets_neighborhood_id(): + data = _load("priceOfMeter__9858274.json") + metrics = parse_price_metrics_payload(data) + assert metrics, "fixture should yield neighborhood metrics" + # Every metric must now carry a neighborhood_id (previously always NULL). + assert all(m.neighborhood_id is not None for m in metrics) + m = metrics[0] + assert m.month is not None and m.year is not None + assert m.metrics_type is not None + assert m.average_price_of_meter is not None diff --git a/tests/unit/test_enrichment_strategies.py b/tests/unit/test_enrichment_strategies.py index b79b63f..2ad74b8 100644 --- a/tests/unit/test_enrichment_strategies.py +++ b/tests/unit/test_enrichment_strategies.py @@ -1,5 +1,5 @@ import pytest -from meshic_pipeline.enrichment import strategies +from suhail_pipeline.enrichment import strategies def test_quote_table_name_valid(): diff --git a/tests/unit/test_geometric_cli.py b/tests/unit/test_geometric_cli.py index f57e171..182e432 100644 --- a/tests/unit/test_geometric_cli.py +++ b/tests/unit/test_geometric_cli.py @@ -1,5 +1,5 @@ from typer.testing import CliRunner -from meshic_pipeline.run_geometric_pipeline import app +from suhail_pipeline.run_geometric_pipeline import app runner = CliRunner() diff --git a/tests/unit/test_live_tile_schema.py b/tests/unit/test_live_tile_schema.py new file mode 100644 index 0000000..2683aa5 --- /dev/null +++ b/tests/unit/test_live_tile_schema.py @@ -0,0 +1,91 @@ +"""Contract tests: decode a real (gzipped) live Suhail MVT tile and assert the +decoder + canonical SCHEMA_MAP now capture the fields the 2026 tile schema exposes. + +Fixture tiles captured 2026-07-15 (see docs/SUHAIL_SOURCE_AUDIT_2026-07.md). +The residential tile 20640/14060 contains parcels with recent transactions. +""" +import gzip +from pathlib import Path + +import pytest + +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder +from suhail_pipeline.persistence.postgis_persister import SCHEMA_MAP + +TILES = Path(__file__).resolve().parents[1] / "fixtures" / "suhail_live_2026_07" / "tiles" +RESIDENTIAL = TILES / "riyadh_15_20640_14060.vector.pbf.gz" # z/x/y = 15/20640/14060 +DOWNTOWN = TILES / "riyadh_15_20636_14069.vector.pbf.gz" # has all 17 layers + + +def _decode(path, z, x, y): + raw = path.read_bytes() + data = gzip.decompress(raw) if raw[:2] == b"\x1f\x8b" else raw + return MVTDecoder().decode_to_gdf(data, z, x, y) + + +def _schema_filter(gdf, layer): + """Mimic the orchestrator: arabic-map then filter to SCHEMA_MAP columns.""" + gdf = MVTDecoder.apply_arabic_column_mapping(gdf) + allowed = set(SCHEMA_MAP.get(layer, {}).keys()) | {"geometry"} + return gdf[[c for c in gdf.columns if c in allowed]] + + +@pytest.fixture(scope="module") +def residential_layers(): + assert RESIDENTIAL.exists(), f"missing fixture tile {RESIDENTIAL}" + return _decode(RESIDENTIAL, 15, 20640, 14060) + + +@pytest.fixture(scope="module") +def downtown_layers(): + assert DOWNTOWN.exists(), f"missing fixture tile {DOWNTOWN}" + return _decode(DOWNTOWN, 15, 20636, 14069) + + +def test_expected_layers_present(downtown_layers): + # The downtown tile carries the full transit + metrics layer set. + for layer in ("parcels", "neighborhoods", "subdivisions", "bus_lines", + "metro_stations", "riyadh_bus_stations", "qi_population_metrics"): + assert layer in downtown_layers, f"layer {layer} missing from live tile" + + +def test_parcels_market_timeseries_survive_schema_filter(residential_layers): + gdf = _schema_filter(residential_layers["parcels"], "parcels") + cols = set(gdf.columns) + for f in ( + "transaction_price_1w", "transaction_price_12m", + "price_of_meter_1m", "price_of_meter_6m", + "transactions_count_12m", "transaction_date_12m", + "zoning_group", + ): + assert f in cols, f"parcels field {f} dropped by schema filter" + # Arabic neighbourhood name now retained on parcels. + assert "neighborhood_ar" in cols + + +def test_parcels_have_nonzero_market_data(residential_layers): + """At least some residential parcels carry real 12-month transaction data.""" + gdf = residential_layers["parcels"] + assert "transactions_count_12m" in gdf.columns + hits = (gdf["transactions_count_12m"].fillna(0).astype(float) > 0).sum() + assert hits > 0, "expected some parcels with 12m transactions in this tile" + + +def test_bus_lines_fields_now_captured(downtown_layers): + """Regression: bus_lines previously captured zero attributes (name mismatch).""" + gdf = _schema_filter(downtown_layers["bus_lines"], "bus_lines") + cols = set(gdf.columns) + assert {"busroute", "color", "type"} <= cols + assert len(gdf) > 0 + + +def test_metro_stations_coordinates_captured(downtown_layers): + gdf = _schema_filter(downtown_layers["metro_stations"], "metro_stations") + assert {"station_long", "station_lat", "station_code"} <= set(gdf.columns) + + +def test_qi_population_metrics_reshaped_fields_captured(downtown_layers): + gdf = _schema_filter(downtown_layers["qi_population_metrics"], "qi_population_metrics") + cols = set(gdf.columns) + for f in ("population_density", "rent_apartment", "purchasing_power", "poi_count"): + assert f in cols, f"qi_population_metrics field {f} not captured" diff --git a/tests/unit/test_new_layer_keys.py b/tests/unit/test_new_layer_keys.py new file mode 100644 index 0000000..8cbab69 --- /dev/null +++ b/tests/unit/test_new_layer_keys.py @@ -0,0 +1,95 @@ +"""Unit tests for the dimensions / building_detection write-mode plumbing: +- decode stamps `source_tile` and the synthetic `bd_id`, and they survive the + SCHEMA_MAP column filter; +- `compute_synthetic_pk` is deterministic, geometry-sensitive, and keeps distinct + rows distinct. +""" +import gzip +from pathlib import Path + +import pytest +from shapely.geometry import Point + +from suhail_pipeline.pipeline_orchestrator import decode_and_validate_tile +from suhail_pipeline.decoder.mvt_decoder import MVTDecoder +from suhail_pipeline.persistence.postgis_persister import ( + SCHEMA_MAP, + TILE_SCOPED_LAYERS, + SYNTHETIC_PK_CONFIG, + compute_synthetic_pk, +) +from suhail_pipeline.config import settings + +import geopandas as gpd + +TILES = Path(__file__).resolve().parents[1] / "fixtures" / "suhail_live_2026_07" / "tiles" +DOWNTOWN = TILES / "riyadh_15_20636_14069.vector.pbf.gz" + + +def _decoded(layer): + raw = DOWNTOWN.read_bytes() + data = gzip.decompress(raw) + out = decode_and_validate_tile((15, 20636, 14069), data, [layer], settings.default_crs) + for name, gdf in out: + if name == layer: + return gdf + return None + + +def _schema_filter(gdf, layer): + gdf = MVTDecoder.apply_arabic_column_mapping(gdf) + allowed = set(SCHEMA_MAP.get(layer, {}).keys()) | {"geometry"} + return gdf[[c for c in gdf.columns if c in allowed]] + + +def test_config_wiring(): + assert "dimensions" in settings.layers_to_process + assert "building_detection" in settings.layers_to_process + # dimensions must NOT be keyed (would collapse edges); building_detection is synthetic-keyed + assert settings.id_column_per_layer.get("dimensions") is None + assert settings.id_column_per_layer.get("building_detection") == "bd_id" + assert "dimensions" in TILE_SCOPED_LAYERS + assert "building_detection" in SYNTHETIC_PK_CONFIG + + +def test_dimensions_source_tile_stamped_and_survives_filter(): + gdf = _decoded("dimensions") + assert gdf is not None and len(gdf) > 100 + assert (gdf["source_tile"] == "15/20636/14069").all() + filtered = _schema_filter(gdf, "dimensions") + assert "source_tile" in filtered.columns + assert {"parcel_objectid", "length_m", "azimuth"} <= set(filtered.columns) + # many rows per parcel (edges) — confirms why it can't be keyed on parcel_objectid + assert gdf["parcel_objectid"].nunique() < len(gdf) + + +def test_building_detection_synthetic_key_stamped_and_unique_enough(): + gdf = _decoded("building_detection") + assert gdf is not None and len(gdf) > 100 + assert "bd_id" in gdf.columns + assert (gdf["source_tile"] == "15/20636/14069").all() + # every row got an id, and ids are (near-)unique for distinct geometries + assert gdf["bd_id"].notna().all() + assert gdf["bd_id"].nunique() >= len(gdf) * 0.99 + + +def test_compute_synthetic_pk_is_deterministic_and_geometry_sensitive(): + g = gpd.GeoDataFrame( + {"region_id": [10, 10], "class_pred": ["1", "1"], "prediction_year": [2025, 2025]}, + geometry=[Point(46.7, 24.6), Point(46.8, 24.7)], + crs="EPSG:4326", + ) + out1 = compute_synthetic_pk(g, "building_detection") + out2 = compute_synthetic_pk(g, "building_detection") + # deterministic + assert list(out1["bd_id"]) == list(out2["bd_id"]) + # geometry-sensitive: same attrs, different geometry -> different id + assert out1["bd_id"].iloc[0] != out1["bd_id"].iloc[1] + # fits in a signed BIGINT + assert all(0 <= v < 2**63 for v in out1["bd_id"]) + + +def test_compute_synthetic_pk_noop_for_unconfigured_layer(): + g = gpd.GeoDataFrame({"a": [1]}, geometry=[Point(0, 0)], crs="EPSG:4326") + out = compute_synthetic_pk(g, "parcels") + assert "bd_id" not in out.columns diff --git a/tests/unit/test_province_loader.py b/tests/unit/test_province_loader.py index 5926c83..e2c51d2 100644 --- a/tests/unit/test_province_loader.py +++ b/tests/unit/test_province_loader.py @@ -1,5 +1,5 @@ -from meshic_pipeline.config import settings -from meshic_pipeline.utils.tile_list_generator import tiles_from_bbox_z +from suhail_pipeline.config import settings +from suhail_pipeline.utils.tile_list_generator import tiles_from_bbox_z def test_provinces_loaded(monkeypatch): diff --git a/tests/unit/test_smart_pipeline_enrich.py b/tests/unit/test_smart_pipeline_enrich.py index 554d002..4a6b550 100644 --- a/tests/unit/test_smart_pipeline_enrich.py +++ b/tests/unit/test_smart_pipeline_enrich.py @@ -1,11 +1,11 @@ -import meshic_pipeline.run_enrichment_pipeline as rep +import suhail_pipeline.run_enrichment_pipeline as rep def test_smart_pipeline_import(monkeypatch): called = {} def fake_main(): called['ok'] = True - monkeypatch.setattr('meshic_pipeline.run_geometric_pipeline.main', fake_main) + monkeypatch.setattr('suhail_pipeline.run_geometric_pipeline.main', fake_main) # Call the command function directly to ensure import works rep.smart_pipeline_enrich(geometric_first=True, trigger_after=False, batch_size=300, bbox=None) assert called.get('ok') diff --git a/tests/unit/test_stitcher.py b/tests/unit/test_stitcher.py index b426283..fb4b60e 100644 --- a/tests/unit/test_stitcher.py +++ b/tests/unit/test_stitcher.py @@ -1,5 +1,5 @@ import geopandas as gpd -from meshic_pipeline.geometry.stitcher import GeometryStitcher +from suhail_pipeline.geometry.stitcher import GeometryStitcher class DummyPersister: def __init__(self): diff --git a/tests/unit/test_validator.py b/tests/unit/test_validator.py index 470cdf4..eb24064 100644 --- a/tests/unit/test_validator.py +++ b/tests/unit/test_validator.py @@ -1,7 +1,7 @@ import geopandas as gpd from shapely.geometry import Polygon -from meshic_pipeline.geometry.validator import validate_geometries +from suhail_pipeline.geometry.validator import validate_geometries def test_validate_geometries_empty(): diff --git a/uv.lock b/uv.lock index d04804c..f9bcc02 100644 --- a/uv.lock +++ b/uv.lock @@ -854,7 +854,7 @@ wheels = [ ] [[package]] -name = "meshic-pipeline" +name = "suhail-pipeline" version = "0.1.0" source = { editable = "." } dependencies = [