DBT'ify pluto staging models

.envrc

@@ -1,8 +1,13 @@
 #!/usr/bin/env bash
 
 # Load all variables from .env file
-dotenv .env
-watch_file .env
+if [ -n "$DATA_ENGINEERING_DOT_FILE_PATH" ] && [ -f "$DATA_ENGINEERING_DOT_FILE_PATH" ]; then
+    dotenv "$DATA_ENGINEERING_DOT_FILE_PATH"
+    watch_file "$DATA_ENGINEERING_DOT_FILE_PATH"
+elif [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
 
 export PROJECT_ROOT_PATH="$PWD"
 

.github/workflows/pluto_build.yml

@@ -81,10 +81,11 @@ jobs:
     - name: Dataloading
       working-directory: products/pluto
       run: python3 -m dcpy lifecycle builds load load --recipe-path ${{ inputs.recipe_file
-        }}.lock.yml
+        }}.lock.yml --cache-schema recipe_cache --cached-entity-type view
 
-    - name: Load Local Data
-      run: ./01_load_local_csvs.sh
+    - name: Load Local Data and DBT Staging
+      working-directory: products/pluto
+      run: ./pluto_build/01a_dbt_staging.sh
 
     - name: building ...
       run: ./02_build.sh

bash/bin/load_direnv.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Load direnv environment variables for the current directory. Mostly for processes
+# that, ahem, make use of non-interactive terminals
+eval "$(direnv export bash)"

products/cbbr/.envrc

@@ -0,0 +1,11 @@
+source_up
+
+export BUILD_ENGINE_DB=db-cbbr
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
+
+export BUILD_ENGINE=$(echo_build_engine)

products/cdbg/.envrc

@@ -0,0 +1,11 @@
+source_up
+
+export BUILD_ENGINE_DB=db-cdbg
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
+
+export BUILD_ENGINE=$(echo_build_engine)

products/ceqr/.envrc

@@ -0,0 +1,11 @@
+source_up
+
+export BUILD_ENGINE_DB=db-ceqr
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
+
+export BUILD_ENGINE=$(echo_build_engine)

products/checkbook/.envrc

@@ -0,0 +1,11 @@
+source_up
+
+export BUILD_ENGINE_DB=db-checkbook
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
+
+export BUILD_ENGINE=$(echo_build_engine)

products/colp/.envrc

@@ -0,0 +1,11 @@
+source_up
+
+export BUILD_ENGINE_DB=db-colp
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
+
+export BUILD_ENGINE=$(echo_build_engine)

products/cpdb/.envrc

@@ -0,0 +1,11 @@
+source_up
+
+export BUILD_ENGINE_DB=db-cpdb
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
+
+export BUILD_ENGINE=$(echo_build_engine)

products/cscl/.envrc

@@ -0,0 +1,11 @@
+source_up
+
+export BUILD_ENGINE_DB=db-cscl
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
+
+export BUILD_ENGINE=$(echo_build_engine)

products/developments/.envrc

@@ -0,0 +1,11 @@
+source_up
+
+export BUILD_ENGINE_DB=db-devdb
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
+
+export BUILD_ENGINE=$(echo_build_engine)

products/edde/.envrc

@@ -0,0 +1,7 @@
+source_up
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi

products/facilities/.envrc

@@ -1,9 +1,11 @@
 source_up
 
+export BUILD_ENGINE_DB=db-facilities
+
 # Load local .env if it exists
 if [ -f .env ]; then
     dotenv .env
     watch_file .env
 fi
 
-export BUILD_ENGINE=$(dcp_echo_build_engine)
+export BUILD_ENGINE=$(echo_build_engine)

products/factfinder/.envrc

@@ -0,0 +1,11 @@
+source_up
+
+export BUILD_ENGINE_DB=db-factfinder
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
+
+export BUILD_ENGINE=$(echo_build_engine)

products/green_fast_track/.envrc

@@ -0,0 +1,11 @@
+source_up
+
+export BUILD_ENGINE_DB=db-green_fast_track
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi
+
+export BUILD_ENGINE=$(echo_build_engine)

products/knownprojects/.envrc

@@ -0,0 +1,7 @@
+source_up
+
+# Load local .env if it exists
+if [ -f .env ]; then
+    dotenv .env
+    watch_file .env
+fi

products/pluto/.envrc

@@ -1,9 +1,11 @@
 source_up
 
+export BUILD_ENGINE_DB=db-pluto
+
 # Load local .env if it exists
 if [ -f .env ]; then
     dotenv .env
     watch_file .env
 fi
 
-export BUILD_ENGINE=$(dcp_echo_build_engine)
+export BUILD_ENGINE=$(echo_build_engine)

products/pluto/dbt_project.yml

@@ -5,10 +5,14 @@ profile: "dcp-de-postgres"
 model-paths: [ "models" ]
 test-paths: [ "tests" ]
 
+seeds:
+  pluto:
+    +quote_columns: true
+
 tests:
   +store_failures: true
   +severity: "{{ env_var('TEST_SEVERITY', 'error') }}"
-  schema: "_tests"
+  +schema: "_tests"
 
 vars:
   version: "{{ env_var('VERSION') }}"

products/pluto/models/_sources.yml

@@ -4,7 +4,43 @@ sources:
 - name: recipe_sources
   schema: "{{ env_var('BUILD_ENGINE_SCHEMA') }}"
   tables:
-  - name: pluto_input_research
+  - name: pluto_input_cama_dof
+  - name: pluto_input_geocodes_raw
+  - name: pluto_input_numbldgs
+  - name: pluto_pts
+  - name: previous_pluto
+  - name: lpc_landmarks
+  - name: lpc_historic_districts
+  - name: dcp_edesignation
+  - name: dcp_councildistricts_wi
+  - name: dcp_cb2010_wi
+  - name: dcp_cb2020_wi
+  - name: dcp_ct2010_wi
+  - name: dcp_ct2020_wi
+  - name: dcp_cdboundaries_wi
+  - name: doitt_zipcodeboundaries
+  - name: dcp_school_districts
+  - name: dcp_zoningdistricts
+  - name: dcp_commercialoverlay
+  - name: dcp_limitedheight
+  - name: dcp_specialpurpose
+  - name: dcp_specialpurposesubdistricts
+  - name: dcp_zoningmapamendments
+  - name: dcp_zoningmapindex
+  - name: dcp_firecompanies
+  - name: dcp_policeprecincts
+  - name: dcp_healthareas
+  - name: dcp_healthcenters
+  - name: dsny_frequencies
+  - name: dcp_colp
+  - name: dpr_greenthumb
+  - name: dof_condo
+  - name: dof_dtm
+  - name: dof_shoreline
+  - name: fema_firms2007_100yr
+  - name: fema_pfirms2015_100yr
+  - name: dcp_transit_zones
+  - name: dcp_gis_mandatory_inclusionary_housing
   - name: dcp_developments
     description: >-
       Contains changes in units resulting from new buildings, major alterations, and
@@ -44,8 +80,6 @@ sources:
             ]
     - name: date_complete
 
-  - name: pluto_pts
-
 - name: build_sources
   schema: "{{ env_var('BUILD_ENGINE_SCHEMA') }}"
   tables:
@@ -59,5 +93,4 @@ sources:
   - name: previous_pluto
   - name: export_pluto
     description: final PLUTO table
-  - name: dcp_zoningdistricts
   - name: dcp_zoning_maxfar

products/pluto/models/qaqc/intermediate/qaqc_int__active_condo_bbl_unitsres_corrections.sql

@@ -6,7 +6,7 @@ WITH historical_condo_unit_corrections AS (
     SELECT
         bbl,
         old_value::numeric
-    FROM {{ source("recipe_sources", "pluto_input_research") }}
+    FROM {{ ref("pluto_input_research") }}
     WHERE
         field = 'unitsres'
         AND substring(bbl, 7, 2) = '75'

products/pluto/models/staging/stg__dcp_cb2010_wi.sql

@@ -0,0 +1,13 @@
+{{
+  config(
+    materialized='table',
+    indexes=[
+      {'columns': ['geom'], 'type': 'gist'}
+    ]
+  )
+}}
+
+SELECT
+    *,
+    wkb_geometry AS geom
+FROM {{ source('recipe_sources', 'dcp_cb2010_wi') }}

products/pluto/models/staging/stg__dcp_cb2020_wi.sql

@@ -0,0 +1,13 @@
+{{
+  config(
+    materialized='table',
+    indexes=[
+      {'columns': ['geom'], 'type': 'gist'}
+    ]
+  )
+}}
+
+SELECT
+    *,
+    wkb_geometry AS geom
+FROM {{ source('recipe_sources', 'dcp_cb2020_wi') }}

products/pluto/models/staging/stg__dcp_cdboundaries_wi.sql

@@ -0,0 +1,13 @@
+{{
+  config(
+    materialized='table',
+    indexes=[
+      {'columns': ['geom'], 'type': 'gist'}
+    ]
+  )
+}}
+
+SELECT
+    *,
+    wkb_geometry AS geom
+FROM {{ source('recipe_sources', 'dcp_cdboundaries_wi') }}

products/pluto/models/staging/stg__dcp_colp.sql

@@ -0,0 +1,12 @@
+{{
+  config(
+    materialized='table',
+    indexes=[
+      {'columns': ['geom'], 'type': 'gist'}
+    ]
+  )
+}}
+
+SELECT
+    *
+FROM {{ source('recipe_sources', 'dcp_colp') }}

products/pluto/models/staging/stg__dcp_commercialoverlay.sql

@@ -0,0 +1,13 @@
+{{
+  config(
+    materialized='table',
+    indexes=[
+      {'columns': ['geom'], 'type': 'gist'}
+    ]
+  )
+}}
+
+SELECT
+    *,
+    wkb_geometry AS geom
+FROM {{ source('recipe_sources', 'dcp_commercialoverlay') }}

products/pluto/models/staging/stg__dcp_councildistricts_wi.sql

@@ -0,0 +1,13 @@
+{{
+  config(
+    materialized='table',
+    indexes=[
+      {'columns': ['geom'], 'type': 'gist'}
+    ]
+  )
+}}
+
+SELECT
+    *,
+    wkb_geometry AS geom
+FROM {{ source('recipe_sources', 'dcp_councildistricts_wi') }}

products/pluto/models/staging/stg__dcp_ct2010_wi.sql

@@ -0,0 +1,13 @@
+{{
+  config(
+    materialized='table',
+    indexes=[
+      {'columns': ['geom'], 'type': 'gist'}
+    ]
+  )
+}}
+
+SELECT
+    *,
+    wkb_geometry AS geom
+FROM {{ source('recipe_sources', 'dcp_ct2010_wi') }}

products/pluto/models/staging/stg__dcp_ct2020_wi.sql

@@ -0,0 +1,13 @@
+{{
+  config(
+    materialized='table',
+    indexes=[
+      {'columns': ['geom'], 'type': 'gist'}
+    ]
+  )
+}}
+
+SELECT
+    *,
+    wkb_geometry AS geom
+FROM {{ source('recipe_sources', 'dcp_ct2020_wi') }}