diff --git a/.github/workflows/colp_build.yml b/.github/workflows/colp_build.yml index 8f196431c4..788f7183c6 100644 --- a/.github/workflows/colp_build.yml +++ b/.github/workflows/colp_build.yml @@ -47,6 +47,10 @@ jobs: AWS_SECRET_ACCESS_KEY: "op://Data Engineering/DO_keys/AWS_SECRET_ACCESS_KEY" AWS_ACCESS_KEY_ID: "op://Data Engineering/DO_keys/AWS_ACCESS_KEY_ID" BUILD_ENGINE_SERVER: "op://Data Engineering/EDM_DATA/server_url" + BUILD_ENGINE_HOST: "op://Data Engineering/EDM_DATA/server" + BUILD_ENGINE_USER: "op://Data Engineering/EDM_DATA/username" + BUILD_ENGINE_PASSWORD: "op://Data Engineering/EDM_DATA/password" + BUILD_ENGINE_PORT: "op://Data Engineering/EDM_DATA/port" - name: Finish container setup ... working-directory: ./ diff --git a/products/colp/bash/02a_load_data.sh b/products/colp/bash/02a_load_data.sh new file mode 100755 index 0000000000..c1a58cca84 --- /dev/null +++ b/products/colp/bash/02a_load_data.sh @@ -0,0 +1,7 @@ +#!/bin/bash +source bash/config.sh + +# Load seeds via dbt +dbt seed --select usetype_mappings +run_sql_file sql/load_modifications.sql +run_sql_file sql/geo_inputs.sql diff --git a/products/colp/bash/02b_python.sh b/products/colp/bash/02b_python.sh new file mode 100755 index 0000000000..f76a9b951d --- /dev/null +++ b/products/colp/bash/02b_python.sh @@ -0,0 +1,5 @@ +#!/bin/bash +source bash/config.sh + +python3 -m python.geocode +python3 -m python.geo_qaqc diff --git a/products/colp/bash/02_build.sh b/products/colp/bash/02c_sql_build.sh similarity index 69% rename from products/colp/bash/02_build.sh rename to products/colp/bash/02c_sql_build.sh index 7c28934e7e..a2157dfc1c 100755 --- a/products/colp/bash/02_build.sh +++ b/products/colp/bash/02c_sql_build.sh @@ -1,13 +1,6 @@ #!/bin/bash source bash/config.sh -run_sql_file sql/load_modifications.sql - -run_sql_file sql/geo_inputs.sql - -python3 -m python.geocode -python3 -m python.geo_qaqc - run_sql_file sql/_procedures.sql run_sql_file sql/clean_parcelname.sql run_sql_file sql/create_colp.sql diff --git a/products/colp/colp.sh b/products/colp/colp.sh index c769a641f0..3d0f1a4898 100755 --- a/products/colp/colp.sh +++ b/products/colp/colp.sh @@ -2,10 +2,17 @@ case $1 in dataloading ) ./bash/01_dataloading.sh ;; - build ) ./bash/02_build.sh ;; + build ) + ./bash/02a_load_data.sh + ./bash/02b_python.sh + ./bash/02c_sql_build.sh + ;; + load_data ) ./bash/02a_load_data.sh ;; + python ) ./bash/02b_python.sh ;; + sql_build ) ./bash/02c_sql_build.sh ;; qaqc ) ./bash/03_qaqc.sh ;; export ) ./bash/04_export.sh ;; upload ) python3 -m dcpy.connectors.edm.publishing upload -p db-colp -a public-read ;; sql) sql $@ ;; - * ) echo "COMMAND \"$1\" is not found. (valid commands: dataloading|build|export|upload)" ;; + * ) echo "COMMAND \"$1\" is not found. (valid commands: dataloading|build|load_data|python|sql_build|qaqc|export|upload)" ;; esac diff --git a/products/colp/dbt_project.yml b/products/colp/dbt_project.yml new file mode 100644 index 0000000000..f0c62676d3 --- /dev/null +++ b/products/colp/dbt_project.yml @@ -0,0 +1,5 @@ +name: "colp" + +profile: "dcp-de-postgres" + +seed-paths: ["seeds"] diff --git a/products/colp/profiles.yml b/products/colp/profiles.yml new file mode 100644 index 0000000000..6712525864 --- /dev/null +++ b/products/colp/profiles.yml @@ -0,0 +1,11 @@ +dcp-de-postgres: + target: dev + outputs: + dev: + type: postgres + host: "{{ env_var('BUILD_ENGINE_HOST') }}" + user: "{{ env_var('BUILD_ENGINE_USER') }}" + password: "{{ env_var('BUILD_ENGINE_PASSWORD') }}" + port: "{{ env_var('BUILD_ENGINE_PORT') | as_number }}" + dbname: "{{ env_var('BUILD_ENGINE_DB') }}" + schema: "{{ env_var('BUILD_ENGINE_SCHEMA') }}" diff --git a/products/colp/seeds/_seeds.yml b/products/colp/seeds/_seeds.yml new file mode 100644 index 0000000000..5d46565b33 --- /dev/null +++ b/products/colp/seeds/_seeds.yml @@ -0,0 +1,11 @@ +version: 2 + +seeds: + - name: usetype_mappings + description: | + Mapping table for cleaning IPIS use type values. + Maps source use type names to standardized cleaned versions. + config: + column_types: + source: text + cleaned: text diff --git a/products/colp/seeds/usetype_mappings.csv b/products/colp/seeds/usetype_mappings.csv new file mode 100644 index 0000000000..2b93cde082 --- /dev/null +++ b/products/colp/seeds/usetype_mappings.csv @@ -0,0 +1,82 @@ +source,cleaned +NATURAL AREA/ETC,NATURAL AREA/WETLAND/WILDLIFE REFUGE +IN USE-RES STRUCTURE,IN USE-RESIDENTIAL STRUCTURE +PLAYGRND/SPORTS AREA,PLAYGROUND/SPORTS AREA +AGCY EXECUTVE OFFICE,AGENCY EXECUTIVE OFFICE +AGCY OFFICE,AGENCY OFFICE +NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY) +HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP +OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY +NO USE-NON RESIDENTIAL STRUCTURES,NO USE-NON RESIDENTIAL STRUCTURE +NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURE +COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY +MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC +INDOOR PKNG GARAGE,INDOOR PARKING GARAGE +UNDEVELOP OPEN SPACE,UNDEVELOPED OPEN SPACE +OUTDOOR PKNG-AGENCY,OUTDOOR PARKING - AGENCY VEHICLES +TRANSL HSG - FAMILY,TRANSITIONAL HOUSING - FAMILY +RECREATIONL FACILITY,RECREATIONAL FACILITY +NO USE-RES STRUCTURE,NO USE-RESIDENTIAL STRUCTURE +WASTEWATER PUMPING,WASTEWATER PUMPING STATION +OUTDR PKNG-EMPLOYEES,OUTDOOR PARKING - EMPLOYEES +SPECIAL EDUC SCHOOL,SPECIAL EDUCATION SCHOOL +OTHER PUB SAFETY FAC,OTHER PUBLIC SAFETY FACILITY +OTHER CULTURAL FAC,OTHER CULTURAL FACILITY +AMBUL HEALTH FACIL,AMBULATORY HEALTH FACILITY +INDOOR MAINT/STORAGE,INDOOR MAINTENANCE/STORAGE/GARAGE FACILITY +EARLY CHILDHD CENTER,EARLY CHILDHOOD CENTER +INDOOR STORAGE(WHSE),INDOOR STORAGE (WAREHOUSE) +POLLUTION CONTRL FAC,POLLUTION CONTROL FACILITY +TRANSL HSG - ADULT,TRANSITIONAL HOUSING - ADULT +VOCATION HIGH SCHOOL,VOCATIONAL HIGH SCHOOL +WATER POLL CNTRL PLT,WATER POLLUTION CONTROL PLANT +OUTDOOR STRG-EQUIP,OUTDOOR STORAGE - EQUIPMENT +OUTDOOR PKNG-CLIENTS,OUTDOOR PARKING - CLIENTS +OTHER HEALTH FACILTY,OTHER HEALTH FACILITY +RECREATION CTR/GYM,RECREATION CENTER/GYMNASIUM +OTH WATER SUPPLY FAC,OTHER WATER SUPPLY FACILITY +OFC LEASED BY EDUC,OFFICE LEASED BY EDUC +INDOOR STORAGE-EQUIP,INDOOR STORAGE - EQUIPMENT +SECTION STATION,SANITATION SECTION STATION +FACILITIES MAINT,FACILITIES MAINTENANCE +OUTDOOR MAINT/STORGE,OUTDOOR MAINTENANCE/STORAGE FACILITY +OTHER COLLEGE/UNIV,OTHER COLLEGE/UNIVERSITY +OUTDOOR STORAGE-BULK,OUTDOOR STORAGE - BULK MATERIAL +ALTERNTV HIGH SCHOOL,ALTERNATIVE HIGH SCHOOL +OTHR SPC REC-OUTDOOR,OTHER SPECIAL RECREATIONAL FACILITY - OUTDOOR +SOLID WASTE TSFR STN,SOLID WASTE TRANSFER STATION +SPECIAL REC-OUTDOOR,SPECIAL RECREATIONAL FACILITY - OUTDOOR +MUNICIPL PKNG-INDR,MUNICIPAL PARKING - INDOOR PUBLIC +WATER SUPPLY FACIL,WATER SUPPLY FACILITY +NEIGHBORHD SIT AREA,NEIGHBORHOOD SITTING AREA +SPEC RECREATN-INDOOR,SPECIAL RECREATIONAL FACILITY - INDOOR +SLUDGE DE-WATERING,SLUDGE DE-WATERING FACILITY +PUB SAFETY FACILITY,PUBLIC SAFETY FACILITY +OTH OUTDOOR STORAGE,OTHER OUTDOOR STORAGE +TRANS/PUBLIC PARKING,TRANSPORTATION/PUBLIC PARKING FACILITY +INDOOR STRG-PRINTED,INDOOR STORAGE - PRINTED MATERIAL/SUPPLIES +EDUC SKILLS CENTER,EDUCATIONAL SKILLS CENTER +OTHER POLL CNTRL FAC,OTHER POLLUTION CONTROL FACILITY +OTH DETENTION FACIL,OTHER DETENTION FACILITY +OTH OUTDR MAINT/STG,OTHER OUTDOOR MAINTENANCE/STORAGE FACILITY +TRANSL HSG - MEN,TRANSITIONAL HOUSING - ADULT MEN +NON-RES SOC SERV FAC,NON-RESIDENTIAL SOCIAL SERVICE FACILITY +O/D STRG-AGY VEHICLE,OUTDOOR STORAGE - AGENCY VEHICLES +INDOOR STRG-SUPPLIES,INDOOR STORAGE - SUPPLIES +OTHER REC FACILITY,OTHER RECREATIONAL FACILITY +OTH INDOOR MAINT/STG,OTHER INDOOR MAINTENANCE/STORAGE FACILITY +INDOOR STORAGE-BULK,INDOOR STORAGE - BULK MATERIAL +RES HEALTH CARE FAC,RESIDENTIAL HEALTH CARE FACILITY +OTH SECURE DETENTION,OTHER SECURE DETENTION FACILITY +SOCIAL SERVICE FACIL,SOCIAL SERVICE FACILITY +TRAIN FACIL-OUTDOOR,TRAINING FACILITY - OUTDOOR +TRAIN FAC-INDOOR,TRAINING FACILITY - INDOOR +TRANSL HSG - WOMEN,TRANSITIONAL HOUSING - ADULT WOMEN +STORMWATER PUMPING,STORMWATER PUMPING STATION +RES FACIL - CHILDREN,RESIDENTIAL FACILITY FOR CHILDREN +OTHER TESTING FACIL,OTHER TESTING FACILITY +OTH WATER TRANS FAC,OTHER WATERFRONT TRANSPORTATION FACILITY +CRIMINAL JUSTICE FAC,CRIMINAL JUSTICE FACILITY +OTH SOLID WASTE FAC,OTHER SOLID WASTE FACILITY +OUTDOOR TOW POUND,OUTDOOR PARKING - TOW POUND +OTHER SPC REC-INDOOR,OTHER SPECIAL RECREATIONAL FACILITY - INDOOR diff --git a/products/colp/sql/create_colp.sql b/products/colp/sql/create_colp.sql index 2c9342720f..da298c91b8 100644 --- a/products/colp/sql/create_colp.sql +++ b/products/colp/sql/create_colp.sql @@ -205,6 +205,39 @@ normed_name_merge AS ( ON a._parcelname = b.old_name ), +usetype_mapped AS ( + SELECT + a.uid, + a.borough, + a.block, + a.lot, + a.bbl, + a.geo_bbl, + a.mapbbl, + a.hnum, + a._sname, + a._parcelname, + a.parcelname, + a.agency, + a._usecode, + -- Apply usetype mappings, keeping original if no mapping exists + coalesce(b.cleaned, a._usetype) AS _usetype, + a.ownership, + a.leased, + a.finalcom, + a.agreement, + a.xcoord, + a.ycoord, + a.latitude, + a.longitude, + a.geom, + a.cd, + a.sname + FROM normed_name_merge AS a + LEFT JOIN usetype_mappings AS b + ON a._usetype = b.source +), + categorized AS ( SELECT a.*, @@ -309,7 +342,7 @@ categorized AS ( a._usecode = '1410' OR a._usecode = '1400' THEN 'PROPERTY WITH RESIDENTIAL USE' END) AS excatdesc - FROM normed_name_merge AS a + FROM usetype_mapped AS a ) -- Reorder columns for output