From 7cb774061324bba947aef885620c673f93a22394 Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 10:18:00 -0400 Subject: [PATCH 01/11] add initial usetypes mapping --- products/colp/data/usetype_mappings.csv | 83 +++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 products/colp/data/usetype_mappings.csv diff --git a/products/colp/data/usetype_mappings.csv b/products/colp/data/usetype_mappings.csv new file mode 100644 index 0000000000..ab3a3c5cf3 --- /dev/null +++ b/products/colp/data/usetype_mappings.csv @@ -0,0 +1,83 @@ +"curr","old","cnt","row_number" +NATURAL AREA/ETC,NATURAL AREA/WETLAND/WILDLIFE REFUGE,2593,1 +IN USE-RES STRUCTURE,IN USE-RESIDENTIAL STRUCTURE,1178,1 +PLAYGRND/SPORTS AREA,PLAYGROUND/SPORTS AREA,442,1 +AGCY EXECUTVE OFFICE,AGENCY EXECUTIVE OFFICE,348,1 +AGCY OFFICE,AGENCY OFFICE,256,1 +NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY),137,1 +HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP,121,1 +OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY,99,1 +NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURESE,91,1 +COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY,89,1 +MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC,68,1 +INDOOR PKNG GARAGE,INDOOR PARKING GARAGE,52,1 +UNDEVELOP OPEN SPACE,UNDEVELOPED OPEN SPACE,52,1 +OUTDOOR PKNG-AGENCY,OUTDOOR PARKING - AGENCY VEHICLES,49,1 +TRANSL HSG - FAMILY,TRANSITIONAL HOUSING - FAMILY,47,1 +RECREATIONL FACILITY,RECREATIONAL FACILITY,46,1 +NO USE-RES STRUCTURE,NO USE-RESIDENTIAL STRUCTURE,43,1 +WASTEWATER PUMPING,WASTEWATER PUMPING STATION,43,1 +OUTDR PKNG-EMPLOYEES,OUTDOOR PARKING - EMPLOYEES,41,1 +SPECIAL EDUC SCHOOL,SPECIAL EDUCATION SCHOOL,36,1 +OTHER PUB SAFETY FAC,OTHER PUBLIC SAFETY FACILITY,35,1 +OTHER CULTURAL FAC,OTHER CULTURAL FACILITY,34,1 +AMBUL HEALTH FACIL,AMBULATORY HEALTH FACILITY,32,1 +INDOOR MAINT/STORAGE,INDOOR MAINTENANCE/STORAGE/GARAGE FACILITY,32,1 +EARLY CHILDHD CENTER,EARLY CHILDHOOD CENTER,31,1 +INDOOR STORAGE(WHSE),INDOOR STORAGE (WAREHOUSE),30,1 +POLLUTION CONTRL FAC,POLLUTION CONTROL FACILITY,27,1 +TRANSL HSG - ADULT,TRANSITIONAL HOUSING - ADULT,24,1 +VOCATION HIGH SCHOOL,VOCATIONAL HIGH SCHOOL,23,1 +WATER POLL CNTRL PLT,WATER POLLUTION CONTROL PLANT,23,1 +OUTDOOR STRG-EQUIP,OUTDOOR STORAGE - EQUIPMENT,19,1 +OUTDOOR PKNG-CLIENTS,OUTDOOR PARKING - CLIENTS,18,1 +OTHER HEALTH FACILTY,OTHER HEALTH FACILITY,16,1 +RECREATION CTR/GYM,RECREATION CENTER/GYMNASIUM,15,1 +OTH WATER SUPPLY FAC,OTHER WATER SUPPLY FACILITY,15,1 +OFC LEASED BY EDUC,OFFICE LEASED BY EDUC,13,1 +INDOOR STORAGE-EQUIP,INDOOR STORAGE - EQUIPMENT,13,1 +SECTION STATION,SANITATION SECTION STATION,12,1 +FACILITIES MAINT,FACILITIES MAINTENANCE,11,1 +OUTDOOR MAINT/STORGE,OUTDOOR MAINTENANCE/STORAGE FACILITY,11,1 +OTHER COLLEGE/UNIV,OTHER COLLEGE/UNIVERSITY,10,1 +EMERGENCY MEDICL STN,OFFICE,10,1 +OUTDOOR STORAGE-BULK,OUTDOOR STORAGE - BULK MATERIAL,9,1 +ALTERNTV HIGH SCHOOL,ALTERNATIVE HIGH SCHOOL,9,1 +OTHR SPC REC-OUTDOOR,OTHER SPECIAL RECREATIONAL FACILITY - OUTDOOR,9,1 +SOLID WASTE TSFR STN,SOLID WASTE TRANSFER STATION,9,1 +SPECIAL REC-OUTDOOR,SPECIAL RECREATIONAL FACILITY - OUTDOOR,9,1 +MUNICIPL PKNG-INDR,MUNICIPAL PARKING - INDOOR PUBLIC,8,1 +WATER SUPPLY FACIL,WATER SUPPLY FACILITY,8,1 +NEIGHBORHD SIT AREA,NEIGHBORHOOD SITTING AREA,7,1 +SPEC RECREATN-INDOOR,SPECIAL RECREATIONAL FACILITY - INDOOR,7,1 +SLUDGE DE-WATERING,SLUDGE DE-WATERING FACILITY,7,1 +COURT (SPORTS),TENNIS COURT BASKETBALL COURT ETC,7,1 +PUB SAFETY FACILITY,PUBLIC SAFETY FACILITY,6,1 +OTH OUTDOOR STORAGE,OTHER OUTDOOR STORAGE,6,1 +TRANS/PUBLIC PARKING,TRANSPORTATION/PUBLIC PARKING FACILITY,6,1 +INDOOR STRG-PRINTED,INDOOR STORAGE - PRINTED MATERIAL/SUPPLIES,4,1 +EDUC SKILLS CENTER,EDUCATIONAL SKILLS CENTER,4,1 +OTHER POLL CNTRL FAC,OTHER POLLUTION CONTROL FACILITY,4,1 +OTH DETENTION FACIL,OTHER DETENTION FACILITY,4,1 +OTH OUTDR MAINT/STG,OTHER OUTDOOR MAINTENANCE/STORAGE FACILITY,4,1 +TRANSL HSG - MEN,TRANSITIONAL HOUSING - ADULT MEN,4,1 +NON-RES SOC SERV FAC,NON-RESIDENTIAL SOCIAL SERVICE FACILITY,3,1 +O/D STRG-AGY VEHICLE,OUTDOOR STORAGE - AGENCY VEHICLES,3,1 +INDOOR STRG-SUPPLIES,INDOOR STORAGE - SUPPLIES,3,1 +OTHER REC FACILITY,OTHER RECREATIONAL FACILITY,3,1 +OUTDOOR TOW POUND,NO USE,3,1 +CRIMINAL JUSTICE FAC,OFFICE,3,1 +OTH INDOOR MAINT/STG,OTHER INDOOR MAINTENANCE/STORAGE FACILITY,3,1 +INDOOR STORAGE-BULK,INDOOR STORAGE - BULK MATERIAL,3,1 +RES HEALTH CARE FAC,RESIDENTIAL HEALTH CARE FACILITY,3,1 +OTH SECURE DETENTION,OTHER SECURE DETENTION FACILITY,2,1 +SOCIAL SERVICE FACIL,SOCIAL SERVICE FACILITY,2,1 +TRAIN FAC-INDOOR,INDOOR MAINTENANCE/STORAGE/GARAGE FACILITY,1,1 +TRAIN FACIL-OUTDOOR,TRAINING FACILITY - OUTDOOR,1,1 +TRANSL HSG - WOMEN,TRANSITIONAL HOUSING - ADULT WOMEN,1,1 +STORMWATER PUMPING,STORMWATER PUMPING STATION,1,1 +RES FACIL - CHILDREN,RESIDENTIAL FACILITY FOR CHILDREN,1,1 +OTHER SPC REC-INDOOR,THEATER,1,1 +OTHER TESTING FACIL,OTHER TESTING FACILITY,1,1 +OTH SOLID WASTE FAC,PARK,1,1 +OTH WATER TRANS FAC,OTHER WATERFRONT TRANSPORTATION FACILITY,1,1 From 158d02a6d6bb9e42ab2a90ab652e9cc87cf9042e Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 11:02:25 -0400 Subject: [PATCH 02/11] trim the uses --- products/colp/data/usetype_mappings.csv | 159 +++++++++++------------- 1 file changed, 76 insertions(+), 83 deletions(-) diff --git a/products/colp/data/usetype_mappings.csv b/products/colp/data/usetype_mappings.csv index ab3a3c5cf3..789cca341e 100644 --- a/products/colp/data/usetype_mappings.csv +++ b/products/colp/data/usetype_mappings.csv @@ -1,83 +1,76 @@ -"curr","old","cnt","row_number" -NATURAL AREA/ETC,NATURAL AREA/WETLAND/WILDLIFE REFUGE,2593,1 -IN USE-RES STRUCTURE,IN USE-RESIDENTIAL STRUCTURE,1178,1 -PLAYGRND/SPORTS AREA,PLAYGROUND/SPORTS AREA,442,1 -AGCY EXECUTVE OFFICE,AGENCY EXECUTIVE OFFICE,348,1 -AGCY OFFICE,AGENCY OFFICE,256,1 -NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY),137,1 -HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP,121,1 -OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY,99,1 -NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURESE,91,1 -COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY,89,1 -MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC,68,1 -INDOOR PKNG GARAGE,INDOOR PARKING GARAGE,52,1 -UNDEVELOP OPEN SPACE,UNDEVELOPED OPEN SPACE,52,1 -OUTDOOR PKNG-AGENCY,OUTDOOR PARKING - AGENCY VEHICLES,49,1 -TRANSL HSG - FAMILY,TRANSITIONAL HOUSING - FAMILY,47,1 -RECREATIONL FACILITY,RECREATIONAL FACILITY,46,1 -NO USE-RES STRUCTURE,NO USE-RESIDENTIAL STRUCTURE,43,1 -WASTEWATER PUMPING,WASTEWATER PUMPING STATION,43,1 -OUTDR PKNG-EMPLOYEES,OUTDOOR PARKING - EMPLOYEES,41,1 -SPECIAL EDUC SCHOOL,SPECIAL EDUCATION SCHOOL,36,1 -OTHER PUB SAFETY FAC,OTHER PUBLIC SAFETY FACILITY,35,1 -OTHER CULTURAL FAC,OTHER CULTURAL FACILITY,34,1 -AMBUL HEALTH FACIL,AMBULATORY HEALTH FACILITY,32,1 -INDOOR MAINT/STORAGE,INDOOR MAINTENANCE/STORAGE/GARAGE FACILITY,32,1 -EARLY CHILDHD CENTER,EARLY CHILDHOOD CENTER,31,1 -INDOOR STORAGE(WHSE),INDOOR STORAGE (WAREHOUSE),30,1 -POLLUTION CONTRL FAC,POLLUTION CONTROL FACILITY,27,1 -TRANSL HSG - ADULT,TRANSITIONAL HOUSING - ADULT,24,1 -VOCATION HIGH SCHOOL,VOCATIONAL HIGH SCHOOL,23,1 -WATER POLL CNTRL PLT,WATER POLLUTION CONTROL PLANT,23,1 -OUTDOOR STRG-EQUIP,OUTDOOR STORAGE - EQUIPMENT,19,1 -OUTDOOR PKNG-CLIENTS,OUTDOOR PARKING - CLIENTS,18,1 -OTHER HEALTH FACILTY,OTHER HEALTH FACILITY,16,1 -RECREATION CTR/GYM,RECREATION CENTER/GYMNASIUM,15,1 -OTH WATER SUPPLY FAC,OTHER WATER SUPPLY FACILITY,15,1 -OFC LEASED BY EDUC,OFFICE LEASED BY EDUC,13,1 -INDOOR STORAGE-EQUIP,INDOOR STORAGE - EQUIPMENT,13,1 -SECTION STATION,SANITATION SECTION STATION,12,1 -FACILITIES MAINT,FACILITIES MAINTENANCE,11,1 -OUTDOOR MAINT/STORGE,OUTDOOR MAINTENANCE/STORAGE FACILITY,11,1 -OTHER COLLEGE/UNIV,OTHER COLLEGE/UNIVERSITY,10,1 -EMERGENCY MEDICL STN,OFFICE,10,1 -OUTDOOR STORAGE-BULK,OUTDOOR STORAGE - BULK MATERIAL,9,1 -ALTERNTV HIGH SCHOOL,ALTERNATIVE HIGH SCHOOL,9,1 -OTHR SPC REC-OUTDOOR,OTHER SPECIAL RECREATIONAL FACILITY - OUTDOOR,9,1 -SOLID WASTE TSFR STN,SOLID WASTE TRANSFER STATION,9,1 -SPECIAL REC-OUTDOOR,SPECIAL RECREATIONAL FACILITY - OUTDOOR,9,1 -MUNICIPL PKNG-INDR,MUNICIPAL PARKING - INDOOR PUBLIC,8,1 -WATER SUPPLY FACIL,WATER SUPPLY FACILITY,8,1 -NEIGHBORHD SIT AREA,NEIGHBORHOOD SITTING AREA,7,1 -SPEC RECREATN-INDOOR,SPECIAL RECREATIONAL FACILITY - INDOOR,7,1 -SLUDGE DE-WATERING,SLUDGE DE-WATERING FACILITY,7,1 -COURT (SPORTS),TENNIS COURT BASKETBALL COURT ETC,7,1 -PUB SAFETY FACILITY,PUBLIC SAFETY FACILITY,6,1 -OTH OUTDOOR STORAGE,OTHER OUTDOOR STORAGE,6,1 -TRANS/PUBLIC PARKING,TRANSPORTATION/PUBLIC PARKING FACILITY,6,1 -INDOOR STRG-PRINTED,INDOOR STORAGE - PRINTED MATERIAL/SUPPLIES,4,1 -EDUC SKILLS CENTER,EDUCATIONAL SKILLS CENTER,4,1 -OTHER POLL CNTRL FAC,OTHER POLLUTION CONTROL FACILITY,4,1 -OTH DETENTION FACIL,OTHER DETENTION FACILITY,4,1 -OTH OUTDR MAINT/STG,OTHER OUTDOOR MAINTENANCE/STORAGE FACILITY,4,1 -TRANSL HSG - MEN,TRANSITIONAL HOUSING - ADULT MEN,4,1 -NON-RES SOC SERV FAC,NON-RESIDENTIAL SOCIAL SERVICE FACILITY,3,1 -O/D STRG-AGY VEHICLE,OUTDOOR STORAGE - AGENCY VEHICLES,3,1 -INDOOR STRG-SUPPLIES,INDOOR STORAGE - SUPPLIES,3,1 -OTHER REC FACILITY,OTHER RECREATIONAL FACILITY,3,1 -OUTDOOR TOW POUND,NO USE,3,1 -CRIMINAL JUSTICE FAC,OFFICE,3,1 -OTH INDOOR MAINT/STG,OTHER INDOOR MAINTENANCE/STORAGE FACILITY,3,1 -INDOOR STORAGE-BULK,INDOOR STORAGE - BULK MATERIAL,3,1 -RES HEALTH CARE FAC,RESIDENTIAL HEALTH CARE FACILITY,3,1 -OTH SECURE DETENTION,OTHER SECURE DETENTION FACILITY,2,1 -SOCIAL SERVICE FACIL,SOCIAL SERVICE FACILITY,2,1 -TRAIN FAC-INDOOR,INDOOR MAINTENANCE/STORAGE/GARAGE FACILITY,1,1 -TRAIN FACIL-OUTDOOR,TRAINING FACILITY - OUTDOOR,1,1 -TRANSL HSG - WOMEN,TRANSITIONAL HOUSING - ADULT WOMEN,1,1 -STORMWATER PUMPING,STORMWATER PUMPING STATION,1,1 -RES FACIL - CHILDREN,RESIDENTIAL FACILITY FOR CHILDREN,1,1 -OTHER SPC REC-INDOOR,THEATER,1,1 -OTHER TESTING FACIL,OTHER TESTING FACILITY,1,1 -OTH SOLID WASTE FAC,PARK,1,1 -OTH WATER TRANS FAC,OTHER WATERFRONT TRANSPORTATION FACILITY,1,1 +source,cleaned +NATURAL AREA/ETC,NATURAL AREA/WETLAND/WILDLIFE REFUGE +IN USE-RES STRUCTURE,IN USE-RESIDENTIAL STRUCTURE +PLAYGRND/SPORTS AREA,PLAYGROUND/SPORTS AREA +AGCY EXECUTVE OFFICE,AGENCY EXECUTIVE OFFICE +AGCY OFFICE,AGENCY OFFICE +NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY) +HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP +OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY +NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURESE +COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY +MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC +INDOOR PKNG GARAGE,INDOOR PARKING GARAGE +UNDEVELOP OPEN SPACE,UNDEVELOPED OPEN SPACE +OUTDOOR PKNG-AGENCY,OUTDOOR PARKING - AGENCY VEHICLES +TRANSL HSG - FAMILY,TRANSITIONAL HOUSING - FAMILY +RECREATIONL FACILITY,RECREATIONAL FACILITY +NO USE-RES STRUCTURE,NO USE-RESIDENTIAL STRUCTURE +WASTEWATER PUMPING,WASTEWATER PUMPING STATION +OUTDR PKNG-EMPLOYEES,OUTDOOR PARKING - EMPLOYEES +SPECIAL EDUC SCHOOL,SPECIAL EDUCATION SCHOOL +OTHER PUB SAFETY FAC,OTHER PUBLIC SAFETY FACILITY +OTHER CULTURAL FAC,OTHER CULTURAL FACILITY +AMBUL HEALTH FACIL,AMBULATORY HEALTH FACILITY +INDOOR MAINT/STORAGE,INDOOR MAINTENANCE/STORAGE/GARAGE FACILITY +EARLY CHILDHD CENTER,EARLY CHILDHOOD CENTER +INDOOR STORAGE(WHSE),INDOOR STORAGE (WAREHOUSE) +POLLUTION CONTRL FAC,POLLUTION CONTROL FACILITY +TRANSL HSG - ADULT,TRANSITIONAL HOUSING - ADULT +VOCATION HIGH SCHOOL,VOCATIONAL HIGH SCHOOL +WATER POLL CNTRL PLT,WATER POLLUTION CONTROL PLANT +OUTDOOR STRG-EQUIP,OUTDOOR STORAGE - EQUIPMENT +OUTDOOR PKNG-CLIENTS,OUTDOOR PARKING - CLIENTS +OTHER HEALTH FACILTY,OTHER HEALTH FACILITY +RECREATION CTR/GYM,RECREATION CENTER/GYMNASIUM +OTH WATER SUPPLY FAC,OTHER WATER SUPPLY FACILITY +OFC LEASED BY EDUC,OFFICE LEASED BY EDUC +INDOOR STORAGE-EQUIP,INDOOR STORAGE - EQUIPMENT +SECTION STATION,SANITATION SECTION STATION +FACILITIES MAINT,FACILITIES MAINTENANCE +OUTDOOR MAINT/STORGE,OUTDOOR MAINTENANCE/STORAGE FACILITY +OTHER COLLEGE/UNIV,OTHER COLLEGE/UNIVERSITY +OUTDOOR STORAGE-BULK,OUTDOOR STORAGE - BULK MATERIAL +ALTERNTV HIGH SCHOOL,ALTERNATIVE HIGH SCHOOL +OTHR SPC REC-OUTDOOR,OTHER SPECIAL RECREATIONAL FACILITY - OUTDOOR +SOLID WASTE TSFR STN,SOLID WASTE TRANSFER STATION +SPECIAL REC-OUTDOOR,SPECIAL RECREATIONAL FACILITY - OUTDOOR +MUNICIPL PKNG-INDR,MUNICIPAL PARKING - INDOOR PUBLIC +WATER SUPPLY FACIL,WATER SUPPLY FACILITY +NEIGHBORHD SIT AREA,NEIGHBORHOOD SITTING AREA +SPEC RECREATN-INDOOR,SPECIAL RECREATIONAL FACILITY - INDOOR +SLUDGE DE-WATERING,SLUDGE DE-WATERING FACILITY +PUB SAFETY FACILITY,PUBLIC SAFETY FACILITY +OTH OUTDOOR STORAGE,OTHER OUTDOOR STORAGE +TRANS/PUBLIC PARKING,TRANSPORTATION/PUBLIC PARKING FACILITY +INDOOR STRG-PRINTED,INDOOR STORAGE - PRINTED MATERIAL/SUPPLIES +EDUC SKILLS CENTER,EDUCATIONAL SKILLS CENTER +OTHER POLL CNTRL FAC,OTHER POLLUTION CONTROL FACILITY +OTH DETENTION FACIL,OTHER DETENTION FACILITY +OTH OUTDR MAINT/STG,OTHER OUTDOOR MAINTENANCE/STORAGE FACILITY +TRANSL HSG - MEN,TRANSITIONAL HOUSING - ADULT MEN +NON-RES SOC SERV FAC,NON-RESIDENTIAL SOCIAL SERVICE FACILITY +O/D STRG-AGY VEHICLE,OUTDOOR STORAGE - AGENCY VEHICLES +INDOOR STRG-SUPPLIES,INDOOR STORAGE - SUPPLIES +OTHER REC FACILITY,OTHER RECREATIONAL FACILITY +OTH INDOOR MAINT/STG,OTHER INDOOR MAINTENANCE/STORAGE FACILITY +INDOOR STORAGE-BULK,INDOOR STORAGE - BULK MATERIAL +RES HEALTH CARE FAC,RESIDENTIAL HEALTH CARE FACILITY +OTH SECURE DETENTION,OTHER SECURE DETENTION FACILITY +SOCIAL SERVICE FACIL,SOCIAL SERVICE FACILITY +TRAIN FACIL-OUTDOOR,TRAINING FACILITY - OUTDOOR +TRANSL HSG - WOMEN,TRANSITIONAL HOUSING - ADULT WOMEN +STORMWATER PUMPING,STORMWATER PUMPING STATION +RES FACIL - CHILDREN,RESIDENTIAL FACILITY FOR CHILDREN +OTHER TESTING FACIL,OTHER TESTING FACILITY +OTH WATER TRANS FAC,OTHER WATERFRONT TRANSPORTATION FACILITY \ No newline at end of file From 7434ca6802e8e0cc354b07542d4ced844036f86b Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 11:10:36 -0400 Subject: [PATCH 03/11] remove carriage-returns in usetype_mappings.csv --- products/colp/data/usetype_mappings.csv | 152 ++++++++++++------------ 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/products/colp/data/usetype_mappings.csv b/products/colp/data/usetype_mappings.csv index 789cca341e..331c4c0516 100644 --- a/products/colp/data/usetype_mappings.csv +++ b/products/colp/data/usetype_mappings.csv @@ -1,76 +1,76 @@ -source,cleaned -NATURAL AREA/ETC,NATURAL AREA/WETLAND/WILDLIFE REFUGE -IN USE-RES STRUCTURE,IN USE-RESIDENTIAL STRUCTURE -PLAYGRND/SPORTS AREA,PLAYGROUND/SPORTS AREA -AGCY EXECUTVE OFFICE,AGENCY EXECUTIVE OFFICE -AGCY OFFICE,AGENCY OFFICE -NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY) -HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP -OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY -NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURESE -COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY -MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC -INDOOR PKNG GARAGE,INDOOR PARKING GARAGE -UNDEVELOP OPEN SPACE,UNDEVELOPED OPEN SPACE -OUTDOOR PKNG-AGENCY,OUTDOOR PARKING - AGENCY VEHICLES -TRANSL HSG - FAMILY,TRANSITIONAL HOUSING - FAMILY -RECREATIONL FACILITY,RECREATIONAL FACILITY -NO USE-RES STRUCTURE,NO USE-RESIDENTIAL STRUCTURE -WASTEWATER PUMPING,WASTEWATER PUMPING STATION -OUTDR PKNG-EMPLOYEES,OUTDOOR PARKING - EMPLOYEES -SPECIAL EDUC SCHOOL,SPECIAL EDUCATION SCHOOL -OTHER PUB SAFETY FAC,OTHER PUBLIC SAFETY FACILITY -OTHER CULTURAL FAC,OTHER CULTURAL FACILITY -AMBUL HEALTH FACIL,AMBULATORY HEALTH FACILITY -INDOOR MAINT/STORAGE,INDOOR MAINTENANCE/STORAGE/GARAGE FACILITY -EARLY CHILDHD CENTER,EARLY CHILDHOOD CENTER -INDOOR STORAGE(WHSE),INDOOR STORAGE (WAREHOUSE) -POLLUTION CONTRL FAC,POLLUTION CONTROL FACILITY -TRANSL HSG - ADULT,TRANSITIONAL HOUSING - ADULT -VOCATION HIGH SCHOOL,VOCATIONAL HIGH SCHOOL -WATER POLL CNTRL PLT,WATER POLLUTION CONTROL PLANT -OUTDOOR STRG-EQUIP,OUTDOOR STORAGE - EQUIPMENT -OUTDOOR PKNG-CLIENTS,OUTDOOR PARKING - CLIENTS -OTHER HEALTH FACILTY,OTHER HEALTH FACILITY -RECREATION CTR/GYM,RECREATION CENTER/GYMNASIUM -OTH WATER SUPPLY FAC,OTHER WATER SUPPLY FACILITY -OFC LEASED BY EDUC,OFFICE LEASED BY EDUC -INDOOR STORAGE-EQUIP,INDOOR STORAGE - EQUIPMENT -SECTION STATION,SANITATION SECTION STATION -FACILITIES MAINT,FACILITIES MAINTENANCE -OUTDOOR MAINT/STORGE,OUTDOOR MAINTENANCE/STORAGE FACILITY -OTHER COLLEGE/UNIV,OTHER COLLEGE/UNIVERSITY -OUTDOOR STORAGE-BULK,OUTDOOR STORAGE - BULK MATERIAL -ALTERNTV HIGH SCHOOL,ALTERNATIVE HIGH SCHOOL -OTHR SPC REC-OUTDOOR,OTHER SPECIAL RECREATIONAL FACILITY - OUTDOOR -SOLID WASTE TSFR STN,SOLID WASTE TRANSFER STATION -SPECIAL REC-OUTDOOR,SPECIAL RECREATIONAL FACILITY - OUTDOOR -MUNICIPL PKNG-INDR,MUNICIPAL PARKING - INDOOR PUBLIC -WATER SUPPLY FACIL,WATER SUPPLY FACILITY -NEIGHBORHD SIT AREA,NEIGHBORHOOD SITTING AREA -SPEC RECREATN-INDOOR,SPECIAL RECREATIONAL FACILITY - INDOOR -SLUDGE DE-WATERING,SLUDGE DE-WATERING FACILITY -PUB SAFETY FACILITY,PUBLIC SAFETY FACILITY -OTH OUTDOOR STORAGE,OTHER OUTDOOR STORAGE -TRANS/PUBLIC PARKING,TRANSPORTATION/PUBLIC PARKING FACILITY -INDOOR STRG-PRINTED,INDOOR STORAGE - PRINTED MATERIAL/SUPPLIES -EDUC SKILLS CENTER,EDUCATIONAL SKILLS CENTER -OTHER POLL CNTRL FAC,OTHER POLLUTION CONTROL FACILITY -OTH DETENTION FACIL,OTHER DETENTION FACILITY -OTH OUTDR MAINT/STG,OTHER OUTDOOR MAINTENANCE/STORAGE FACILITY -TRANSL HSG - MEN,TRANSITIONAL HOUSING - ADULT MEN -NON-RES SOC SERV FAC,NON-RESIDENTIAL SOCIAL SERVICE FACILITY -O/D STRG-AGY VEHICLE,OUTDOOR STORAGE - AGENCY VEHICLES -INDOOR STRG-SUPPLIES,INDOOR STORAGE - SUPPLIES -OTHER REC FACILITY,OTHER RECREATIONAL FACILITY -OTH INDOOR MAINT/STG,OTHER INDOOR MAINTENANCE/STORAGE FACILITY -INDOOR STORAGE-BULK,INDOOR STORAGE - BULK MATERIAL -RES HEALTH CARE FAC,RESIDENTIAL HEALTH CARE FACILITY -OTH SECURE DETENTION,OTHER SECURE DETENTION FACILITY -SOCIAL SERVICE FACIL,SOCIAL SERVICE FACILITY -TRAIN FACIL-OUTDOOR,TRAINING FACILITY - OUTDOOR -TRANSL HSG - WOMEN,TRANSITIONAL HOUSING - ADULT WOMEN -STORMWATER PUMPING,STORMWATER PUMPING STATION -RES FACIL - CHILDREN,RESIDENTIAL FACILITY FOR CHILDREN -OTHER TESTING FACIL,OTHER TESTING FACILITY -OTH WATER TRANS FAC,OTHER WATERFRONT TRANSPORTATION FACILITY \ No newline at end of file +source,cleaned +NATURAL AREA/ETC,NATURAL AREA/WETLAND/WILDLIFE REFUGE +IN USE-RES STRUCTURE,IN USE-RESIDENTIAL STRUCTURE +PLAYGRND/SPORTS AREA,PLAYGROUND/SPORTS AREA +AGCY EXECUTVE OFFICE,AGENCY EXECUTIVE OFFICE +AGCY OFFICE,AGENCY OFFICE +NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY) +HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP +OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY +NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURESE +COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY +MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC +INDOOR PKNG GARAGE,INDOOR PARKING GARAGE +UNDEVELOP OPEN SPACE,UNDEVELOPED OPEN SPACE +OUTDOOR PKNG-AGENCY,OUTDOOR PARKING - AGENCY VEHICLES +TRANSL HSG - FAMILY,TRANSITIONAL HOUSING - FAMILY +RECREATIONL FACILITY,RECREATIONAL FACILITY +NO USE-RES STRUCTURE,NO USE-RESIDENTIAL STRUCTURE +WASTEWATER PUMPING,WASTEWATER PUMPING STATION +OUTDR PKNG-EMPLOYEES,OUTDOOR PARKING - EMPLOYEES +SPECIAL EDUC SCHOOL,SPECIAL EDUCATION SCHOOL +OTHER PUB SAFETY FAC,OTHER PUBLIC SAFETY FACILITY +OTHER CULTURAL FAC,OTHER CULTURAL FACILITY +AMBUL HEALTH FACIL,AMBULATORY HEALTH FACILITY +INDOOR MAINT/STORAGE,INDOOR MAINTENANCE/STORAGE/GARAGE FACILITY +EARLY CHILDHD CENTER,EARLY CHILDHOOD CENTER +INDOOR STORAGE(WHSE),INDOOR STORAGE (WAREHOUSE) +POLLUTION CONTRL FAC,POLLUTION CONTROL FACILITY +TRANSL HSG - ADULT,TRANSITIONAL HOUSING - ADULT +VOCATION HIGH SCHOOL,VOCATIONAL HIGH SCHOOL +WATER POLL CNTRL PLT,WATER POLLUTION CONTROL PLANT +OUTDOOR STRG-EQUIP,OUTDOOR STORAGE - EQUIPMENT +OUTDOOR PKNG-CLIENTS,OUTDOOR PARKING - CLIENTS +OTHER HEALTH FACILTY,OTHER HEALTH FACILITY +RECREATION CTR/GYM,RECREATION CENTER/GYMNASIUM +OTH WATER SUPPLY FAC,OTHER WATER SUPPLY FACILITY +OFC LEASED BY EDUC,OFFICE LEASED BY EDUC +INDOOR STORAGE-EQUIP,INDOOR STORAGE - EQUIPMENT +SECTION STATION,SANITATION SECTION STATION +FACILITIES MAINT,FACILITIES MAINTENANCE +OUTDOOR MAINT/STORGE,OUTDOOR MAINTENANCE/STORAGE FACILITY +OTHER COLLEGE/UNIV,OTHER COLLEGE/UNIVERSITY +OUTDOOR STORAGE-BULK,OUTDOOR STORAGE - BULK MATERIAL +ALTERNTV HIGH SCHOOL,ALTERNATIVE HIGH SCHOOL +OTHR SPC REC-OUTDOOR,OTHER SPECIAL RECREATIONAL FACILITY - OUTDOOR +SOLID WASTE TSFR STN,SOLID WASTE TRANSFER STATION +SPECIAL REC-OUTDOOR,SPECIAL RECREATIONAL FACILITY - OUTDOOR +MUNICIPL PKNG-INDR,MUNICIPAL PARKING - INDOOR PUBLIC +WATER SUPPLY FACIL,WATER SUPPLY FACILITY +NEIGHBORHD SIT AREA,NEIGHBORHOOD SITTING AREA +SPEC RECREATN-INDOOR,SPECIAL RECREATIONAL FACILITY - INDOOR +SLUDGE DE-WATERING,SLUDGE DE-WATERING FACILITY +PUB SAFETY FACILITY,PUBLIC SAFETY FACILITY +OTH OUTDOOR STORAGE,OTHER OUTDOOR STORAGE +TRANS/PUBLIC PARKING,TRANSPORTATION/PUBLIC PARKING FACILITY +INDOOR STRG-PRINTED,INDOOR STORAGE - PRINTED MATERIAL/SUPPLIES +EDUC SKILLS CENTER,EDUCATIONAL SKILLS CENTER +OTHER POLL CNTRL FAC,OTHER POLLUTION CONTROL FACILITY +OTH DETENTION FACIL,OTHER DETENTION FACILITY +OTH OUTDR MAINT/STG,OTHER OUTDOOR MAINTENANCE/STORAGE FACILITY +TRANSL HSG - MEN,TRANSITIONAL HOUSING - ADULT MEN +NON-RES SOC SERV FAC,NON-RESIDENTIAL SOCIAL SERVICE FACILITY +O/D STRG-AGY VEHICLE,OUTDOOR STORAGE - AGENCY VEHICLES +INDOOR STRG-SUPPLIES,INDOOR STORAGE - SUPPLIES +OTHER REC FACILITY,OTHER RECREATIONAL FACILITY +OTH INDOOR MAINT/STG,OTHER INDOOR MAINTENANCE/STORAGE FACILITY +INDOOR STORAGE-BULK,INDOOR STORAGE - BULK MATERIAL +RES HEALTH CARE FAC,RESIDENTIAL HEALTH CARE FACILITY +OTH SECURE DETENTION,OTHER SECURE DETENTION FACILITY +SOCIAL SERVICE FACIL,SOCIAL SERVICE FACILITY +TRAIN FACIL-OUTDOOR,TRAINING FACILITY - OUTDOOR +TRANSL HSG - WOMEN,TRANSITIONAL HOUSING - ADULT WOMEN +STORMWATER PUMPING,STORMWATER PUMPING STATION +RES FACIL - CHILDREN,RESIDENTIAL FACILITY FOR CHILDREN +OTHER TESTING FACIL,OTHER TESTING FACILITY +OTH WATER TRANS FAC,OTHER WATERFRONT TRANSPORTATION FACILITY From 55ea8c3e5238877e44ed27385f909684ed9f0fbb Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 12:19:18 -0400 Subject: [PATCH 04/11] dbt'ify COLP seeds --- products/colp/bash/02_build.sh | 3 + products/colp/dbt_project.yml | 5 ++ products/colp/profiles.yml | 11 ++++ products/colp/seeds/_seeds.yml | 11 ++++ products/colp/seeds/usetype_mappings.csv | 76 ++++++++++++++++++++++++ products/colp/sql/create_colp.sql | 33 ++++++++++ 6 files changed, 139 insertions(+) create mode 100644 products/colp/dbt_project.yml create mode 100644 products/colp/profiles.yml create mode 100644 products/colp/seeds/_seeds.yml create mode 100644 products/colp/seeds/usetype_mappings.csv diff --git a/products/colp/bash/02_build.sh b/products/colp/bash/02_build.sh index 7c28934e7e..f786171ab0 100755 --- a/products/colp/bash/02_build.sh +++ b/products/colp/bash/02_build.sh @@ -1,6 +1,9 @@ #!/bin/bash source bash/config.sh +# Load seeds via dbt +dbt seed --select usetype_mappings + run_sql_file sql/load_modifications.sql run_sql_file sql/geo_inputs.sql diff --git a/products/colp/dbt_project.yml b/products/colp/dbt_project.yml new file mode 100644 index 0000000000..f0c62676d3 --- /dev/null +++ b/products/colp/dbt_project.yml @@ -0,0 +1,5 @@ +name: "colp" + +profile: "dcp-de-postgres" + +seed-paths: ["seeds"] diff --git a/products/colp/profiles.yml b/products/colp/profiles.yml new file mode 100644 index 0000000000..6712525864 --- /dev/null +++ b/products/colp/profiles.yml @@ -0,0 +1,11 @@ +dcp-de-postgres: + target: dev + outputs: + dev: + type: postgres + host: "{{ env_var('BUILD_ENGINE_HOST') }}" + user: "{{ env_var('BUILD_ENGINE_USER') }}" + password: "{{ env_var('BUILD_ENGINE_PASSWORD') }}" + port: "{{ env_var('BUILD_ENGINE_PORT') | as_number }}" + dbname: "{{ env_var('BUILD_ENGINE_DB') }}" + schema: "{{ env_var('BUILD_ENGINE_SCHEMA') }}" diff --git a/products/colp/seeds/_seeds.yml b/products/colp/seeds/_seeds.yml new file mode 100644 index 0000000000..5d46565b33 --- /dev/null +++ b/products/colp/seeds/_seeds.yml @@ -0,0 +1,11 @@ +version: 2 + +seeds: + - name: usetype_mappings + description: | + Mapping table for cleaning IPIS use type values. + Maps source use type names to standardized cleaned versions. + config: + column_types: + source: text + cleaned: text diff --git a/products/colp/seeds/usetype_mappings.csv b/products/colp/seeds/usetype_mappings.csv new file mode 100644 index 0000000000..331c4c0516 --- /dev/null +++ b/products/colp/seeds/usetype_mappings.csv @@ -0,0 +1,76 @@ +source,cleaned +NATURAL AREA/ETC,NATURAL AREA/WETLAND/WILDLIFE REFUGE +IN USE-RES STRUCTURE,IN USE-RESIDENTIAL STRUCTURE +PLAYGRND/SPORTS AREA,PLAYGROUND/SPORTS AREA +AGCY EXECUTVE OFFICE,AGENCY EXECUTIVE OFFICE +AGCY OFFICE,AGENCY OFFICE +NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY) +HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP +OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY +NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURESE +COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY +MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC +INDOOR PKNG GARAGE,INDOOR PARKING GARAGE +UNDEVELOP OPEN SPACE,UNDEVELOPED OPEN SPACE +OUTDOOR PKNG-AGENCY,OUTDOOR PARKING - AGENCY VEHICLES +TRANSL HSG - FAMILY,TRANSITIONAL HOUSING - FAMILY +RECREATIONL FACILITY,RECREATIONAL FACILITY +NO USE-RES STRUCTURE,NO USE-RESIDENTIAL STRUCTURE +WASTEWATER PUMPING,WASTEWATER PUMPING STATION +OUTDR PKNG-EMPLOYEES,OUTDOOR PARKING - EMPLOYEES +SPECIAL EDUC SCHOOL,SPECIAL EDUCATION SCHOOL +OTHER PUB SAFETY FAC,OTHER PUBLIC SAFETY FACILITY +OTHER CULTURAL FAC,OTHER CULTURAL FACILITY +AMBUL HEALTH FACIL,AMBULATORY HEALTH FACILITY +INDOOR MAINT/STORAGE,INDOOR MAINTENANCE/STORAGE/GARAGE FACILITY +EARLY CHILDHD CENTER,EARLY CHILDHOOD CENTER +INDOOR STORAGE(WHSE),INDOOR STORAGE (WAREHOUSE) +POLLUTION CONTRL FAC,POLLUTION CONTROL FACILITY +TRANSL HSG - ADULT,TRANSITIONAL HOUSING - ADULT +VOCATION HIGH SCHOOL,VOCATIONAL HIGH SCHOOL +WATER POLL CNTRL PLT,WATER POLLUTION CONTROL PLANT +OUTDOOR STRG-EQUIP,OUTDOOR STORAGE - EQUIPMENT +OUTDOOR PKNG-CLIENTS,OUTDOOR PARKING - CLIENTS +OTHER HEALTH FACILTY,OTHER HEALTH FACILITY +RECREATION CTR/GYM,RECREATION CENTER/GYMNASIUM +OTH WATER SUPPLY FAC,OTHER WATER SUPPLY FACILITY +OFC LEASED BY EDUC,OFFICE LEASED BY EDUC +INDOOR STORAGE-EQUIP,INDOOR STORAGE - EQUIPMENT +SECTION STATION,SANITATION SECTION STATION +FACILITIES MAINT,FACILITIES MAINTENANCE +OUTDOOR MAINT/STORGE,OUTDOOR MAINTENANCE/STORAGE FACILITY +OTHER COLLEGE/UNIV,OTHER COLLEGE/UNIVERSITY +OUTDOOR STORAGE-BULK,OUTDOOR STORAGE - BULK MATERIAL +ALTERNTV HIGH SCHOOL,ALTERNATIVE HIGH SCHOOL +OTHR SPC REC-OUTDOOR,OTHER SPECIAL RECREATIONAL FACILITY - OUTDOOR +SOLID WASTE TSFR STN,SOLID WASTE TRANSFER STATION +SPECIAL REC-OUTDOOR,SPECIAL RECREATIONAL FACILITY - OUTDOOR +MUNICIPL PKNG-INDR,MUNICIPAL PARKING - INDOOR PUBLIC +WATER SUPPLY FACIL,WATER SUPPLY FACILITY +NEIGHBORHD SIT AREA,NEIGHBORHOOD SITTING AREA +SPEC RECREATN-INDOOR,SPECIAL RECREATIONAL FACILITY - INDOOR +SLUDGE DE-WATERING,SLUDGE DE-WATERING FACILITY +PUB SAFETY FACILITY,PUBLIC SAFETY FACILITY +OTH OUTDOOR STORAGE,OTHER OUTDOOR STORAGE +TRANS/PUBLIC PARKING,TRANSPORTATION/PUBLIC PARKING FACILITY +INDOOR STRG-PRINTED,INDOOR STORAGE - PRINTED MATERIAL/SUPPLIES +EDUC SKILLS CENTER,EDUCATIONAL SKILLS CENTER +OTHER POLL CNTRL FAC,OTHER POLLUTION CONTROL FACILITY +OTH DETENTION FACIL,OTHER DETENTION FACILITY +OTH OUTDR MAINT/STG,OTHER OUTDOOR MAINTENANCE/STORAGE FACILITY +TRANSL HSG - MEN,TRANSITIONAL HOUSING - ADULT MEN +NON-RES SOC SERV FAC,NON-RESIDENTIAL SOCIAL SERVICE FACILITY +O/D STRG-AGY VEHICLE,OUTDOOR STORAGE - AGENCY VEHICLES +INDOOR STRG-SUPPLIES,INDOOR STORAGE - SUPPLIES +OTHER REC FACILITY,OTHER RECREATIONAL FACILITY +OTH INDOOR MAINT/STG,OTHER INDOOR MAINTENANCE/STORAGE FACILITY +INDOOR STORAGE-BULK,INDOOR STORAGE - BULK MATERIAL +RES HEALTH CARE FAC,RESIDENTIAL HEALTH CARE FACILITY +OTH SECURE DETENTION,OTHER SECURE DETENTION FACILITY +SOCIAL SERVICE FACIL,SOCIAL SERVICE FACILITY +TRAIN FACIL-OUTDOOR,TRAINING FACILITY - OUTDOOR +TRANSL HSG - WOMEN,TRANSITIONAL HOUSING - ADULT WOMEN +STORMWATER PUMPING,STORMWATER PUMPING STATION +RES FACIL - CHILDREN,RESIDENTIAL FACILITY FOR CHILDREN +OTHER TESTING FACIL,OTHER TESTING FACILITY +OTH WATER TRANS FAC,OTHER WATERFRONT TRANSPORTATION FACILITY diff --git a/products/colp/sql/create_colp.sql b/products/colp/sql/create_colp.sql index 2c9342720f..bf2fca3072 100644 --- a/products/colp/sql/create_colp.sql +++ b/products/colp/sql/create_colp.sql @@ -205,6 +205,39 @@ normed_name_merge AS ( ON a._parcelname = b.old_name ), +usetype_mapped AS ( + SELECT + a.uid, + a.borough, + a.block, + a.lot, + a.bbl, + a.geo_bbl, + a.mapbbl, + a.hnum, + a._sname, + a._parcelname, + a.parcelname, + a.agency, + a._usecode, + -- Apply usetype mappings, keeping original if no mapping exists + coalesce(b.cleaned, a._usetype) AS _usetype, + a.ownership, + a.leased, + a.finalcom, + a.agreement, + a.xcoord, + a.ycoord, + a.latitude, + a.longitude, + a.geom, + a.cd, + a.sname + FROM normed_name_merge AS a + LEFT JOIN usetype_mappings AS b + ON a._usetype = b.source +), + categorized AS ( SELECT a.*, From e58d5df28e6fb4ac8c1707db338853d5cb42d52a Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 12:30:09 -0400 Subject: [PATCH 05/11] Add all the build-engine vars --- .github/workflows/colp_build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/colp_build.yml b/.github/workflows/colp_build.yml index 8f196431c4..788f7183c6 100644 --- a/.github/workflows/colp_build.yml +++ b/.github/workflows/colp_build.yml @@ -47,6 +47,10 @@ jobs: AWS_SECRET_ACCESS_KEY: "op://Data Engineering/DO_keys/AWS_SECRET_ACCESS_KEY" AWS_ACCESS_KEY_ID: "op://Data Engineering/DO_keys/AWS_ACCESS_KEY_ID" BUILD_ENGINE_SERVER: "op://Data Engineering/EDM_DATA/server_url" + BUILD_ENGINE_HOST: "op://Data Engineering/EDM_DATA/server" + BUILD_ENGINE_USER: "op://Data Engineering/EDM_DATA/username" + BUILD_ENGINE_PASSWORD: "op://Data Engineering/EDM_DATA/password" + BUILD_ENGINE_PORT: "op://Data Engineering/EDM_DATA/port" - name: Finish container setup ... working-directory: ./ From c28a075d1f964a1d0936a812ca6393bd8b3c86a8 Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 14:32:04 -0400 Subject: [PATCH 06/11] Integrate usetype_mappings into COLP build - Update categorized CTE to use usetype_mapped instead of normed_name_merge - This ensures cleaned usetype values from usetype_mappings seed are used - All 75 mapped values now display cleaned versions in final COLP output - Closes de-3oz.3 --- products/colp/sql/create_colp.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/colp/sql/create_colp.sql b/products/colp/sql/create_colp.sql index bf2fca3072..da298c91b8 100644 --- a/products/colp/sql/create_colp.sql +++ b/products/colp/sql/create_colp.sql @@ -342,7 +342,7 @@ categorized AS ( a._usecode = '1410' OR a._usecode = '1400' THEN 'PROPERTY WITH RESIDENTIAL USE' END) AS excatdesc - FROM normed_name_merge AS a + FROM usetype_mapped AS a ) -- Reorder columns for output From 661cd819a146dbe32a565b9855678bd0bc9d0f80 Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 16:06:46 -0400 Subject: [PATCH 07/11] add a few more mappings --- products/colp/seeds/usetype_mappings.csv | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/products/colp/seeds/usetype_mappings.csv b/products/colp/seeds/usetype_mappings.csv index 331c4c0516..a75d9c9c46 100644 --- a/products/colp/seeds/usetype_mappings.csv +++ b/products/colp/seeds/usetype_mappings.csv @@ -7,7 +7,8 @@ AGCY OFFICE,AGENCY OFFICE NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY) HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY -NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURESE +NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURES +NO USE-NON RESIDENTIAL STRUCTURE,NO USE-NON RESIDENTIAL STRUCTURES COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC INDOOR PKNG GARAGE,INDOOR PARKING GARAGE @@ -69,8 +70,13 @@ RES HEALTH CARE FAC,RESIDENTIAL HEALTH CARE FACILITY OTH SECURE DETENTION,OTHER SECURE DETENTION FACILITY SOCIAL SERVICE FACIL,SOCIAL SERVICE FACILITY TRAIN FACIL-OUTDOOR,TRAINING FACILITY - OUTDOOR +TRAIN FAC-INDOOR,TRAINING FACILITY - INDOOR TRANSL HSG - WOMEN,TRANSITIONAL HOUSING - ADULT WOMEN STORMWATER PUMPING,STORMWATER PUMPING STATION RES FACIL - CHILDREN,RESIDENTIAL FACILITY FOR CHILDREN OTHER TESTING FACIL,OTHER TESTING FACILITY OTH WATER TRANS FAC,OTHER WATERFRONT TRANSPORTATION FACILITY +CRIMINAL JUSTICE FAC,CRIMINAL JUSTICE FACILITY +OTH SOLID WASTE FAC,OTHER SOLID WASTE FACILITY +OUTDOOR TOW POUND,OUTDOOR PARKING - TOW POUND +OTHER SPC REC-INDOOR,OTHER SPECIAL RECREATIONAL FACILITY - INDOOR From 44cb32e66bc1cb8df724eac8c8ceff2e6d0186cf Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 16:33:40 -0400 Subject: [PATCH 08/11] split the scripts (for local running) --- products/colp/bash/02a_load_data.sh | 7 +++++++ products/colp/bash/02b_python.sh | 5 +++++ products/colp/bash/{02_build.sh => 02c_sql_build.sh} | 10 ---------- products/colp/colp.sh | 11 +++++++++-- products/colp/seeds/usetype_mappings.csv | 3 +-- 5 files changed, 22 insertions(+), 14 deletions(-) create mode 100644 products/colp/bash/02a_load_data.sh create mode 100644 products/colp/bash/02b_python.sh rename products/colp/bash/{02_build.sh => 02c_sql_build.sh} (61%) mode change 100755 => 100644 diff --git a/products/colp/bash/02a_load_data.sh b/products/colp/bash/02a_load_data.sh new file mode 100644 index 0000000000..c1a58cca84 --- /dev/null +++ b/products/colp/bash/02a_load_data.sh @@ -0,0 +1,7 @@ +#!/bin/bash +source bash/config.sh + +# Load seeds via dbt +dbt seed --select usetype_mappings +run_sql_file sql/load_modifications.sql +run_sql_file sql/geo_inputs.sql diff --git a/products/colp/bash/02b_python.sh b/products/colp/bash/02b_python.sh new file mode 100644 index 0000000000..f76a9b951d --- /dev/null +++ b/products/colp/bash/02b_python.sh @@ -0,0 +1,5 @@ +#!/bin/bash +source bash/config.sh + +python3 -m python.geocode +python3 -m python.geo_qaqc diff --git a/products/colp/bash/02_build.sh b/products/colp/bash/02c_sql_build.sh old mode 100755 new mode 100644 similarity index 61% rename from products/colp/bash/02_build.sh rename to products/colp/bash/02c_sql_build.sh index f786171ab0..a2157dfc1c --- a/products/colp/bash/02_build.sh +++ b/products/colp/bash/02c_sql_build.sh @@ -1,16 +1,6 @@ #!/bin/bash source bash/config.sh -# Load seeds via dbt -dbt seed --select usetype_mappings - -run_sql_file sql/load_modifications.sql - -run_sql_file sql/geo_inputs.sql - -python3 -m python.geocode -python3 -m python.geo_qaqc - run_sql_file sql/_procedures.sql run_sql_file sql/clean_parcelname.sql run_sql_file sql/create_colp.sql diff --git a/products/colp/colp.sh b/products/colp/colp.sh index c769a641f0..3d0f1a4898 100755 --- a/products/colp/colp.sh +++ b/products/colp/colp.sh @@ -2,10 +2,17 @@ case $1 in dataloading ) ./bash/01_dataloading.sh ;; - build ) ./bash/02_build.sh ;; + build ) + ./bash/02a_load_data.sh + ./bash/02b_python.sh + ./bash/02c_sql_build.sh + ;; + load_data ) ./bash/02a_load_data.sh ;; + python ) ./bash/02b_python.sh ;; + sql_build ) ./bash/02c_sql_build.sh ;; qaqc ) ./bash/03_qaqc.sh ;; export ) ./bash/04_export.sh ;; upload ) python3 -m dcpy.connectors.edm.publishing upload -p db-colp -a public-read ;; sql) sql $@ ;; - * ) echo "COMMAND \"$1\" is not found. (valid commands: dataloading|build|export|upload)" ;; + * ) echo "COMMAND \"$1\" is not found. (valid commands: dataloading|build|load_data|python|sql_build|qaqc|export|upload)" ;; esac diff --git a/products/colp/seeds/usetype_mappings.csv b/products/colp/seeds/usetype_mappings.csv index a75d9c9c46..825ef634db 100644 --- a/products/colp/seeds/usetype_mappings.csv +++ b/products/colp/seeds/usetype_mappings.csv @@ -7,8 +7,7 @@ AGCY OFFICE,AGENCY OFFICE NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY) HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY -NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURES -NO USE-NON RESIDENTIAL STRUCTURE,NO USE-NON RESIDENTIAL STRUCTURES +NO USE-NON RESIDENTIAL STRUCTURES,NO USE-NON RESIDENTIAL STRUCTURE COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC INDOOR PKNG GARAGE,INDOOR PARKING GARAGE From 194b8fb66d482e87df4d11a43001afc669e4f6d5 Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 16:37:48 -0400 Subject: [PATCH 09/11] chmod! --- products/colp/bash/02a_load_data.sh | 0 products/colp/bash/02b_python.sh | 0 products/colp/bash/02c_sql_build.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 products/colp/bash/02a_load_data.sh mode change 100644 => 100755 products/colp/bash/02b_python.sh mode change 100644 => 100755 products/colp/bash/02c_sql_build.sh diff --git a/products/colp/bash/02a_load_data.sh b/products/colp/bash/02a_load_data.sh old mode 100644 new mode 100755 diff --git a/products/colp/bash/02b_python.sh b/products/colp/bash/02b_python.sh old mode 100644 new mode 100755 diff --git a/products/colp/bash/02c_sql_build.sh b/products/colp/bash/02c_sql_build.sh old mode 100644 new mode 100755 From 7c0e058d749b71d854dc4d56883110954d7a0e7f Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 16:51:28 -0400 Subject: [PATCH 10/11] fixup! add a few more mappings --- products/colp/seeds/usetype_mappings.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/products/colp/seeds/usetype_mappings.csv b/products/colp/seeds/usetype_mappings.csv index 825ef634db..2b93cde082 100644 --- a/products/colp/seeds/usetype_mappings.csv +++ b/products/colp/seeds/usetype_mappings.csv @@ -8,6 +8,7 @@ NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY) HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY NO USE-NON RESIDENTIAL STRUCTURES,NO USE-NON RESIDENTIAL STRUCTURE +NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURE COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC INDOOR PKNG GARAGE,INDOOR PARKING GARAGE From 6f5991bd5740ae970173b2c55ac1b817322d4fda Mon Sep 17 00:00:00 2001 From: Alex Richey Date: Thu, 19 Mar 2026 16:55:29 -0400 Subject: [PATCH 11/11] kill my old usetype_mappings (non seed) csv --- products/colp/data/usetype_mappings.csv | 76 ------------------------- 1 file changed, 76 deletions(-) delete mode 100644 products/colp/data/usetype_mappings.csv diff --git a/products/colp/data/usetype_mappings.csv b/products/colp/data/usetype_mappings.csv deleted file mode 100644 index 331c4c0516..0000000000 --- a/products/colp/data/usetype_mappings.csv +++ /dev/null @@ -1,76 +0,0 @@ -source,cleaned -NATURAL AREA/ETC,NATURAL AREA/WETLAND/WILDLIFE REFUGE -IN USE-RES STRUCTURE,IN USE-RESIDENTIAL STRUCTURE -PLAYGRND/SPORTS AREA,PLAYGROUND/SPORTS AREA -AGCY EXECUTVE OFFICE,AGENCY EXECUTIVE OFFICE -AGCY OFFICE,AGENCY OFFICE -NO USE-WFT (ANY AGY),NO USE-WATERFRONT (ANY AGENCY) -HIGHWAY STRIP/ETC,MALL/TRIANGLE/HIGHWAY STRIP/PARK STRIP -OTHER TRANS FACILITY,OTHER TRANSPORTATION FACILITY -NO USE-NON RES STRC,NO USE-NON RESIDENTIAL STRUCTURESE -COMBINED MAINT/STRG,COMBINED MAINTENANCE/STORAGE FACILITY -MUNICPL PKNG-OUTDR,MUNICIPAL PARKING - OUTDOOR PUBLIC -INDOOR PKNG GARAGE,INDOOR PARKING GARAGE -UNDEVELOP OPEN SPACE,UNDEVELOPED OPEN SPACE -OUTDOOR PKNG-AGENCY,OUTDOOR PARKING - AGENCY VEHICLES -TRANSL HSG - FAMILY,TRANSITIONAL HOUSING - FAMILY -RECREATIONL FACILITY,RECREATIONAL FACILITY -NO USE-RES STRUCTURE,NO USE-RESIDENTIAL STRUCTURE -WASTEWATER PUMPING,WASTEWATER PUMPING STATION -OUTDR PKNG-EMPLOYEES,OUTDOOR PARKING - EMPLOYEES -SPECIAL EDUC SCHOOL,SPECIAL EDUCATION SCHOOL -OTHER PUB SAFETY FAC,OTHER PUBLIC SAFETY FACILITY -OTHER CULTURAL FAC,OTHER CULTURAL FACILITY -AMBUL HEALTH FACIL,AMBULATORY HEALTH FACILITY -INDOOR MAINT/STORAGE,INDOOR MAINTENANCE/STORAGE/GARAGE FACILITY -EARLY CHILDHD CENTER,EARLY CHILDHOOD CENTER -INDOOR STORAGE(WHSE),INDOOR STORAGE (WAREHOUSE) -POLLUTION CONTRL FAC,POLLUTION CONTROL FACILITY -TRANSL HSG - ADULT,TRANSITIONAL HOUSING - ADULT -VOCATION HIGH SCHOOL,VOCATIONAL HIGH SCHOOL -WATER POLL CNTRL PLT,WATER POLLUTION CONTROL PLANT -OUTDOOR STRG-EQUIP,OUTDOOR STORAGE - EQUIPMENT -OUTDOOR PKNG-CLIENTS,OUTDOOR PARKING - CLIENTS -OTHER HEALTH FACILTY,OTHER HEALTH FACILITY -RECREATION CTR/GYM,RECREATION CENTER/GYMNASIUM -OTH WATER SUPPLY FAC,OTHER WATER SUPPLY FACILITY -OFC LEASED BY EDUC,OFFICE LEASED BY EDUC -INDOOR STORAGE-EQUIP,INDOOR STORAGE - EQUIPMENT -SECTION STATION,SANITATION SECTION STATION -FACILITIES MAINT,FACILITIES MAINTENANCE -OUTDOOR MAINT/STORGE,OUTDOOR MAINTENANCE/STORAGE FACILITY -OTHER COLLEGE/UNIV,OTHER COLLEGE/UNIVERSITY -OUTDOOR STORAGE-BULK,OUTDOOR STORAGE - BULK MATERIAL -ALTERNTV HIGH SCHOOL,ALTERNATIVE HIGH SCHOOL -OTHR SPC REC-OUTDOOR,OTHER SPECIAL RECREATIONAL FACILITY - OUTDOOR -SOLID WASTE TSFR STN,SOLID WASTE TRANSFER STATION -SPECIAL REC-OUTDOOR,SPECIAL RECREATIONAL FACILITY - OUTDOOR -MUNICIPL PKNG-INDR,MUNICIPAL PARKING - INDOOR PUBLIC -WATER SUPPLY FACIL,WATER SUPPLY FACILITY -NEIGHBORHD SIT AREA,NEIGHBORHOOD SITTING AREA -SPEC RECREATN-INDOOR,SPECIAL RECREATIONAL FACILITY - INDOOR -SLUDGE DE-WATERING,SLUDGE DE-WATERING FACILITY -PUB SAFETY FACILITY,PUBLIC SAFETY FACILITY -OTH OUTDOOR STORAGE,OTHER OUTDOOR STORAGE -TRANS/PUBLIC PARKING,TRANSPORTATION/PUBLIC PARKING FACILITY -INDOOR STRG-PRINTED,INDOOR STORAGE - PRINTED MATERIAL/SUPPLIES -EDUC SKILLS CENTER,EDUCATIONAL SKILLS CENTER -OTHER POLL CNTRL FAC,OTHER POLLUTION CONTROL FACILITY -OTH DETENTION FACIL,OTHER DETENTION FACILITY -OTH OUTDR MAINT/STG,OTHER OUTDOOR MAINTENANCE/STORAGE FACILITY -TRANSL HSG - MEN,TRANSITIONAL HOUSING - ADULT MEN -NON-RES SOC SERV FAC,NON-RESIDENTIAL SOCIAL SERVICE FACILITY -O/D STRG-AGY VEHICLE,OUTDOOR STORAGE - AGENCY VEHICLES -INDOOR STRG-SUPPLIES,INDOOR STORAGE - SUPPLIES -OTHER REC FACILITY,OTHER RECREATIONAL FACILITY -OTH INDOOR MAINT/STG,OTHER INDOOR MAINTENANCE/STORAGE FACILITY -INDOOR STORAGE-BULK,INDOOR STORAGE - BULK MATERIAL -RES HEALTH CARE FAC,RESIDENTIAL HEALTH CARE FACILITY -OTH SECURE DETENTION,OTHER SECURE DETENTION FACILITY -SOCIAL SERVICE FACIL,SOCIAL SERVICE FACILITY -TRAIN FACIL-OUTDOOR,TRAINING FACILITY - OUTDOOR -TRANSL HSG - WOMEN,TRANSITIONAL HOUSING - ADULT WOMEN -STORMWATER PUMPING,STORMWATER PUMPING STATION -RES FACIL - CHILDREN,RESIDENTIAL FACILITY FOR CHILDREN -OTHER TESTING FACIL,OTHER TESTING FACILITY -OTH WATER TRANS FAC,OTHER WATERFRONT TRANSPORTATION FACILITY