build DevDB 25Q4

dcpy/library/script/dob_cofos.py

@@ -6,6 +6,9 @@
 from . import df_to_tempfile
 from .scriptor import ScriptorInterface
 
+from dcpy.connectors.edm import recipes
+from dcpy.utils import s3
+
 
 class Scriptor(ScriptorInterface):
     @property
@@ -18,7 +21,12 @@ def previous_version(self) -> str:
         return str(version)
 
     def ingest(self) -> pd.DataFrame:
-        df = pd.read_csv(self.source["path"], dtype=str)
+        data = s3.get_file_as_stream(
+            "edm-private",
+            f"dob_cofos/{self.version}/dob_cofos.csv",
+        )
+        df = pd.read_csv(data)
+
         df.insert(0, "v", self.version)
         # add the extra column and assign the missing columns to None
         df.insert(df.shape[1], "docstatus", None)

dcpy/library/templates/dob_cofos.yml

@@ -3,10 +3,9 @@ dataset:
   acl: public-read
   source:
     script:
-      path: .library/upload/dob_cofos.csv
       ## This field must be updated manually when archiving. Should eventually be refactored out of here
       ## Do not check in a specific version, leave blank version check in - this has led to issues in the past
-      previous_version: PLACEHOLDER
+      previous_version: 20251025
     options:
       - "AUTODETECT_TYPE=NO"
       - "EMPTY_STRING_AS_NULL=YES"

products/developments/data/corrections/hny_corrections.csv

@@ -211,8 +211,6 @@
 65503/991158,Q00608144,65503,add
 70744/997721,X00122977,70744,add
 67286/986841,420663589,67286,add
-64886/996947,420667558,64886,add
-67286/986841,420663589,67286,add
 64885/983133,420663712,64885,add
 67286/989998,420663570,67286,add
 70082/1000221,420667488,70082,add
@@ -430,7 +428,7 @@
 70072/995279,421133026,70072,add
 70072/995280,421133035,70072,add
 63776/991993,321597606,63776,add
-71819/100042,B00621076,71819,add
+71819/1000421,B00621076,71819,add
 71247/998761,M00537657,71247,add
 71822/100417,M00574359,71822,add
 71927/100423,Q00649207,71927,add
@@ -779,9 +777,8 @@
 32109/928419,104935171,32109,add
 12922/010567,103293348,12922,add
 23232/834569,102918023,23232,add
-72815/100680,320914310,72815,add
+72815/1006800,320914310,72815,add
 58852/968043,321191410,58852,add
-70805/100285,321359766,70805,add
 53192/972982,320627121,53192,add
 64297/978835,321504555,64297,add
 72944/340962,321588705,72944,add
@@ -795,11 +792,10 @@
 73695/423434,421264429,73695,add
 71703/100458,240278828,71703,add
 52610/950310,220343420,52610,add
-68766/100434,210181364,68766,add
+68766/1004347,210181364,68766,add
 73729/120148,210179260,73729,add
 39631/239944,301519364,39631,add
 19618/213617,301535435,19618,add
-73730/042644,140990890,73730,add
 68808/032654,121204268,68808,add
 68747/991167,121205524,68747,add
 55398/854774,301358386,55398,add
@@ -856,7 +852,6 @@
 13091/089089,201066657,13091,add
 38868/934917,210050362,38868,add
 23185/858514,200816037,23185,add
-33568/887693,104766479,33568,add
 25537/864985,104256804,25537,add
 30801/882537,104844581,30801,add
 68426/989997,210180196,68246,add
@@ -1198,7 +1193,7 @@
 73246/079723,220672742,73246,add 
 44319/955280,320266083,44319,add
 45303/972798,320592523,45303,add
-71808/100285,B00501131,71808,add
+71808/1002856,B00501131,71808,add
 73239/106852,220514799,73239,add
 71851/999436,220550062,71851,add 
 44225/955261,220111689,44225,add 
@@ -1211,74 +1206,145 @@
 67899/100442,B00685211,67899,add 
 65315/927561,321386629,65315,add 
 72777/991143,B00539478,72777,add
-72129/100483,B00571785,72129,add
+72129/1004838,B00571785,72129,add
 73252/005064,121205454,73252,add
-74157/100475,420664481,74157,add
+74157/1004758,420664481,74157,add
 70739/997772,340765602,70739,add
-74438/101201,M00672872,74438,add
-73209/100476,420664490,73209,add
+74438/1012012,M00672872,74438,add
+73209/1004762,420664490,73209,add
 70577/997055,321599178,70577,add
 74849/412315,Q00660436,74849,add
-64887/101138,421134178,64887,add
+64887/1011389,421134178,64887,add
 74246/115248,X00695344,74246,add
 71445/100621,B00594970,71445,add
-69565/100458,210182309,69565,add
+69565/1004583,210182309,69565,add
 71072/088949,210181747,71072,add
-66956/101239,M00607118,66956,add
-72364/100289,121209138,72364,add
-74926/101139,S00563710,74926,add
+66956/1012391,M00607118,66956,add
+72364/1002898,121209138,72364,add
+74926/1011399,S00563710,74926,add
 74229/335461,321595733,74229,add
 71771/100004,B00614012,71771,add
 73241/998462,421728973,73241,add
-73211/100912,421249615,73211,add
+73211/1009127,421249615,73211,add
 74955/438858,421786856,74955,add
 74232/057172,X00618402,74232,add
 75432/417658,421633897,75432,add
 74998/101151,B00687734,74998,add
 74595/101040,240280753,74595,add
 74859/101080,B00674201,74859,add
 74570/101009,421643993,74570,add
-73720/101239,421673406,73720,add
+73720/1012396,421673406,73720,add
 74845/014156,B00508605,74845,add
 74931/100953,B00501024,74931,add
 65315/927561,321386629,65315,add
-64436/101293,210177627,64436,add
 75702/067120,210181989,75702,add
 74985/660259,421616914,74985,add
 75701/117820,210181051,75701,add
 74773/101266,B00667885,74773,add
 67301/986524,421401673,67301,add
 75706/101330,B00694424,75706,add
-70483/101261,X08055447,70483,add
+70483/1012611,X08055447,70483,add
 74145/992500,420664819,74145,add
 74145/992500,420664846,74145,add
 75711/101311,B00665500,75711,add
 64645/976833,121190424,64645,add
 74948/965078,320874951,74948,add
 74154/989274,321595403,74154,add
-74770/101227,B00715375,74770,add
+74770/1012279,B00715375,74770,add
 75708/089079,X00486240,75708,add
-75697/101312,B00695754,75697,add
-72412/101260,X00980196,72412,add
+75697/1013122,B00695754,75697,add
+72412/1012608,X00980196,72412,add
 75425/340928,B00731678,75425,add
 63886/998370,321593673,63886,add
-64436/101293,210177627,64436,add
 73469/100763,B00699404,73469,add
-72803/101007,B08015349,72803,add
+72803/1010077,B08015349,72803,add
 75700/307563,B00588594,75700,add
 75709/107982,X00553144,75709,add
 74137/997430,321595608,74137,add
 75707/101345,121209174,75707,add
-75672/101318,321592059,75672,add
+75672/1013180,321592059,75672,add
 75704/133497,B00741243,75704,add
 75710/101252,X00702219,75710,add
 66933/997257,420665284,66933,add
 75705/101345,321189423,75705,add
-75135/101307,B00680820,75135,add
-68854/101338,B00587387,68854,add
-75011/101155,B00648034,75011,add
+75135/1013077,B00680820,75135,add
+68854/1013384,B00587387,68854,add
+75011/1011557,B00648034,75011,add
 72296/100292,B00653276,72296,add
 74142/100922,220520915,74142,add
 67899/100442,B00685211,67899,add
-63764/100022,321589134,63764,add
+63764/1000227,321589134,63764,add
 75703/971016,320595030,75703,add
+49143/945536,420602217,49143,add
+76709/1016902,321588705,76709,add
+70813/998863,340751813,70813,add
+76743/1016905,B00664860,76743,add
+69331/1009477,421550681,69331,add
+75057/1017587,Q00646756,75057,add
+77123/061000,240281173,77123,add
+68372/183004,321148619,68372,add
+75474/210985,321594155,75474,add
+68158/395298,321387076,68158,add
+74233/958510,240299226,74233,add
+75925/1015567,B00923376,75925,add
+73566/1014223,B00851732,73566,add
+75456/417871,421484888,75456,add
+55080/975747,321600479,55080,add
+72851/1008506,B00653960,72851,add
+71223/1002602,B00509455,71223,add
+67717/1013778,B01007946,67717,add
+73540/1017632,B01087711,73540,add
+76780/307568,B00965124,76780,add
+71973/1009997,B00640761,71973,add
+75335/1013320,B00942812,75335,add
+76215/123540,220694700,76215,add
+77066/1017375,B00577160,77066,add
+71818/999505,B00542600,71818,add
+75699/417641,421133179,75699,add
+76405/122357,X00705496,76405,add
+58852/1017603,321191394,58852,add
+58852/101760,321191410,58852,add
+67768/092679,X08030377,67768,add
+76765/007525,M08025179,76765,add
+72745/1004751,121209575,72745,add
+77126/008787,121208709,77126,add
+74771/1013235,B00712795,74771,add
+75655/628850,Q00645316,75655,add
+77048/1017329,B00570830,77048,add
+71820/1000400,B00645685,71820,add
+76270/121039,X00747209,76270,add
+76779/005286,M01039403,76779,add
+71459/491545,420976448,71459,add
+72258/416871,420663561,72258,add
+75052/1017588,Q00646746,75052,add
+75397/063898,220212589,75397,add
+75925/1014053,B00923109,75925,add
+65302/1016884,Q00954283,65302,add
+76384/978965,B00661493,76384,add
+73384/654860,Q00665670,73384,add
+76371/439503,Q00564746,76371,add
+76318/1014854,421133632,76318,add
+75053/1017589,Q00646751,75053,add
+71917/1000376,B00637594,71917,add
+77114/1016803,240301794,77114,add
+76404/369117,B00693081,76404,add
+67898/991371,210181079,67898,add
+73942/1008272,B00600073,73942,add
+76784/129541,B00580365,76784,add
+72340/1003249,M00666777,72340,add
+64436/1012932,210177627,64436,add
+64436/1012933,210177627,64436,add
+64436/1012934,210177627,64436,add
+74167/354883,B00543029,74167,add
+63777/1008645,420663865,63777,add
+75015/994323,321719173,75015,add
+76740/971695,320350678,76740,remove
+72478/347745,321358268,72478,add
+55080/1017566,321600255,55080,add
+75135/1013072,B00680820,75135,add
+75135/1013072,321180155,75135,add
+76425/158251,321821748,76425,add
+63757/1008199,210182158,63757,add
+65412/1002871,321598945,65412,add
+75416/1015144,Q00650506,75416,add
+70466/1011318,121253785,70466,add