Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
82 commits
Select commit Hold shift + click to select a range
0f1554b
update imports
mdeshotel Feb 13, 2026
1893375
add consts.py
mdeshotel Feb 13, 2026
32c9446
split GeoMeta class into a base class and children classes
mdeshotel Feb 13, 2026
a3c9487
perform check before initializing GeoMeta
mdeshotel Feb 17, 2026
ab19b1b
add type hints and correctly handle new properties/attributes
mdeshotel Feb 17, 2026
5b9dac4
assume bounds are float instead of int
mdeshotel Feb 17, 2026
472936b
fix malformed function calls for slopes
mdeshotel Feb 18, 2026
1246c6e
linting
mdeshotel Feb 18, 2026
ccc84a3
modularize the classes in geoMod.py
mdeshotel Feb 22, 2026
f39fb41
fix property decorators
mdeshotel Feb 22, 2026
47e37a8
move intialization of InputForcings to its __ini__.
mdeshotel Feb 23, 2026
b3a96dc
move static dictionaries to conts.py
mdeshotel Feb 23, 2026
c3f1099
add type hints
mdeshotel Feb 23, 2026
e6e8922
move conditionals to geoMod.py
mdeshotel Feb 23, 2026
0877b2a
update type hint
mdeshotel Feb 23, 2026
542aa3e
rename variables for better consistency across the repo
mdeshotel Feb 24, 2026
d78a7a4
update variable names to be consistent across the repo
mdeshotel Feb 24, 2026
259632a
split bmi model into multiple classes based on specified discretization
mdeshotel Feb 24, 2026
832534d
remove commented out items
mdeshotel Feb 24, 2026
965dcd5
split input forcing into multiple classes based on specified discreti…
mdeshotel Feb 24, 2026
a9546e0
update variable names across the repo
mdeshotel Feb 25, 2026
cacbe50
resolve circular imports
mdeshotel Feb 25, 2026
b10ce67
get grid type before initializing
mdeshotel Feb 25, 2026
8545a65
make parse_config a function instead of a method
mdeshotel Feb 25, 2026
bdce41e
rename geo_meta and move init logic out of the __init__ method
mdeshotel Feb 25, 2026
b9142ef
move bmi_model conts to the bmi_model dict
mdeshotel Feb 25, 2026
130a024
fix method calls and reference base class for consts
mdeshotel Feb 25, 2026
3e42cef
refactor bugs 1
mdeshotel Mar 20, 2026
fc69fba
dynamically manage discretization-specific GeoMeta classes
mdeshotel Mar 24, 2026
7f29a3e
fix geomod bugs
mdeshotel Mar 24, 2026
acb7ad7
add additional constants for mapping old to new variable names
mdeshotel Mar 24, 2026
d6ca3fa
fix bugs in forcingInputMod
mdeshotel Mar 24, 2026
10f2724
split consts dict to multiple dicts
mdeshotel Mar 26, 2026
9be38db
add setters for properties
mdeshotel Mar 26, 2026
193c55b
remove comas after dicts
mdeshotel Mar 27, 2026
481ee18
replace np.empty with np.full
mdeshotel Mar 28, 2026
f22c974
revert dtypes; use @cached_property
mdeshotel Mar 29, 2026
cbd64c6
remove duplicated setter
mdeshotel Mar 29, 2026
422ad4e
add additioanl type hints
mdeshotel Mar 30, 2026
9aa59a3
remove duplicate attribute definition
mdeshotel Mar 30, 2026
a9b1478
make docstring triple quotes
mdeshotel Mar 30, 2026
30f30b1
make _initialize_config_options a private method and increase verbosi…
mdeshotel Mar 30, 2026
2c56eeb
add comment for initializing attributes to none
mdeshotel Mar 30, 2026
b6e83e3
rename local height var to hgt
mdeshotel Mar 30, 2026
ed65691
replace @Property + @lru_cache with @cached_property
mdeshotel Mar 30, 2026
2415cdb
further dryify conditionals
mdeshotel Mar 31, 2026
05c7006
formatting
mdeshotel Mar 31, 2026
1c477ac
fix hasattr
mdeshotel Mar 31, 2026
765c861
replace @cached_property @property when setters are used to manually …
mdeshotel Mar 31, 2026
62c0ce0
update docstring
mdeshotel Apr 3, 2026
0d92a43
remove cached_property import
mdeshotel Apr 3, 2026
41344f4
update type hints for setters to None
mdeshotel Apr 5, 2026
8307082
add args to class __init__ methods
mdeshotel Apr 5, 2026
37eed33
move logic to properties
mdeshotel Apr 5, 2026
6e475e8
add missing setter
mdeshotel Apr 5, 2026
788e753
add custom_count arg
mdeshotel Apr 5, 2026
0d23434
load geogrid_ds
mdeshotel Apr 5, 2026
fea3244
fix longitude_grid
mdeshotel Apr 5, 2026
571dce2
fix docstrings for hydrofabric doc strings
mdeshotel Apr 5, 2026
49bcb85
raise value error for slope cals if unable to calculate given input
mdeshotel Apr 5, 2026
f61705e
fix bounds
mdeshotel Apr 5, 2026
5691c5f
don't restrict to rank 0
mdeshotel Apr 5, 2026
6973b14
fix bmi_model reference to self
mdeshotel Apr 5, 2026
79f4bdc
update property decorators to use cached_property and update doc strings
mdeshotel Apr 6, 2026
29dced9
remove old helper methods and fix necessary metadata extraction methods
mdeshotel Apr 6, 2026
40812dd
remove @barrier
mdeshotel Apr 6, 2026
62de1a4
check list of vars to map from config_options
mdeshotel Apr 6, 2026
8106d5a
assert vars and add to docstring
mdeshotel Apr 6, 2026
5e68459
fix include_lqfrac logic for unstructured
mdeshotel Apr 6, 2026
8310a5a
fix logic for grib_vars
mdeshotel Apr 6, 2026
7cda5a8
fix type hint
mdeshotel Apr 6, 2026
ceae791
fix typo
mdeshotel Apr 6, 2026
62bd6ef
update getter to manually cache the data
mdeshotel Apr 6, 2026
bc3cb56
rename variables
mdeshotel Apr 6, 2026
63793b1
fix grib_vars logic
mdeshotel Apr 7, 2026
f155f98
limit get_bound to rank=0
mdeshotel Apr 7, 2026
8734565
Merge branch 'mdeshotel_pre_dev' into mdeshotel-9712_forcingInputMod2
mdeshotel Apr 7, 2026
54603e4
fix ewts imports
mdeshotel Apr 7, 2026
2998e21
remove old ewts
mdeshotel Apr 7, 2026
e6cb416
rename variable
mdeshotel Apr 7, 2026
2b8e12e
update imports
mdeshotel Apr 7, 2026
521425b
fix logging
mdeshotel Apr 7, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 53 additions & 24 deletions Forcing_Extraction_Scripts/forecast_download_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,15 @@
import time
import uuid
from abc import ABC, abstractmethod
from datetime import datetime, timezone, timedelta
from urllib import request, error
from datetime import datetime, timedelta, timezone
from urllib import error, request

# Use the Error, Warning, and Trapping System Package for logging
import ewts
import requests
from bs4 import BeautifulSoup

from nextgen_forcings_ewts import MODULE_NAME

LOG = logging.getLogger(MODULE_NAME)
if not LOG.handlers:
# No handlers attached — fallback to default root logger
logging.basicConfig()
LOG = logging.getLogger()
LOG = ewts.get_logger(ewts.FORCING_ID)


class ForecastDownloader(ABC):
Expand All @@ -40,7 +36,15 @@ class ForecastDownloader(ABC):
default_cleanback = 240
default_lagback = 6

def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback_hours, ens_number):
def __init__(
self,
out_dir,
start_time,
lookback_hours,
cleanback_hours,
lagback_hours,
ens_number,
):
"""
Initialize downloader with common configuration.

Expand All @@ -50,6 +54,15 @@ def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback
:param cleanback_hours: How far back to clean old files
:param lagback_hours: How many hours to lag before starting to fetch
"""
global LOG
if hasattr(LOG, "bind"):
# This is required prior to the first log message for the ewts package
LOG.bind()
else:
# Fallback to default root logger
logging.basicConfig()
LOG = logging.getLogger()

if lookback_hours <= lagback_hours:
raise ValueError(
f"Invalid configuration: lookback_hours ({lookback_hours}) must be greater than "
Expand All @@ -64,7 +77,9 @@ def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback
self.ens_number = ens_number

# Current hour, rounded to the top of the hour in UTC
self.d_now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0)
self.d_now = datetime.now(timezone.utc).replace(
minute=0, second=0, microsecond=0
)

# Format ens_number
self.ens_number = str(self.ens_number).zfill(2)
Expand Down Expand Up @@ -100,12 +115,14 @@ def from_cli_args(cls):
Also prints the parsed arguments for logging/debugging.
"""
parser = argparse.ArgumentParser()
parser.add_argument('outDir', type=str, help="Output directory path")
parser.add_argument('startTime', type=lambda s: datetime.strptime(s, "%Y-%m-%d %H:%M:%S"))
parser.add_argument('--lookBackHours', type=int, default=cls.default_lookback)
parser.add_argument('--cleanBackHours', type=int, default=cls.default_cleanback)
parser.add_argument('--lagBackHours', type=int, default=cls.default_lagback)
parser.add_argument('--ensNumber', type=int, default=None)
parser.add_argument("outDir", type=str, help="Output directory path")
parser.add_argument(
"startTime", type=lambda s: datetime.strptime(s, "%Y-%m-%d %H:%M:%S")
)
parser.add_argument("--lookBackHours", type=int, default=cls.default_lookback)
parser.add_argument("--cleanBackHours", type=int, default=cls.default_cleanback)
parser.add_argument("--lagBackHours", type=int, default=cls.default_lagback)
parser.add_argument("--ensNumber", type=int, default=None)
args = parser.parse_args()

print(f"{cls.__name__} args:", vars(args))
Expand All @@ -116,7 +133,7 @@ def from_cli_args(cls):
lookback_hours=args.lookBackHours,
cleanback_hours=args.cleanBackHours,
lagback_hours=args.lagBackHours,
ens_number=args.ensNumber
ens_number=args.ensNumber,
)

def run(self):
Expand Down Expand Up @@ -254,7 +271,9 @@ def _download_data(self):
Download forecast files by iterating over the desired time range and download targets.
Each timestamp may have one or more targets to process.
"""
LOG.info(f"ForecastDownloader: Download data. lookback: {self.lookback_hours} lagback: {self.effective_lagback()}")
LOG.info(
f"ForecastDownloader: Download data. lookback: {self.lookback_hours} lagback: {self.effective_lagback()}"
)
for hour in range(self.lookback_hours, self.effective_lagback(), -1):
d_start = self.start_time - timedelta(hours=hour)

Expand All @@ -271,7 +290,9 @@ def _download_data(self):

targets = self.get_download_targets(d_start)
for target in targets:
url, filename = self.build_file_url_and_name(d_start, target, self.ens_number)
url, filename = self.build_file_url_and_name(
d_start, target, self.ens_number
)
out_path = os.path.join(output_dir, filename)

LOG.info(f"Looking for file {out_path}")
Expand Down Expand Up @@ -341,7 +362,9 @@ def _download_file(self, url, out_path):

except FileExistsError:
# Another process already published the file.
LOG.info(f"{out_path} already exists; another process wrote it first. Removing temp.")
LOG.info(
f"{out_path} already exists; another process wrote it first. Removing temp."
)
os.remove(temp_path)
return

Expand Down Expand Up @@ -402,7 +425,9 @@ def get_file_specs(self, d_start) -> list[tuple[str, str]]:
pass

def _download_data(self):
LOG.info(f"FixedFileDownloader: Download data. lookback: {self.lookback_hours} lagback: {self.effective_lagback()}")
LOG.info(
f"FixedFileDownloader: Download data. lookback: {self.lookback_hours} lagback: {self.effective_lagback()}"
)
for hour in range(self.lookback_hours, self.effective_lagback(), -1):
d_start = self.start_time - timedelta(hours=hour)

Expand Down Expand Up @@ -450,10 +475,14 @@ def get_download_targets(self, _):
return [0] # Satisfy the abstract method; not used for scraping

def build_file_url_and_name(self, d_start, target):
raise NotImplementedError("ScrapedFileDownloader uses scraping logic instead of build_file_url_and_name().")
raise NotImplementedError(
"ScrapedFileDownloader uses scraping logic instead of build_file_url_and_name()."
)

def _download_data(self):
LOG.info(f"ScrapedFileDownloader: Download data. lookback: {self.lookback_hours} lagback: {self.effective_lagback()}")
LOG.info(
f"ScrapedFileDownloader: Download data. lookback: {self.lookback_hours} lagback: {self.effective_lagback()}"
)
for hour in range(self.lookback_hours, self.effective_lagback(), -1):
d_start = self.start_time - timedelta(hours=hour)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@

if TYPE_CHECKING:
from numpy.typing import NDArray
import logging

from nextgen_forcings_ewts import MODULE_NAME
# Use the Error, Warning, and Trapping System Package for logging
import ewts

LOG = logging.getLogger(MODULE_NAME)
LOG = ewts.get_logger(ewts.FORCING_ID)

_error_on_grid_type: bool = False

Expand Down
Loading