diff --git a/examples/hercules_input_example.yaml b/examples/hercules_input_example.yaml index 898897da..f7c53f31 100644 --- a/examples/hercules_input_example.yaml +++ b/examples/hercules_input_example.yaml @@ -14,10 +14,19 @@ verbose: False # Enable verbose console output (True/False) log_every_n: 10 # Log output every N time steps (positive integer, default: 1) # Output file configuration +output_dir: outputs # output folder output_file: outputs/hercules_output.h5 # Output HDF5 file path (automatically adds .h5 extension if missing) output_use_compression: True # Enable HDF5 compression (True/False, default: True) output_buffer_size: 50000 # Memory buffer size for writing data in rows (default: 50000) +logging: + logger_name: "hercules" + log_file: "log_hercules.log" + console_output: True + console_prefix: "HERCULES" + log_level: "INFO" + use_outputs_dir: True + outputs_dir: "outputs" # Plant-level configuration plant: interconnect_limit: 201300 # kW - grid interconnection capacity limit @@ -175,4 +184,3 @@ wind_farm: controller: # Add controller parameters here if using WHOC or other controllers # Example controller configuration would go here - diff --git a/hercules/hercules_model.py b/hercules/hercules_model.py index c1ad5cca..e381a749 100644 --- a/hercules/hercules_model.py +++ b/hercules/hercules_model.py @@ -21,8 +21,6 @@ LOGFILE = str(dt.datetime.now()).replace(":", "_").replace(" ", "_").replace(".", "_") -Path("outputs").mkdir(parents=True, exist_ok=True) - class HerculesModel: def __init__(self, input_file): @@ -35,14 +33,18 @@ def __init__(self, input_file): """ + # Load and validate the input file + h_dict = self._load_hercules_input(input_file) + + # set default output directory to cwd / "outputs" + output_dir = Path(h_dict.get("output_dir", "outputs")).absolute() + # Make sure output folder exists - Path("outputs").mkdir(parents=True, exist_ok=True) + Path(output_dir).mkdir(parents=True, exist_ok=True) # Set up logging - self.logger = self._setup_logging() - - # Load and validate the input file - h_dict = self._load_hercules_input(input_file) + logging_inputs = h_dict.get("logging", {}) | {"outputs_dir": output_dir} + self.logger = self._setup_logging(**logging_inputs) # Initialize the flattened h_dict self.h_dict_flat = {} @@ -83,8 +85,19 @@ def __init__(self, input_file): # Ensure .h5 extension if not self.output_file.endswith(".h5"): self.output_file = self.output_file.rsplit(".", 1)[0] + ".h5" + + if "/" in self.output_file: + # check if folder was specfied in output_file + if Path(self.output_file).parent != output_dir: + # if folder of output_file does not match output_dir, then + # just use the name of the output file + self.output_file = output_dir / self.output_file.split("/")[-1] + else: + self.output_file = output_dir / self.output_file else: - self.output_file = "outputs/hercules_output.h5" + self.output_file = output_dir / "hercules_output.h5" + + self.output_file = Path(self.output_file).absolute() # Initialize HDF5 output system self.hdf5_file = None @@ -124,7 +137,7 @@ def __init__(self, input_file): # starttime_utc is required and should already be set, but ensure it's still present self.starttime_utc = self.h_dict["starttime_utc"] - def _setup_logging(self, logfile="log_hercules.log", console_output=True): + def _setup_logging(self, logfile="log_hercules.log", console_output=True, **kwargs): """Set up logging to file and console. Creates 'outputs' directory and configures file/console logging with timestamps. @@ -137,12 +150,20 @@ def _setup_logging(self, logfile="log_hercules.log", console_output=True): Returns: logging.Logger: Configured logger instance. """ - return setup_logging( - logger_name="hercules", - log_file=logfile, - console_output=console_output, - console_prefix="HERCULES", - ) + + logging_defaults = { + "logger_name": "hercules", + "log_file": logfile, + "console_output": console_output, + "console_prefix": None, + "log_level": "INFO", + "use_outputs_dir": True, + "outputs_dir": Path("outputs").absolute(), + } + + # Update the defaults with any input kwargs + logging_inputs = logging_defaults | kwargs + return setup_logging(**logging_inputs) def _load_hercules_input(self, filename): """Load and validate Hercules input file. diff --git a/hercules/plant_components/component_base.py b/hercules/plant_components/component_base.py index dae43879..c07cff25 100644 --- a/hercules/plant_components/component_base.py +++ b/hercules/plant_components/component_base.py @@ -1,6 +1,6 @@ # Base class for plant components in Hercules. - +from pathlib import Path from typing import ClassVar from hercules.utilities import setup_logging @@ -60,13 +60,16 @@ def __init__(self, h_dict, component_name): self.component_type = type(self).__name__ # Set up logging + output_dir = Path(h_dict.get("output_dir", "outputs")).absolute() + logging_inputs = h_dict[component_name].get("logging", {}) | {"outputs_dir": output_dir} + # Check if log_file_name is defined in the h_dict[component_name] if "log_file_name" in h_dict[component_name]: self.log_file_name = h_dict[component_name]["log_file_name"] else: - self.log_file_name = f"outputs/log_{component_name}.log" + self.log_file_name = f"log_{component_name}.log" - self.logger = self._setup_logging(self.log_file_name) + self.logger = self._setup_logging(self.log_file_name, **logging_inputs) # Parse log_channels from the h_dict if "log_channels" in h_dict[component_name]: @@ -102,7 +105,7 @@ def __init__(self, h_dict, component_name): self.verbose = h_dict["verbose"] self.logger.info(f"read in verbose flag = {self.verbose}") - def _setup_logging(self, log_file_name): + def _setup_logging(self, log_file_name, **kwargs): """Set up logging for the component. @@ -116,13 +119,20 @@ def _setup_logging(self, log_file_name): Returns: logging.Logger: Configured logger instance for the component. """ - return setup_logging( - logger_name=self.component_name, - log_file=log_file_name, - console_output=True, - console_prefix=self.component_name.upper(), - use_outputs_dir=False, # log_file_name is already a full path - ) + logging_defaults = { + "logger_name": self.component_name, + "log_file": log_file_name, + "console_output": True, + "console_prefix": self.component_name.upper(), + "log_level": "INFO", + "use_outputs_dir": True, # log_file_name is already a full path + # the use_outputs_dir used to default to False. + "outputs_dir": Path("outputs").absolute(), + } + + # Update the defaults with any input kwargs + logging_inputs = logging_defaults | kwargs + return setup_logging(**logging_inputs) def __del__(self): """Cleanup method to properly close log file handlers.""" diff --git a/hercules/utilities.py b/hercules/utilities.py index d8a9eb82..b7f63b00 100644 --- a/hercules/utilities.py +++ b/hercules/utilities.py @@ -218,8 +218,10 @@ def load_hercules_input(filename): other_keys = [ "name", "description", + "logging", "controller", "verbose", + "output_dir", "output_file", "log_every_n", "external_data_file", @@ -372,6 +374,7 @@ def setup_logging( console_prefix=None, log_level=logging.INFO, use_outputs_dir=True, + outputs_dir=Path("outputs"), ): """Set up logging to file and console with flexible configuration. @@ -400,9 +403,8 @@ def setup_logging( # Determine the log file path if use_outputs_dir and (os.sep not in log_file and "/" not in log_file): # Simple filename - use outputs directory - log_dir = os.path.join(os.getcwd(), "outputs") - os.makedirs(log_dir, exist_ok=True) - log_file_path = os.path.join(log_dir, log_file) + os.makedirs(outputs_dir, exist_ok=True) + log_file_path = os.path.join(outputs_dir, log_file) else: # Full path or use_outputs_dir=False - use as-is but ensure directory exists log_file_path = log_file @@ -416,7 +418,11 @@ def setup_logging( for handler in logger.handlers[:]: logger.removeHandler(handler) - logger.setLevel(log_level) + if isinstance(log_level, str): + log_level_int = logging.getLevelName(log_level.upper()) + logger.setLevel(log_level_int) + else: + logger.setLevel(log_level) # Add file handler file_handler = logging.FileHandler(log_file_path) diff --git a/hercules/utilities_examples.py b/hercules/utilities_examples.py index ba3cf984..ce8eec4b 100644 --- a/hercules/utilities_examples.py +++ b/hercules/utilities_examples.py @@ -64,7 +64,7 @@ def ensure_example_inputs_exist(): generate_example_inputs() -def prepare_output_directory(output_dir="outputs"): +def prepare_output_directory(output_dir=Path("outputs")): """Remove and recreate an output directory for clean runs. If the output directory exists, it will be deleted and recreated. diff --git a/tests/hercules_model_test.py b/tests/hercules_model_test.py index b135b873..d78a7779 100644 --- a/tests/hercules_model_test.py +++ b/tests/hercules_model_test.py @@ -1,3 +1,5 @@ +from pathlib import Path + import numpy as np import pandas as pd from hercules.hercules_model import HerculesModel @@ -79,7 +81,7 @@ def test_HerculesModel_instantiation(): hmodel = HerculesModel(test_h_dict) # Check default settings - assert hmodel.output_file == "outputs/hercules_output.h5" + assert hmodel.output_file == Path("outputs/hercules_output.h5").absolute() assert hmodel.log_every_n == 1 assert hmodel.external_signals_all == {} @@ -107,7 +109,66 @@ def test_HerculesModel_instantiation(): assert hmodel.external_signals_all["power_reference"][-1] == 1000 # At time 6.0 # Check custom output file - assert hmodel.output_file == "test_output.h5" + assert hmodel.output_file.name == "test_output.h5" + + +def test_specified_outputs_dir(): + """Test that a user-specified output directory can be used for log files and output files""" + import os + import shutil + + import hercules + + HERCULES_ROOT = Path(hercules.__file__).parent.parent + TEST_DIR = HERCULES_ROOT / "tests" + + starting_cwd = str(Path.cwd().absolute()) + + temp_test_folder = HERCULES_ROOT / "tmp_tests" + + # If temp testing folder already exists, remove all existing files + if temp_test_folder.exists(): + shutil.rmtree(temp_test_folder) + Path(temp_test_folder).mkdir(parents=True, exist_ok=True) + + # change cwd to test folder + os.chdir(temp_test_folder) + + test_h_dict = h_dict_solar.copy() + # Enforce new loader policy: remove preset start/end and rely on *_utc + test_h_dict.pop("starttime", None) + test_h_dict.pop("endtime", None) + test_h_dict.pop("time", None) + test_h_dict.pop("step", None) + + outputs_folder_name = "nondefault_output_dirname" + test_h_dict["output_dir"] = outputs_folder_name + test_h_dict["output_file"] = f"{outputs_folder_name}/hercules_test_output.h5" + test_h_dict["solar_farm"]["solar_input_filename"] = str( + Path(TEST_DIR / "test_inputs" / "solar_pysam_data.csv").absolute() + ) + + expected_hercules_log = temp_test_folder / outputs_folder_name / "log_hercules.log" + expected_solar_log_fpath = temp_test_folder / outputs_folder_name / "log_solar_farm.log" + + hmodel = HerculesModel(test_h_dict) + + folders = [f for f in temp_test_folder.iterdir()] + + assert len(folders) == 1 + assert folders[0] == temp_test_folder / outputs_folder_name + assert expected_hercules_log.exists() + assert expected_solar_log_fpath.exists() + assert ( + hmodel.output_file.absolute() + == temp_test_folder / outputs_folder_name / "hercules_test_output.h5" + ) + + # Reset cwd + os.chdir(Path(starting_cwd)) + + # Remove all files and folders from temporary test dir + shutil.rmtree(temp_test_folder) def test_log_data_to_hdf5():