From b44c038e047b20049190d5e4a44f211c66a29498 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Thu, 18 Dec 2025 12:26:03 -0500 Subject: [PATCH 01/13] first commit with new netcdfcreator class --- include/core/NetCDFCreator.hpp | 21 +++++++++++++ src/NGen.cpp | 6 +++- src/NetCDFCreator.cpp | 30 +++++++++++++++++++ .../catchment/Bmi_Multi_Formulation.cpp | 2 +- 4 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 include/core/NetCDFCreator.hpp create mode 100644 src/NetCDFCreator.cpp diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp new file mode 100644 index 0000000000..886874e60d --- /dev/null +++ b/include/core/NetCDFCreator.hpp @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +using namespace netCDF; + +class NetCDFCreator +{ +public: + NetCDFCreator(std::shared_ptr manager, + const std::string& output_name, Simulation_Time const& sim_time); + NetCDFCreator() = delete; + ~NetCDFCreator(); + +private: + std::shared_ptr sim_time_; + NcDim timeDim; + NcDim catchmentsDim; + std::map> formulations; +}; \ No newline at end of file diff --git a/src/NGen.cpp b/src/NGen.cpp index c554ef751a..e946b40ee5 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -4,12 +4,14 @@ #include #include #include +#include #include #include "realizations/catchment/Formulation_Manager.hpp" #include #include +#include #if NGEN_WITH_SQLITE3 #include @@ -657,7 +659,7 @@ int main(int argc, char* argv[]) { std::vector> layers; layers.resize(keys.size()); - + for (long i = 0; i < keys.size(); ++i) { auto& desc = layer_meta_data.get_layer(keys[i]); std::vector cat_ids; @@ -706,6 +708,8 @@ int main(int argc, char* argv[]) { } #endif // NGEN_WITH_ROUTING + auto ncCreator = std::make_unique(manager,"catchment_output",*sim_time); + auto simulation = std::make_unique(*sim_time, layers, std::move(catchment_indexes), diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp new file mode 100644 index 0000000000..430510169b --- /dev/null +++ b/src/NetCDFCreator.cpp @@ -0,0 +1,30 @@ +#include +#include +#include + +using namespace netCDF; + + +NetCDFCreator::NetCDFCreator(std::shared_ptr manager, + const std::string& output_name,Simulation_Time const& sim_time) + :catchmentNcFile_(manager->get_output_root() + output_name + ".nc",NcFile::replace) + ,sim_time_(std::make_shared(sim_time)) +{ + timeDim = catchmentNcFile_.addDim("time", NC_UNLIMITED); //unlimited dimension to append data efficiently without redefining the entire file structure + NcVar timeVar = catchmentNcFile_.addVar("time", NC_INT64, timeDim); + catchmentsDim = catchmentNcFile_.addDim("catchments", manager->get_size()); + + int start_t = sim_time_->get_current_epoch_time(); + timeVar.putVar(&start_t); + for (int t_step = 0; t_step < sim_time_->get_total_output_times(); t_step++) { + int next_t = sim_time_->next_timestep_epoch_time(); + timeVar.putVar(&next_t); + } + + formulations = std::map>(manager->begin(), manager->end()); + for (auto const& catchment : formulations){ + + } +} + +NetCDFCreator::~NetCDFCreator() = default; diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index edeed9e66b..98b34c8937 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -19,6 +19,7 @@ void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap proper if (needs_param_validation) { validate_parameters(properties); } + LOG("Inside create multi_formulation", LogLevel::INFO); // Required parameters first, except for "modules" set_bmi_main_output_var(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MAIN_OUT_VAR).as_string()); set_model_type_name(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MODEL_TYPE).as_string()); @@ -476,7 +477,6 @@ std::string Bmi_Multi_Formulation::get_output_line_for_timestep(int timestep, st *output_text_stream << delimiter << value; //with delimiter for the rest. } } - LOG("BMI Multi Formulation: " + output_text_stream->str(), LogLevel::INFO); return output_text_stream->str(); } } From 6fcb1806cd85b24c94d07fdd897b62fd5d89ab13 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Thu, 18 Dec 2025 12:26:03 -0500 Subject: [PATCH 02/13] first commit with new netcdfcreator class --- include/core/NetCDFCreator.hpp | 21 +++++++++++++ src/NGen.cpp | 6 +++- src/NetCDFCreator.cpp | 30 +++++++++++++++++++ .../catchment/Bmi_Multi_Formulation.cpp | 1 + 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 include/core/NetCDFCreator.hpp create mode 100644 src/NetCDFCreator.cpp diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp new file mode 100644 index 0000000000..886874e60d --- /dev/null +++ b/include/core/NetCDFCreator.hpp @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +using namespace netCDF; + +class NetCDFCreator +{ +public: + NetCDFCreator(std::shared_ptr manager, + const std::string& output_name, Simulation_Time const& sim_time); + NetCDFCreator() = delete; + ~NetCDFCreator(); + +private: + std::shared_ptr sim_time_; + NcDim timeDim; + NcDim catchmentsDim; + std::map> formulations; +}; \ No newline at end of file diff --git a/src/NGen.cpp b/src/NGen.cpp index c554ef751a..e946b40ee5 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -4,12 +4,14 @@ #include #include #include +#include #include #include "realizations/catchment/Formulation_Manager.hpp" #include #include +#include #if NGEN_WITH_SQLITE3 #include @@ -657,7 +659,7 @@ int main(int argc, char* argv[]) { std::vector> layers; layers.resize(keys.size()); - + for (long i = 0; i < keys.size(); ++i) { auto& desc = layer_meta_data.get_layer(keys[i]); std::vector cat_ids; @@ -706,6 +708,8 @@ int main(int argc, char* argv[]) { } #endif // NGEN_WITH_ROUTING + auto ncCreator = std::make_unique(manager,"catchment_output",*sim_time); + auto simulation = std::make_unique(*sim_time, layers, std::move(catchment_indexes), diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp new file mode 100644 index 0000000000..430510169b --- /dev/null +++ b/src/NetCDFCreator.cpp @@ -0,0 +1,30 @@ +#include +#include +#include + +using namespace netCDF; + + +NetCDFCreator::NetCDFCreator(std::shared_ptr manager, + const std::string& output_name,Simulation_Time const& sim_time) + :catchmentNcFile_(manager->get_output_root() + output_name + ".nc",NcFile::replace) + ,sim_time_(std::make_shared(sim_time)) +{ + timeDim = catchmentNcFile_.addDim("time", NC_UNLIMITED); //unlimited dimension to append data efficiently without redefining the entire file structure + NcVar timeVar = catchmentNcFile_.addVar("time", NC_INT64, timeDim); + catchmentsDim = catchmentNcFile_.addDim("catchments", manager->get_size()); + + int start_t = sim_time_->get_current_epoch_time(); + timeVar.putVar(&start_t); + for (int t_step = 0; t_step < sim_time_->get_total_output_times(); t_step++) { + int next_t = sim_time_->next_timestep_epoch_time(); + timeVar.putVar(&next_t); + } + + formulations = std::map>(manager->begin(), manager->end()); + for (auto const& catchment : formulations){ + + } +} + +NetCDFCreator::~NetCDFCreator() = default; diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 145b7dfb73..98b34c8937 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -19,6 +19,7 @@ void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap proper if (needs_param_validation) { validate_parameters(properties); } + LOG("Inside create multi_formulation", LogLevel::INFO); // Required parameters first, except for "modules" set_bmi_main_output_var(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MAIN_OUT_VAR).as_string()); set_model_type_name(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MODEL_TYPE).as_string()); From 3dcac1a054298369ded1b2f59dc7c3bca7ae17b1 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Tue, 23 Dec 2025 14:26:04 -0500 Subject: [PATCH 03/13] Removed all class variables and constructor implementation for NetCDFCreator. Currently, the code is skeletal with the focus on getting a successful compile. --- CMakeLists.txt | 7 +++++++ include/core/NetCDFCreator.hpp | 12 ++++------- src/NGen.cpp | 4 +++- src/NetCDFCreator.cpp | 21 ++++--------------- .../catchment/Bmi_Multi_Formulation.cpp | 1 - 5 files changed, 18 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ecf309e715..d2b2eb7d29 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,6 +355,13 @@ if(NGEN_WITH_SQLITE) target_include_directories(partitionGenerator PUBLIC AFTER "${NGEN_INC_DIR}/geopackage") endif() +if(NGEN_WITH_NETCDF) + add_library(NetCDFCreator STATIC "${NGEN_SRC_DIR}/NetCDFCreator.cpp") + target_include_directories(NetCDFCreator PUBLIC "${NGEN_INC_DIR}/core") + target_link_libraries(NetCDFCreator PRIVATE NetCDF) + target_link_libraries(ngen PUBLIC NetCDFCreator) +endif() + target_link_libraries(partitionGenerator PUBLIC NGen::core NGen::geojson) if(NGEN_WITH_SQLITE) target_link_libraries(partitionGenerator PUBLIC NGen::geopackage) diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp index 886874e60d..ebce46f391 100644 --- a/include/core/NetCDFCreator.hpp +++ b/include/core/NetCDFCreator.hpp @@ -1,10 +1,10 @@ +#if NGEN_WITH_NETCDF + #include #include #include #include -using namespace netCDF; - class NetCDFCreator { public: @@ -12,10 +12,6 @@ class NetCDFCreator const std::string& output_name, Simulation_Time const& sim_time); NetCDFCreator() = delete; ~NetCDFCreator(); +}; -private: - std::shared_ptr sim_time_; - NcDim timeDim; - NcDim catchmentsDim; - std::map> formulations; -}; \ No newline at end of file +#endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/NGen.cpp b/src/NGen.cpp index e946b40ee5..e65cf79d0b 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -4,14 +4,16 @@ #include #include #include -#include #include #include "realizations/catchment/Formulation_Manager.hpp" #include #include + +#if NGEN_WITH_NETCDF #include +#endif #if NGEN_WITH_SQLITE3 #include diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp index 430510169b..fc5b0d2c15 100644 --- a/src/NetCDFCreator.cpp +++ b/src/NetCDFCreator.cpp @@ -1,30 +1,17 @@ +#if NGEN_WITH_NETCDF #include + #include #include using namespace netCDF; - NetCDFCreator::NetCDFCreator(std::shared_ptr manager, const std::string& output_name,Simulation_Time const& sim_time) - :catchmentNcFile_(manager->get_output_root() + output_name + ".nc",NcFile::replace) - ,sim_time_(std::make_shared(sim_time)) { - timeDim = catchmentNcFile_.addDim("time", NC_UNLIMITED); //unlimited dimension to append data efficiently without redefining the entire file structure - NcVar timeVar = catchmentNcFile_.addVar("time", NC_INT64, timeDim); - catchmentsDim = catchmentNcFile_.addDim("catchments", manager->get_size()); - int start_t = sim_time_->get_current_epoch_time(); - timeVar.putVar(&start_t); - for (int t_step = 0; t_step < sim_time_->get_total_output_times(); t_step++) { - int next_t = sim_time_->next_timestep_epoch_time(); - timeVar.putVar(&next_t); - } - - formulations = std::map>(manager->begin(), manager->end()); - for (auto const& catchment : formulations){ - - } } NetCDFCreator::~NetCDFCreator() = default; + +#endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 98b34c8937..145b7dfb73 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -19,7 +19,6 @@ void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap proper if (needs_param_validation) { validate_parameters(properties); } - LOG("Inside create multi_formulation", LogLevel::INFO); // Required parameters first, except for "modules" set_bmi_main_output_var(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MAIN_OUT_VAR).as_string()); set_model_type_name(properties.at(BMI_REALIZATION_CFG_PARAM_REQ__MODEL_TYPE).as_string()); From 74a624ab9f80193093478ca0902d46dd5ac9c3f6 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Wed, 24 Dec 2025 09:17:12 -0500 Subject: [PATCH 04/13] Included proper headers and target directories to CMakeFile to make the program compilable. --- CMakeLists.txt | 14 +++++++++++++- include/core/NetCDFCreator.hpp | 3 ++- src/NetCDFCreator.cpp | 3 ++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d2b2eb7d29..18ddcff47c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -357,7 +357,19 @@ endif() if(NGEN_WITH_NETCDF) add_library(NetCDFCreator STATIC "${NGEN_SRC_DIR}/NetCDFCreator.cpp") - target_include_directories(NetCDFCreator PUBLIC "${NGEN_INC_DIR}/core") + target_include_directories(NetCDFCreator PUBLIC + ${NGEN_INC_DIR} + ${NGEN_INC_DIR}/core + ${NGEN_INC_DIR}/core/catchment + ${NGEN_INC_DIR}/realizations/catchment + ${NGEN_INC_DIR}/simulation_time + ${NGEN_INC_DIR}/utilities + ${NGEN_INC_DIR}/geojson + ${NGEN_INC_DIR}/forcing + ${NGEN_INC_DIR}/core/mediator + ${Python_INCLUDE_DIRS} + ) + target_link_libraries(NetCDFCreator PUBLIC NGen::config_header) target_link_libraries(NetCDFCreator PRIVATE NetCDF) target_link_libraries(ngen PUBLIC NetCDFCreator) endif() diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp index ebce46f391..2a9a8d3224 100644 --- a/include/core/NetCDFCreator.hpp +++ b/include/core/NetCDFCreator.hpp @@ -1,5 +1,6 @@ -#if NGEN_WITH_NETCDF +#include +#if NGEN_WITH_NETCDF #include #include #include diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp index fc5b0d2c15..07ca01457f 100644 --- a/src/NetCDFCreator.cpp +++ b/src/NetCDFCreator.cpp @@ -1,6 +1,7 @@ +#include + #if NGEN_WITH_NETCDF #include - #include #include From 53f70227d0ec20d2c39932b11ef54c69ec9c8a57 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Sun, 11 Jan 2026 17:14:34 -0500 Subject: [PATCH 05/13] NetCDF for catchment outputs funcitonality implemented. --- CMakeLists.txt | 2 +- include/core/Layer.hpp | 4 + include/core/NetCDFCreator.hpp | 29 +++- include/core/NgenSimulation.hpp | 10 ++ .../catchment/Bmi_Formulation.hpp | 18 +++ .../catchment/Formulation_Manager.hpp | 4 + src/NGen.cpp | 7 +- src/NetCDFCreator.cpp | 142 +++++++++++++++++- src/core/Layer.cpp | 9 ++ src/core/NgenSimulation.cpp | 14 +- .../catchment/Bmi_Module_Formulation.cpp | 1 + .../catchment/Bmi_Multi_Formulation.cpp | 2 +- 12 files changed, 228 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 18ddcff47c..a1dbb92180 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -371,7 +371,7 @@ if(NGEN_WITH_NETCDF) ) target_link_libraries(NetCDFCreator PUBLIC NGen::config_header) target_link_libraries(NetCDFCreator PRIVATE NetCDF) - target_link_libraries(ngen PUBLIC NetCDFCreator) + target_link_libraries(ngen PRIVATE NetCDFCreator) endif() target_link_libraries(partitionGenerator PUBLIC NGen::core NGen::geojson) diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index 5c3c4481fa..c7e84a4db9 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -9,6 +9,7 @@ #include "State_Exception.hpp" #include "geojson/FeatureBuilder.hpp" #include +#include namespace hy_features { @@ -110,6 +111,8 @@ namespace ngen std::unordered_map &nexus_indexes, int current_step); + std::map get_catchment_output_data_for_timestep(); + protected: const LayerDescription description; @@ -121,6 +124,7 @@ namespace ngen //TODO is this really required at the top level? or can this be moved to SurfaceLayer? const geojson::GeoJSON catchment_data; long output_time_index; + std::map catchment_output_values; }; } diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp index 2a9a8d3224..08115dbf6a 100644 --- a/include/core/NetCDFCreator.hpp +++ b/include/core/NetCDFCreator.hpp @@ -1,11 +1,18 @@ +#ifndef NGEN_NETCDF_CREATOR_HPP +#define NGEN_NETCDF_CREATOR_HPP + #include #if NGEN_WITH_NETCDF -#include +#include #include #include #include +namespace netCDF { + class NcVar; + class NcFile; +} class NetCDFCreator { public: @@ -13,6 +20,22 @@ class NetCDFCreator const std::string& output_name, Simulation_Time const& sim_time); NetCDFCreator() = delete; ~NetCDFCreator(); -}; + + void write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values); + +protected: + void add_output_variable_info_from_formulation(); -#endif // NGEN_WITH_NETCDF \ No newline at end of file + std::vector string_split(std::string str, char delimiter); + + void close_ncfile(); + +private: + std::shared_ptr catchmentNcFile; + std::shared_ptr manager_; + std::shared_ptr sim_time_; + std::vector catchments; + std::vector nc_output_variables; +}; +#endif // NGEN_WITH_NETCDF +#endif // NGEN_NETCDF_CREATOR_HPP \ No newline at end of file diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index 00e5ef49eb..533e569cfc 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -4,8 +4,13 @@ #include #include +#include #include +#if NGEN_WITH_NETCDF + #include +#endif + namespace hy_features { class HY_Features; @@ -59,6 +64,8 @@ class NgenSimulation size_t get_num_output_times() const; std::string get_timestamp_for_step(int step) const; + void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name); + private: void advance_models_one_output_step(); @@ -81,6 +88,9 @@ class NgenSimulation // Serialization template will be defined and instantiated in the .cpp file template void serialize(Archive& ar); + + //Pointer to netcdfcreator to write simulation output per timestep. + std::unique_ptr nc_writer_; }; #endif diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index f2a13074e8..26ca6be448 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -180,6 +180,15 @@ namespace realization { return REQUIRED_PARAMETERS; } + /** + * Get the values making up the header line from get_output_header_line(), but organized as a vector of strings. + * + * @return The values making up the header line from get_output_header_line() organized as a vector. + */ + const std::vector &get_output_variable_units() const { + return output_variable_units; + } + virtual bool is_bmi_input_variable(const std::string &var_name) const = 0; /** @@ -238,6 +247,10 @@ namespace realization { output_header_fields = output_headers; } + void set_output_variable_units(const std::vector &output_units) { + output_variable_units = output_units; + } + /** * Set the names of variables in formulation output. * @@ -264,6 +277,11 @@ namespace realization { * the BMI module output variables accessible to the instance. */ std::vector output_variable_names; + /** + * Output units corresponding to the variables output by the realization, as defined in + * `output_variable_names`. + */ + std::vector output_variable_units; /** The degree of precision in output values when converting to text. */ int output_precision; diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index df6599f2a5..be34ca6aef 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -232,6 +232,10 @@ namespace realization { return this->formulations.at(id); } + std::map> get_all_formulations() const { + return this->formulations; + } + std::shared_ptr get_domain_formulation(long id) const { return this->domain_formulations.at(id); } diff --git a/src/NGen.cpp b/src/NGen.cpp index e65cf79d0b..bffd480922 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -710,20 +710,21 @@ int main(int argc, char* argv[]) { } #endif // NGEN_WITH_ROUTING - auto ncCreator = std::make_unique(manager,"catchment_output",*sim_time); - auto simulation = std::make_unique(*sim_time, layers, std::move(catchment_indexes), std::move(nexus_indexes), mpi_rank, mpi_num_procs); - + #if NGEN_WITH_NETCDF + simulation->create_netcdf_writer(manager, "catchment_output"); + #endif auto time_done_init = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); simulation->run_catchments(); + #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp index 07ca01457f..ed65b58d24 100644 --- a/src/NetCDFCreator.cpp +++ b/src/NetCDFCreator.cpp @@ -2,17 +2,151 @@ #if NGEN_WITH_NETCDF #include -#include +#include #include -using namespace netCDF; - NetCDFCreator::NetCDFCreator(std::shared_ptr manager, const std::string& output_name,Simulation_Time const& sim_time) { + manager_ = manager; + sim_time_ = std::make_shared(sim_time); + try{ + std::string ncOutputFileName = manager->get_output_root() + output_name + ".nc"; + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::replace); + if(!catchmentNcFile){ + LOG("Catchment output netcdf file creation failed: " + ncOutputFileName, LogLevel::FATAL); + throw std::runtime_error("Catchment output netcdf file creation failed: " + ncOutputFileName); + } + //Add dimension and coordinate variable for time + //TO DO: create a separate function if this action is initiated from outside the class. + //TO DO: convert seconds epoch to minutes or days? + int num_timesteps = sim_time_->get_total_output_times(); + auto time_dim = catchmentNcFile->addDim("time", num_timesteps); + auto time_var = catchmentNcFile->addVar("time", NC_INT, time_dim); + time_var.putAtt("units", "Seconds since 1970-01-01 00:00:00"); + time_var.putAtt("calendar", "gregorian"); + std::vector time_epoch_seconds(num_timesteps); + time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); + for(int time_index = 1; time_index < num_timesteps; time_index++) + { + sim_time_->advance_timestep(); + time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + } + time_var.putVar(time_epoch_seconds.data()); + + //Add dimension and coordinate variable for catchments + //TO DO: create a separate function if this action is initiated from outside the class. + auto catchments_dim = catchmentNcFile->addDim("catchments", manager->get_size()); + auto catchments_var = catchmentNcFile->addVar("catchments", NC_STRING, catchments_dim); + catchments_var.putAtt("Catchment ID", "Catchment identifier in input"); + std::vector catchment_ids; + catchment_ids.reserve(manager->get_size()); + catchments.reserve(manager->get_size()); //populate the catchment IDs in a vector to be used for getvar() later + int item_index = 0; + std::vector index; + index.resize(1); + for (auto const& formulation_info : manager->get_all_formulations()) + { + std::string catchm = formulation_info.first; + catchments.push_back(catchm); + index[0] = item_index; + catchments_var.putVar(index,catchm); + item_index++; + } + + //Add output data variables information such as headers, variable names, units to netcdf + //TO DO: change scope of this function if this is initiated from outside the class. + add_output_variable_info_from_formulation(); + + //Add global attributes + catchmentNcFile->putAtt("title", "NextGen Catchment Output Data"); + catchmentNcFile->putAtt("description", "NetCDF file containing catchment-level output data from NextGen simulation"); + catchmentNcFile->putAtt("institution", "NOAA"); + + }catch (const netCDF::exceptions::NcException& e){ + LOG(std::string("Error in catchments NetCDF initiation: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in catchments NetCDF initiation: ") + e.what()); + } +} + +void NetCDFCreator::add_output_variable_info_from_formulation() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + std::vectoroutput_units = r_c->get_output_variable_units(); + nc_output_variables.resize(output_variables.size()); + std::vector dims = {catchmentNcFile->getDim("time"), catchmentNcFile->getDim("catchments")}; + for(int index = 0; index < output_variables.size(); index ++){ + nc_output_variables[index] = catchmentNcFile->addVar(output_headers[index], NC_DOUBLE, dims); + nc_output_variables[index].putAtt("variable name", output_variables[index]); + nc_output_variables[index].putAtt("variable units", output_units[index]); + nc_output_variables[index].putAtt("_FillValue", NC_DOUBLE, -1.0); //TO DO: Change to another value, if recommended. + nc_output_variables[index].putAtt("missing_value", NC_DOUBLE, -2.0); //TO DO: Change to another value, if recommended. + } + }else{ + LOG("No output variables/headers information provided in the realization config. No output variables writtedn to NetCDF.", LogLevel::WARNING); + } } -NetCDFCreator::~NetCDFCreator() = default; +void NetCDFCreator::write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values) +{ + for (auto const& catchment_val : catchment_output_values) + { + std::string catchment_id = catchment_val.first; + + //iterate through catchment dimension to find the index of the catchment for writing. + //also split the comma separated outputs string to a vector of double values + size_t catchm_index_in_netcdf = -1; + std::vector catchment_output; + for (int c_index = 0; c_index < catchments.size(); ++c_index) + { + if(catchments[c_index] == catchment_id){ + catchment_output = string_split(catchment_val.second, ','); + catchm_index_in_netcdf = c_index; + break; + } + } + if (catchm_index_in_netcdf > -1) //is this check necessary? + { + //populate the outputs to the data variables at a given time and catchment index + std::vector start = {time_index, catchm_index_in_netcdf}; + std::vector count = {1, 1}; + for(int var_index = 0; var_index < nc_output_variables.size(); ++var_index) + { + nc_output_variables[var_index].putVar(start, count, &catchment_output[var_index]); + } + } + else{ + LOG("Catchment ID '" + catchment_id + "' not found in the output. Skip wirting it to NetCDF", LogLevel::WARNING); + } + } +} +std::vector NetCDFCreator::string_split(std::string str, char delimiter) +{ + std::stringstream ss(str); + std::vector res; + std::string token; + while (getline(ss, token, delimiter)) { //will return full string if no comma (or single output variable). + res.push_back(std::stod(token)); + } + return res; +} + +void NetCDFCreator::close_ncfile(){ + if (catchmentNcFile != nullptr) { + catchmentNcFile->close(); + } + catchmentNcFile = nullptr; +} + +NetCDFCreator::~NetCDFCreator() +{ + close_ncfile(); +} #endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 56d6506be6..648675691c 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -45,6 +45,11 @@ void ngen::Layer::update_models(boost::span catchment_outflows, std::string output = std::to_string(output_time_index)+","+current_timestamp+","+ r_c->get_output_line_for_timestep(output_time_index)+"\n"; r_c->write_output(output); + + //capture all the output values for this timestep to write to netcdf + #if NGEN_WITH_NETCDF + catchment_output_values[id] = r_c->get_output_line_for_timestep(output_time_index); + #endif } //TODO put this somewhere else. For now, just trying to ensure we get m^3/s into nexus output double area; @@ -83,3 +88,7 @@ void ngen::Layer::update_models(boost::span catchment_outflows, simulation_time.advance_timestep(); } } + +std::map ngen::Layer::get_catchment_output_data_for_timestep(){ + return catchment_output_values; +} diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index c5cce69103..823e3b42d7 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -21,7 +21,7 @@ NgenSimulation::NgenSimulation( std::unordered_map nexus_indexes, int mpi_rank, int mpi_num_procs - ) + ) : simulation_step_(0) , sim_time_(std::make_shared(sim_time)) , layers_(std::move(layers)) @@ -40,7 +40,6 @@ void NgenSimulation::run_catchments() { // Now loop some time, iterate catchments, do stuff for total number of output times auto num_times = get_num_output_times(); - for (; simulation_step_ < num_times; simulation_step_++) { // Make room for this output step's results catchment_outflows_.resize(catchment_outflows_.size() + catchment_indexes_.size(), 0.0); @@ -95,6 +94,12 @@ void NgenSimulation::advance_models_one_output_step() nexus_indexes_, simulation_step_ ); // assume update_models() calls time->advance_timestep() + + // After updating the layer, get the output data for that timestep and write to netcdf + #if NGEN_WITH_NETCDF + std::map catchment_output_vals = layer->get_catchment_output_data_for_timestep(); + nc_writer_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); + #endif prev_layer_time = layer_next_time; } else { layer_min_next_time = prev_layer_time = layer->current_timestep_epoch_time(); @@ -269,3 +274,8 @@ void NgenSimulation::serialize(Archive& ar) { ar & nexus_indexes_; ar & nexus_downstream_flows_; } + +void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name) +{ + this->nc_writer_ = std::make_unique(manager,nc_output_file_name,*sim_time_); +} diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 7c541e2d50..46eacc7cd0 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -600,6 +600,7 @@ namespace realization { output_var_units[i] = get_provider_units_for_variable(names[i]); } } + set_output_variable_units(output_var_units); //check if output variable indices (for vector variables) are specified in config. If not, default to zero (first index). if(output_var_indices.size() == 0){ diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 145b7dfb73..f1ab5d7afb 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -233,6 +233,7 @@ void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap proper output_var_units[i] = get_provider_units_for_variable(names[i]); } } + set_output_variable_units(output_var_units); //check if output variable indices (for vector variables) are specified in config. If not, default to zero (first index). if(output_var_indices.size() == 0){ @@ -413,7 +414,6 @@ std::string Bmi_Multi_Formulation::get_output_line_for_timestep(int timestep, st if (timestep != (next_time_step_index - 1)) { throw std::invalid_argument("Only current time step valid when getting multi-module BMI formulation output"); } - // Start by first checking whether we are just using the last module's values if (is_out_vars_from_last_mod) { // The default behavior, which means we either From 43fb963507109029e919acbc2be4e79d85c9946a Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Sun, 11 Jan 2026 21:11:14 -0500 Subject: [PATCH 06/13] Added pybind include directory to check pipeline build --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a1dbb92180..447f01bb99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -368,6 +368,7 @@ if(NGEN_WITH_NETCDF) ${NGEN_INC_DIR}/forcing ${NGEN_INC_DIR}/core/mediator ${Python_INCLUDE_DIRS} + ${pybind11_INCLUDE_DIR} ) target_link_libraries(NetCDFCreator PUBLIC NGen::config_header) target_link_libraries(NetCDFCreator PRIVATE NetCDF) From fc5eec51a73cafb0fd1e45430e6a9d60d7225258 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Mon, 12 Jan 2026 09:19:57 -0500 Subject: [PATCH 07/13] Made changes to enable MPI for catchment NetCDF creation. Build is still failing. --- CMakeLists.txt | 1 + include/core/NetCDFCreator.hpp | 2 +- include/core/NgenSimulation.hpp | 2 +- src/NGen.cpp | 2 +- src/NetCDFCreator.cpp | 109 +++++++++++++++++++------------- src/core/NgenSimulation.cpp | 8 ++- 6 files changed, 74 insertions(+), 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 447f01bb99..6b797ec8c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -373,6 +373,7 @@ if(NGEN_WITH_NETCDF) target_link_libraries(NetCDFCreator PUBLIC NGen::config_header) target_link_libraries(NetCDFCreator PRIVATE NetCDF) target_link_libraries(ngen PRIVATE NetCDFCreator) + target_link_libraries(core PUBLIC MPI::MPI_C MPI::MPI_CXX) endif() target_link_libraries(partitionGenerator PUBLIC NGen::core NGen::geojson) diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp index 08115dbf6a..6c967ddbd6 100644 --- a/include/core/NetCDFCreator.hpp +++ b/include/core/NetCDFCreator.hpp @@ -17,7 +17,7 @@ class NetCDFCreator { public: NetCDFCreator(std::shared_ptr manager, - const std::string& output_name, Simulation_Time const& sim_time); + const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank); NetCDFCreator() = delete; ~NetCDFCreator(); diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index 533e569cfc..311a6ae7da 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -64,7 +64,7 @@ class NgenSimulation size_t get_num_output_times() const; std::string get_timestamp_for_step(int step) const; - void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name); + void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank); private: void advance_models_one_output_step(); diff --git a/src/NGen.cpp b/src/NGen.cpp index bffd480922..ffe7b85f2a 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -717,7 +717,7 @@ int main(int argc, char* argv[]) { mpi_rank, mpi_num_procs); #if NGEN_WITH_NETCDF - simulation->create_netcdf_writer(manager, "catchment_output"); + simulation->create_netcdf_writer(manager, "catchment_output", mpi_rank); #endif auto time_done_init = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_init = time_done_init - time_start; diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp index ed65b58d24..309fd1144a 100644 --- a/src/NetCDFCreator.cpp +++ b/src/NetCDFCreator.cpp @@ -4,65 +4,84 @@ #include #include #include +#if NGEN_WITH_MPI +#include +#endif + NetCDFCreator::NetCDFCreator(std::shared_ptr manager, - const std::string& output_name,Simulation_Time const& sim_time) + const std::string& output_name,Simulation_Time const& sim_time, int mpi_rank) { manager_ = manager; sim_time_ = std::make_shared(sim_time); try{ std::string ncOutputFileName = manager->get_output_root() + output_name + ".nc"; - catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::replace); - if(!catchmentNcFile){ - LOG("Catchment output netcdf file creation failed: " + ncOutputFileName, LogLevel::FATAL); - throw std::runtime_error("Catchment output netcdf file creation failed: " + ncOutputFileName); - } - //Add dimension and coordinate variable for time - //TO DO: create a separate function if this action is initiated from outside the class. - //TO DO: convert seconds epoch to minutes or days? - int num_timesteps = sim_time_->get_total_output_times(); - auto time_dim = catchmentNcFile->addDim("time", num_timesteps); - auto time_var = catchmentNcFile->addVar("time", NC_INT, time_dim); - time_var.putAtt("units", "Seconds since 1970-01-01 00:00:00"); - time_var.putAtt("calendar", "gregorian"); - std::vector time_epoch_seconds(num_timesteps); - time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); - for(int time_index = 1; time_index < num_timesteps; time_index++) - { - sim_time_->advance_timestep(); - time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); - } - time_var.putVar(time_epoch_seconds.data()); + if(mpi_rank == 0){ + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::replace); + if(!catchmentNcFile){ + LOG("Catchment output netcdf file creation failed: " + ncOutputFileName, LogLevel::FATAL); + throw std::runtime_error("Catchment output netcdf file creation failed: " + ncOutputFileName); + } + //Add dimension and coordinate variable for time + //TO DO: create a separate function if this action is initiated from outside the class. + //TO DO: convert seconds epoch to minutes or days? + int num_timesteps = sim_time_->get_total_output_times(); + auto time_dim = catchmentNcFile->addDim("time", num_timesteps); + auto time_var = catchmentNcFile->addVar("time", NC_INT, time_dim); + time_var.putAtt("units", "Seconds since 1970-01-01 00:00:00"); + time_var.putAtt("calendar", "gregorian"); + std::vector time_epoch_seconds(num_timesteps); + time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); + for(int time_index = 1; time_index < num_timesteps; time_index++) + { + sim_time_->advance_timestep(); + time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + } + time_var.putVar(time_epoch_seconds.data()); - //Add dimension and coordinate variable for catchments - //TO DO: create a separate function if this action is initiated from outside the class. - auto catchments_dim = catchmentNcFile->addDim("catchments", manager->get_size()); - auto catchments_var = catchmentNcFile->addVar("catchments", NC_STRING, catchments_dim); - catchments_var.putAtt("Catchment ID", "Catchment identifier in input"); - std::vector catchment_ids; - catchment_ids.reserve(manager->get_size()); - catchments.reserve(manager->get_size()); //populate the catchment IDs in a vector to be used for getvar() later - int item_index = 0; - std::vector index; - index.resize(1); - for (auto const& formulation_info : manager->get_all_formulations()) - { - std::string catchm = formulation_info.first; - catchments.push_back(catchm); - index[0] = item_index; - catchments_var.putVar(index,catchm); - item_index++; + //Add dimension and coordinate variable for catchments + //TO DO: create a separate function if this action is initiated from outside the class. + auto catchments_dim = catchmentNcFile->addDim("catchments", manager->get_size()); + auto catchments_var = catchmentNcFile->addVar("catchments", NC_STRING, catchments_dim); + catchments_var.putAtt("Catchment ID", "Catchment identifier in input"); + //std::vector catchment_ids; + //catchment_ids.reserve(manager->get_size()); + catchments.reserve(manager->get_size()); //populate the catchment IDs in a vector to be used for getvar() later + int item_index = 0; + std::vector index; + index.resize(1); + for (auto const& formulation_info : manager->get_all_formulations()) + { + std::string catchm = formulation_info.first; + catchments.push_back(catchm); + index[0] = item_index; + catchments_var.putVar(index,catchm); + item_index++; + } + + //Add global attributes + catchmentNcFile->putAtt("title", "NextGen Catchment Output Data"); + catchmentNcFile->putAtt("description", "NetCDF file containing catchment-level output data from NextGen simulation"); + catchmentNcFile->putAtt("institution", "NOAA"); } + #if NGEN_WITH_MPI + MPI_Barrier(MPI_COMM_WORLD); + #endif + + if (mpi_rank != 0){ + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::write); + for (auto const& formulation_info : manager->get_all_formulations()) + { + std::string catchm = formulation_info.first; + catchments.push_back(catchm); + } + } //Add output data variables information such as headers, variable names, units to netcdf //TO DO: change scope of this function if this is initiated from outside the class. add_output_variable_info_from_formulation(); + LOG(std::string("Catchments: ") + catchments[0] + "; " + catchments[1] + "; mpi rank: " + std::to_string(mpi_rank), LogLevel::INFO); - //Add global attributes - catchmentNcFile->putAtt("title", "NextGen Catchment Output Data"); - catchmentNcFile->putAtt("description", "NetCDF file containing catchment-level output data from NextGen simulation"); - catchmentNcFile->putAtt("institution", "NOAA"); - }catch (const netCDF::exceptions::NcException& e){ LOG(std::string("Error in catchments NetCDF initiation: ") + e.what(), LogLevel::FATAL); throw std::runtime_error(std::string("Error in catchments NetCDF initiation: ") + e.what()); diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index 823e3b42d7..bf73575d82 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -50,6 +50,10 @@ void NgenSimulation::run_catchments() if (simulation_step_ + 1 < num_times) { sim_time_->advance_timestep(); } + + if(simulation_step_ == 2){ + return; + } } } @@ -275,7 +279,7 @@ void NgenSimulation::serialize(Archive& ar) { ar & nexus_downstream_flows_; } -void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name) +void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank) { - this->nc_writer_ = std::make_unique(manager,nc_output_file_name,*sim_time_); + this->nc_writer_ = std::make_unique(manager,nc_output_file_name,*sim_time_, mpi_rank); } From 915c0c3de7c0c22f33f7bdb97dbf1126d2c8d539 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Tue, 13 Jan 2026 17:24:56 -0500 Subject: [PATCH 08/13] Changes made to enable MPI for netcdf writing. --- CMakeLists.txt | 2 +- include/core/NetCDFCreator.hpp | 4 +- include/core/NgenSimulation.hpp | 2 +- src/NGen.cpp | 2 +- src/NetCDFCreator.cpp | 99 ++++++++++++++++++++------------- src/core/NgenSimulation.cpp | 8 +-- 6 files changed, 69 insertions(+), 48 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b797ec8c2..d3125a1b04 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -369,11 +369,11 @@ if(NGEN_WITH_NETCDF) ${NGEN_INC_DIR}/core/mediator ${Python_INCLUDE_DIRS} ${pybind11_INCLUDE_DIR} + ${MPI_CXX_INCLUDE_DIRS} ) target_link_libraries(NetCDFCreator PUBLIC NGen::config_header) target_link_libraries(NetCDFCreator PRIVATE NetCDF) target_link_libraries(ngen PRIVATE NetCDFCreator) - target_link_libraries(core PUBLIC MPI::MPI_C MPI::MPI_CXX) endif() target_link_libraries(partitionGenerator PUBLIC NGen::core NGen::geojson) diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp index 6c967ddbd6..3b57f86406 100644 --- a/include/core/NetCDFCreator.hpp +++ b/include/core/NetCDFCreator.hpp @@ -17,7 +17,7 @@ class NetCDFCreator { public: NetCDFCreator(std::shared_ptr manager, - const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank); + const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs); NetCDFCreator() = delete; ~NetCDFCreator(); @@ -26,6 +26,8 @@ class NetCDFCreator protected: void add_output_variable_info_from_formulation(); + void retrieve_output_variables_mpi(); + std::vector string_split(std::string str, char delimiter); void close_ncfile(); diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index 311a6ae7da..ba61a1fc88 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -64,7 +64,7 @@ class NgenSimulation size_t get_num_output_times() const; std::string get_timestamp_for_step(int step) const; - void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank); + void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs); private: void advance_models_one_output_step(); diff --git a/src/NGen.cpp b/src/NGen.cpp index ffe7b85f2a..8b63cf5aa8 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -717,7 +717,7 @@ int main(int argc, char* argv[]) { mpi_rank, mpi_num_procs); #if NGEN_WITH_NETCDF - simulation->create_netcdf_writer(manager, "catchment_output", mpi_rank); + simulation->create_netcdf_writer(manager, "catchment_output", mpi_rank, mpi_num_procs); #endif auto time_done_init = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_init = time_done_init - time_start; diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp index 309fd1144a..6c96145d3b 100644 --- a/src/NetCDFCreator.cpp +++ b/src/NetCDFCreator.cpp @@ -4,17 +4,40 @@ #include #include #include + #if NGEN_WITH_MPI #include +#include "parallel_utils.h" #endif - NetCDFCreator::NetCDFCreator(std::shared_ptr manager, - const std::string& output_name,Simulation_Time const& sim_time, int mpi_rank) + const std::string& output_name,Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs) { manager_ = manager; sim_time_ = std::make_shared(sim_time); + catchments.reserve(manager_->get_size()); try{ + + for (auto const& formulation_info : manager->get_all_formulations()) + { + std::string catchm = formulation_info.first; + catchments.push_back(catchm); + } + + #if NGEN_WITH_MPI + if (mpi_num_procs > 1){ + std::vector all_catchments = catchments; + all_catchments = parallel::gather_strings(catchments, mpi_rank, mpi_num_procs); + if (mpi_rank == 0){ + std::sort(all_catchments.begin(), all_catchments.end()); + all_catchments.erase( + std::unique(all_catchments.begin(), all_catchments.end()), + all_catchments.end() + ); + } + catchments = parallel::broadcast_strings(all_catchments, mpi_rank, mpi_num_procs); + } + #endif std::string ncOutputFileName = manager->get_output_root() + output_name + ".nc"; if(mpi_rank == 0){ catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::replace); @@ -41,23 +64,22 @@ NetCDFCreator::NetCDFCreator(std::shared_ptr m //Add dimension and coordinate variable for catchments //TO DO: create a separate function if this action is initiated from outside the class. - auto catchments_dim = catchmentNcFile->addDim("catchments", manager->get_size()); + auto catchments_dim = catchmentNcFile->addDim("catchments", catchments.size()); auto catchments_var = catchmentNcFile->addVar("catchments", NC_STRING, catchments_dim); catchments_var.putAtt("Catchment ID", "Catchment identifier in input"); - //std::vector catchment_ids; - //catchment_ids.reserve(manager->get_size()); - catchments.reserve(manager->get_size()); //populate the catchment IDs in a vector to be used for getvar() later int item_index = 0; std::vector index; index.resize(1); - for (auto const& formulation_info : manager->get_all_formulations()) + for (auto const& catchment : catchments) { - std::string catchm = formulation_info.first; - catchments.push_back(catchm); index[0] = item_index; - catchments_var.putVar(index,catchm); + catchments_var.putVar(index,catchment); item_index++; } + + //Add output data variables information such as headers, variable names, units to netcdf + //TO DO: change scope of this function if this is initiated from outside the class. + add_output_variable_info_from_formulation(); //Add global attributes catchmentNcFile->putAtt("title", "NextGen Catchment Output Data"); @@ -71,18 +93,10 @@ NetCDFCreator::NetCDFCreator(std::shared_ptr m if (mpi_rank != 0){ catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::write); - for (auto const& formulation_info : manager->get_all_formulations()) - { - std::string catchm = formulation_info.first; - catchments.push_back(catchm); - } + retrieve_output_variables_mpi(); } - //Add output data variables information such as headers, variable names, units to netcdf - //TO DO: change scope of this function if this is initiated from outside the class. - add_output_variable_info_from_formulation(); - LOG(std::string("Catchments: ") + catchments[0] + "; " + catchments[1] + "; mpi rank: " + std::to_string(mpi_rank), LogLevel::INFO); - - }catch (const netCDF::exceptions::NcException& e){ + } + catch (const netCDF::exceptions::NcException& e){ LOG(std::string("Error in catchments NetCDF initiation: ") + e.what(), LogLevel::FATAL); throw std::runtime_error(std::string("Error in catchments NetCDF initiation: ") + e.what()); } @@ -108,7 +122,25 @@ void NetCDFCreator::add_output_variable_info_from_formulation() nc_output_variables[index].putAtt("missing_value", NC_DOUBLE, -2.0); //TO DO: Change to another value, if recommended. } }else{ - LOG("No output variables/headers information provided in the realization config. No output variables writtedn to NetCDF.", LogLevel::WARNING); + LOG("No output variables/headers information provided in the realization config. No output variables written to NetCDF.", LogLevel::WARNING); + } +} + +void NetCDFCreator::retrieve_output_variables_mpi() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_headers = r_c->get_output_header_fields(); + nc_output_variables.resize(output_headers.size()); + + std::vector dims = {catchmentNcFile->getDim("time"), catchmentNcFile->getDim("catchments")}; + for(int index = 0; index < output_headers.size(); index ++){ + nc_output_variables[index] = catchmentNcFile->getVar(output_headers[index]); + } + }else{ + LOG("No output variables/headers information provided in the realization config. No output variables written to NetCDF.", LogLevel::WARNING); } } @@ -120,29 +152,20 @@ void NetCDFCreator::write_simulations_response_from_formulation(size_t time_inde //iterate through catchment dimension to find the index of the catchment for writing. //also split the comma separated outputs string to a vector of double values - size_t catchm_index_in_netcdf = -1; std::vector catchment_output; - for (int c_index = 0; c_index < catchments.size(); ++c_index) + std::vector count = {1, 1}; + for (size_t c_index = 0; c_index < catchments.size(); ++c_index) { if(catchments[c_index] == catchment_id){ catchment_output = string_split(catchment_val.second, ','); - catchm_index_in_netcdf = c_index; + std::vector start = {time_index, c_index}; + for(int var_index = 0; var_index < nc_output_variables.size(); ++var_index) + { + nc_output_variables[var_index].putVar(start, count, &catchment_output[var_index]); + } break; } } - if (catchm_index_in_netcdf > -1) //is this check necessary? - { - //populate the outputs to the data variables at a given time and catchment index - std::vector start = {time_index, catchm_index_in_netcdf}; - std::vector count = {1, 1}; - for(int var_index = 0; var_index < nc_output_variables.size(); ++var_index) - { - nc_output_variables[var_index].putVar(start, count, &catchment_output[var_index]); - } - } - else{ - LOG("Catchment ID '" + catchment_id + "' not found in the output. Skip wirting it to NetCDF", LogLevel::WARNING); - } } } diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index bf73575d82..0eee3bb755 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -50,10 +50,6 @@ void NgenSimulation::run_catchments() if (simulation_step_ + 1 < num_times) { sim_time_->advance_timestep(); } - - if(simulation_step_ == 2){ - return; - } } } @@ -279,7 +275,7 @@ void NgenSimulation::serialize(Archive& ar) { ar & nexus_downstream_flows_; } -void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank) +void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs) { - this->nc_writer_ = std::make_unique(manager,nc_output_file_name,*sim_time_, mpi_rank); + this->nc_writer_ = std::make_unique(manager,nc_output_file_name,*sim_time_, mpi_rank, mpi_num_procs_); } From fff50aa1c7f7dfc99d35076c2b8a0a9890d5153d Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Tue, 13 Jan 2026 21:18:32 -0500 Subject: [PATCH 09/13] Added dependent libraries to test_ngen_simulation for a successful build. --- test/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 623dd7a481..7d4bb871a1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -446,7 +446,12 @@ ngen_add_test( NGen::geojson NGen::logging NGen::core_mediator + NGen::realizations_catchment + NGen::forcing NGen::ngen_bmi + NetCDFCreator + REQUIRES + NGEN_WITH_NETCDF ) ########################### Netcdf Forcing Tests From f4b881946abf9fe1184d60743524f77ea63d8240 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Mon, 26 Jan 2026 11:29:36 -0500 Subject: [PATCH 10/13] Unit tests for netcdf creators. This build does not support MPI functionality. --- include/core/NetCDFCreator.hpp | 2 + src/NetCDFCreator.cpp | 4 + test/CMakeLists.txt | 21 +++ test/core/NetCDFCreatorTest.cpp | 315 ++++++++++++++++++++++++++++++++ 4 files changed, 342 insertions(+) create mode 100644 test/core/NetCDFCreatorTest.cpp diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp index 3b57f86406..f8bf7ee4fa 100644 --- a/include/core/NetCDFCreator.hpp +++ b/include/core/NetCDFCreator.hpp @@ -23,6 +23,8 @@ class NetCDFCreator void write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values); + netCDF::NcFile& GetNcFile(); + protected: void add_output_variable_info_from_formulation(); diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp index 6c96145d3b..99b1f64d27 100644 --- a/src/NetCDFCreator.cpp +++ b/src/NetCDFCreator.cpp @@ -180,6 +180,10 @@ std::vector NetCDFCreator::string_split(std::string str, char delimiter) return res; } +netCDF::NcFile& NetCDFCreator::GetNcFile(){ + return *catchmentNcFile; +} + void NetCDFCreator::close_ncfile(){ if (catchmentNcFile != nullptr) { catchmentNcFile->close(); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7d4bb871a1..588dd8f2ed 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -61,6 +61,7 @@ endif() add_executable(test_all) target_link_libraries(test_all PRIVATE gtest gtest_main) +target_link_libraries(test_all PRIVATE NetCDFCreator) set_target_properties(test_all PROPERTIES FOLDER test) target_include_directories(test_all PRIVATE ${CMAKE_CURRENT_LIST_DIR}/bmi) @@ -470,6 +471,26 @@ ngen_add_test( NGEN_WITH_NETCDF ) +########################## NgenSimulation Class Tests +ngen_add_test( + test_netcdf_creator + OBJECTS + core/NetCDFCreatorTest.cpp + LIBRARIES + NGen::core + NGen::geopackage + NGen::geojson + NGen::logging + NGen::core_catchment + NGen::core_mediator + NGen::realizations_catchment + NGen::forcing + NGen::ngen_bmi + NetCDFCreator + REQUIRES + NGEN_WITH_NETCDF +) + ########################## Primary Combined Unit Test Target ngen_add_test( test_unit diff --git a/test/core/NetCDFCreatorTest.cpp b/test/core/NetCDFCreatorTest.cpp new file mode 100644 index 0000000000..aa87186ddc --- /dev/null +++ b/test/core/NetCDFCreatorTest.cpp @@ -0,0 +1,315 @@ +#include + + +#if NGEN_WITH_NETCDF +#include "gtest/gtest.h" +#include +#include +#include "Bmi_Testing_Util.hpp" +#include "FileChecker.h" +#include +#include +#include +#include + +class NetCDFCreatorTest : public ::testing::Test { + protected: + std::stringstream stream_; + std::shared_ptr manager_; + geojson::GeoJSON fabric = std::make_shared(); + std::shared_ptr sim_time_; + std::map> calculated_results; + std::unique_ptr nc_creator; + + typedef struct tm time_type; + std::shared_ptr start_date_time; + std::shared_ptr end_date_time; + + void SetUp() override { + SetupFormulationManager(); + //SetupSimulationResults(); + } + + void TearDown() override { + + } + + //Construct a Formulation Manager and Simulation Time objects + void SetupFormulationManager() { + stream_ << fix_paths(example_config); + boost::property_tree::ptree realization_config; + boost::property_tree::json_parser::read_json(stream_, realization_config); + manager_ = std::make_shared(realization_config); + + auto possible_simulation_time = realization_config.get_child_optional("time"); + if (!possible_simulation_time) { + std::string throw_msg; throw_msg.assign("ERROR: No simulation time period defined."); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); + } + + auto simulation_time_config = realization::config::Time(*possible_simulation_time).make_params(); + sim_time_ = std::make_shared(simulation_time_config); + + std::ostream* raw_pointer = &std::cout; + std::shared_ptr s_ptr(raw_pointer, [](void*) {}); + utils::StreamHandler catchment_output(s_ptr); + + this->add_feature("cat-52"); + this->add_feature("cat-67"); + manager_->read(simulation_time_config, fabric, catchment_output); + } + + //Set up similation time results + void SetupSimulationResults() + { + + double dt = 3600.0; + + for (auto const& formulation : manager_->get_all_formulations()) { + if (calculated_results.count(formulation.first) == 0) { + calculated_results.emplace(formulation.first, std::map()); + } + + double calculation; + + for (long t = 0; t < 4; t++) { + calculation = formulation.second->get_response(t, dt); + std::string resp = formulation.second->get_output_line_for_timestep(t); + calculated_results.at(formulation.first).emplace(t, calculation); + } + } + } + + std::string fix_paths(std::string json) + { + std::vector forcing_paths = { + "./data/forcing/cat-52_2015-12-01 00_00_00_2015-12-30 23_00_00.csv", + "./data/forcing/cat-67_2015-12-01 00_00_00_2015-12-30 23_00_00.csv", + }; + std::vector v = {}; + for(unsigned int i = 0; i < path_options.size(); i++){ + v.push_back( path_options[i] + "data/forcing" ); + } + std::string dir = utils::FileChecker::find_first_readable(v); + if(dir != ""){ + std::string remove = "\"./data/forcing/\""; + std::string replace = "\""+dir+"\""; + boost::replace_all(json, remove , replace); + } + + //BMI_CPP_INIT_DIR_PATH + replace_paths(json, "{{BMI_CPP_INIT_DIR_PATH}}", "data/bmi/test_bmi_cpp"); + //EXTERN_DIR_PATH + replace_paths(json, "{{EXTERN_LIB_DIR_PATH}}", "extern/test_bmi_cpp/cmake_build/"); + + for (unsigned int i = 0; i < forcing_paths.size(); i++) { + if(json.find(forcing_paths[i]) == std::string::npos){ + continue; + } + std::vector v = {}; + for (unsigned int j = 0; j < path_options.size(); j++) { + v.push_back(path_options[j] + forcing_paths[i]); + } + std::string right_path = utils::FileChecker::find_first_readable(v); + if(right_path != forcing_paths[i]){ + std::cerr<<"Replacing "< v{path_options.size()}; + for(unsigned int i = 0; i < path_options.size(); i++) + v[i] = path_options[i] + replacement; + + const std::string dir = utils::FileChecker::find_first_readable(v); + if (dir == "") return; + + boost::replace_all(input, pattern, dir); + } + + void add_feature(std::string id) + { + geojson::three_dimensional_coordinates three_dimensions { + { + {1.0, 2.0}, + {3.0, 4.0}, + {5.0, 6.0} + }, + { + {7.0, 8.0}, + {9.0, 10.0}, + {11.0, 12.0} + } + }; + std::vector bounding_box{1.0, 2.0}; + geojson::PropertyMap properties{}; + + geojson::Feature feature = std::make_shared(geojson::PolygonFeature( + geojson::polygon(three_dimensions), + id, + properties + )); + + fabric->add_feature(feature); + } + + + std::vector path_options = { + "", + "../", + "../../", + "./test/", + "../test/", + "../../test/" + + }; + + const std::string example_config = + "{" + " \"global\": {" + " \"formulations\": [" + " {" + " \"name\": \"bmi_multi\"," + " \"params\": {" + " \"model_type_name\": \"bmi_multi_c++\"," + " \"forcing_file\": \"\"," + " \"init_config\": \"\"," + " \"allow_exceed_end_time\": true," + " \"main_output_variable\": \"OUTPUT_VAR_4\"," + " \"uses_forcing_file\": false," + " \"modules\": [" + " {" + " \"name\": \"bmi_c++\"," + " \"params\": {" + " \"model_type_name\": \"test_bmi_c++\"," + " \"library_file\": \"{{EXTERN_LIB_DIR_PATH}}" BMI_TEST_CPP_LIB_NAME "\"," + " \"init_config\": \"{{BMI_CPP_INIT_DIR_PATH}}/test_bmi_cpp_config_2.txt\"," + " \"allow_exceed_end_time\": true," + " \"main_output_variable\": \"OUTPUT_VAR_4\"," + " \"uses_forcing_file\": false," + " \"model_params\": {" + " \"MODEL_VAR_1\": {" + " \"source\": \"hydrofabric\"," + " \"from\": \"val\"" + " }," + " \"MODEL_VAR_2\": {" + " \"source\": \"hydrofabric\"" + " }" + " }," + " \"" BMI_REALIZATION_CFG_PARAM_OPT__VAR_STD_NAMES "\": {" + " \"INPUT_VAR_1\": \"APCP_surface\"," + " \"INPUT_VAR_2\": \"APCP_surface\"" + " }" + " }" + " }" + " ]" + " }" + " }" + " ]," + " \"forcing\": { " + " \"file_pattern\": \".*{{id}}.*.csv\"," + " \"path\": \"./data/forcing/\"," + " \"provider\": \"CsvPerFeature\"" + " }" + " }," + " \"time\": {" + " \"start_time\": \"2015-12-01 00:00:00\"," + " \"end_time\": \"2015-12-30 23:00:00\"," + " \"output_interval\": 3600" + " }" + "}"; +}; + +TEST_F(NetCDFCreatorTest, TestCatchmentIdentifiers) +{ + nc_creator = std::make_unique(manager_,"catchment_test", *sim_time_, 0, 1); + netCDF::NcFile& ncFile = nc_creator->GetNcFile(); + netCDF::NcVar catchments_var = ncFile.getVar("catchments"); + std::vector nc_dims = catchments_var.getDims(); + size_t len = nc_dims[0].getSize(); + int item_index = 0; + std::vector index; + index.resize(1); + std::vector catchments; + std::string catchment; + for(size_t i = 0; i < len; ++i){ + index[0] = item_index; + catchments_var.getVar(index, &catchment); + catchments.push_back(catchment.c_str()); + item_index++; + } + //delete the netcdf file that was created once the information is obtained. + ncFile.close(); + std::string ncOutputFileName = manager_->get_output_root() + "catchment_test.nc"; + const char* nc_filename = ncOutputFileName.c_str(); + int result = std::remove(nc_filename); + + ASSERT_EQ(catchments.size(), 2); + ASSERT_TRUE(std::find(catchments.begin(), catchments.end(), "cat-52") != catchments.end()); + ASSERT_TRUE(std::find(catchments.begin(), catchments.end(), "cat-67") != catchments.end()); +} + +TEST_F(NetCDFCreatorTest, TestOutputValues) +{ + nc_creator = std::make_unique(manager_,"catchment_test", *sim_time_, 0, 1); //create netcdf file or replace this with an existing file? + + //write outputs to netcdf at time index 0 for cat-52 + std::map catchment_output_values; + auto c_form = std::dynamic_pointer_cast(manager_->get_formulation("cat-52")); + double resp = c_form->get_response(0, 3600); + std::string out_resp = c_form->get_output_line_for_timestep(0); + catchment_output_values["cat-52"] = out_resp; + nc_creator->write_simulations_response_from_formulation(0, catchment_output_values); + + + auto r_c = std::dynamic_pointer_cast(manager_->get_formulation("cat-52")); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + std::vectoroutput_units = r_c->get_output_variable_units(); + + netCDF::NcFile& ncFile = nc_creator->GetNcFile(); //get a handle to the netcdf file object + + //find the index for "cat-52" and retrieve the values for the output variables + double catchment_output_nc; + std::string output_str; + netCDF::NcVar catchments_var = ncFile.getVar("catchments"); + std::vector nc_dims = catchments_var.getDims(); + size_t len = nc_dims[0].getSize(); + size_t catchment_index = 0; + std::vector index; + index.resize(1); + std::vector catchments; + std::string catchment; + for(size_t i = 0; i < len; ++i){ + index[0] = catchment_index; + catchments_var.getVar(index, &catchment); + std::string catchm = catchment.c_str(); + if(catchm == "cat-52"){ + //use the catchment_index to query the value from the netcdf file and write it to a string. + std::vector start = {0, catchment_index}; + std::vector count = {1, 1}; + for(int var_index = 0; var_index < output_variables.size(); var_index ++){ + netCDF::NcVar out_var = ncFile.getVar(output_headers[var_index]); + out_var.getVar(start, count, &catchment_output_nc); + output_str += (output_str.empty() ? "" : ",") + std::to_string(catchment_output_nc); + } + } + catchment_index++; + } + + //delete the netcdf file that was created once the information is obtained. + ncFile.close(); + std::string ncOutputFileName = manager_->get_output_root() + "catchment_test.nc"; + const char* nc_filename = ncOutputFileName.c_str(); + int result = std::remove(nc_filename); + + ASSERT_EQ(out_resp, output_str); + } +} +#endif From cf5e694ce01e6ad9255563d41d287fa0ac6df674 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Mon, 26 Jan 2026 14:52:17 -0500 Subject: [PATCH 11/13] Included num_proc if clause to prevent MPI Barrier being called for a single process. Also, removed an unused function from the test class. --- src/NetCDFCreator.cpp | 2 +- test/core/NetCDFCreatorTest.cpp | 22 ---------------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp index 99b1f64d27..1f8d0a754e 100644 --- a/src/NetCDFCreator.cpp +++ b/src/NetCDFCreator.cpp @@ -88,7 +88,7 @@ NetCDFCreator::NetCDFCreator(std::shared_ptr m } #if NGEN_WITH_MPI - MPI_Barrier(MPI_COMM_WORLD); + if (mpi_num_procs > 1) MPI_Barrier(MPI_COMM_WORLD); #endif if (mpi_rank != 0){ diff --git a/test/core/NetCDFCreatorTest.cpp b/test/core/NetCDFCreatorTest.cpp index aa87186ddc..f062ad4237 100644 --- a/test/core/NetCDFCreatorTest.cpp +++ b/test/core/NetCDFCreatorTest.cpp @@ -27,7 +27,6 @@ class NetCDFCreatorTest : public ::testing::Test { void SetUp() override { SetupFormulationManager(); - //SetupSimulationResults(); } void TearDown() override { @@ -60,27 +59,6 @@ class NetCDFCreatorTest : public ::testing::Test { manager_->read(simulation_time_config, fabric, catchment_output); } - //Set up similation time results - void SetupSimulationResults() - { - - double dt = 3600.0; - - for (auto const& formulation : manager_->get_all_formulations()) { - if (calculated_results.count(formulation.first) == 0) { - calculated_results.emplace(formulation.first, std::map()); - } - - double calculation; - - for (long t = 0; t < 4; t++) { - calculation = formulation.second->get_response(t, dt); - std::string resp = formulation.second->get_output_line_for_timestep(t); - calculated_results.at(formulation.first).emplace(t, calculation); - } - } - } - std::string fix_paths(std::string json) { std::vector forcing_paths = { From b82797bd6df4f95e6dc5031258d21bdb6dbb1478 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Mon, 26 Jan 2026 21:46:41 -0500 Subject: [PATCH 12/13] Added a couple of macro If blocks to prevent attempts to write to a single NetCDF in parallel fashion. --- src/NGen.cpp | 5 ++++- src/core/NgenSimulation.cpp | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index 8b63cf5aa8..2d2f9640dc 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -722,7 +722,10 @@ int main(int argc, char* argv[]) { auto time_done_init = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); - + + #if NGEN_WITH_MPI + LOG("Under MPI mode, catchments output values from simulation runs cannot be added to NetCDF file due to limitations in NetCDFCxx4 library.", LogLevel::WARNING); + #endif simulation->run_catchments(); diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index 0eee3bb755..dd4c7efc35 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -97,9 +97,14 @@ void NgenSimulation::advance_models_one_output_step() // After updating the layer, get the output data for that timestep and write to netcdf #if NGEN_WITH_NETCDF + #if NGEN_WITH_MPI + //do nothing? + #else std::map catchment_output_vals = layer->get_catchment_output_data_for_timestep(); nc_writer_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); - #endif + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF + prev_layer_time = layer_next_time; } else { layer_min_next_time = prev_layer_time = layer->current_timestep_epoch_time(); From 971c8308df7899c1bbf2f82a25c562c535965a74 Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Tue, 27 Jan 2026 16:23:59 -0500 Subject: [PATCH 13/13] Added checks for MPI and output variables in realization to prevent NetCDF creation. Added appropriate log messages. --- include/core/NetCDFCreator.hpp | 4 +- .../catchment/Bmi_Formulation.hpp | 6 +- src/NGen.cpp | 13 +- src/NetCDFCreator.cpp | 174 +++++++++--------- src/core/Layer.cpp | 6 +- src/core/NgenSimulation.cpp | 7 +- test/CMakeLists.txt | 2 +- test/core/NetCDFCreatorTest.cpp | 4 +- 8 files changed, 115 insertions(+), 101 deletions(-) diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp index f8bf7ee4fa..e5a1fe425a 100644 --- a/include/core/NetCDFCreator.hpp +++ b/include/core/NetCDFCreator.hpp @@ -23,7 +23,7 @@ class NetCDFCreator void write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values); - netCDF::NcFile& GetNcFile(); + netCDF::NcFile& get_ncfile(); protected: void add_output_variable_info_from_formulation(); @@ -32,6 +32,8 @@ class NetCDFCreator std::vector string_split(std::string str, char delimiter); + bool create_ncfile(); + void close_ncfile(); private: diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index 26ca6be448..2adaa2dbab 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -181,9 +181,9 @@ namespace realization { } /** - * Get the values making up the header line from get_output_header_line(), but organized as a vector of strings. + * Get the units for the output variables organized as a vector of strings. * - * @return The values making up the header line from get_output_header_line() organized as a vector. + * @return The units for the output variables organized as a vector. */ const std::vector &get_output_variable_units() const { return output_variable_units; @@ -279,7 +279,7 @@ namespace realization { std::vector output_variable_names; /** * Output units corresponding to the variables output by the realization, as defined in - * `output_variable_names`. + * `output_variables`. */ std::vector output_variable_units; /** The degree of precision in output values when converting to text. */ diff --git a/src/NGen.cpp b/src/NGen.cpp index 2d2f9640dc..75019f4fdf 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -661,7 +661,7 @@ int main(int argc, char* argv[]) { std::vector> layers; layers.resize(keys.size()); - + for (long i = 0; i < keys.size(); ++i) { auto& desc = layer_meta_data.get_layer(keys[i]); std::vector cat_ids; @@ -717,15 +717,16 @@ int main(int argc, char* argv[]) { mpi_rank, mpi_num_procs); #if NGEN_WITH_NETCDF - simulation->create_netcdf_writer(manager, "catchment_output", mpi_rank, mpi_num_procs); - #endif + #if NGEN_WITH_MPI + LOG("Under MPI mode, a NetCDF file for catchment output values is not created due to limitations in NetCDFCxx4 library.", LogLevel::INFO); + #else + simulation->create_netcdf_writer(manager, "catchment_output", mpi_rank, mpi_num_procs); //create a NetCDF file only for Non-MPI run. + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF auto time_done_init = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); - #if NGEN_WITH_MPI - LOG("Under MPI mode, catchments output values from simulation runs cannot be added to NetCDF file due to limitations in NetCDFCxx4 library.", LogLevel::WARNING); - #endif simulation->run_catchments(); diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp index 1f8d0a754e..e8cd56c391 100644 --- a/src/NetCDFCreator.cpp +++ b/src/NetCDFCreator.cpp @@ -15,90 +15,96 @@ NetCDFCreator::NetCDFCreator(std::shared_ptr m { manager_ = manager; sim_time_ = std::make_shared(sim_time); - catchments.reserve(manager_->get_size()); - try{ - - for (auto const& formulation_info : manager->get_all_formulations()) - { - std::string catchm = formulation_info.first; - catchments.push_back(catchm); - } - - #if NGEN_WITH_MPI - if (mpi_num_procs > 1){ - std::vector all_catchments = catchments; - all_catchments = parallel::gather_strings(catchments, mpi_rank, mpi_num_procs); - if (mpi_rank == 0){ - std::sort(all_catchments.begin(), all_catchments.end()); - all_catchments.erase( - std::unique(all_catchments.begin(), all_catchments.end()), - all_catchments.end() - ); - } - catchments = parallel::broadcast_strings(all_catchments, mpi_rank, mpi_num_procs); - } - #endif - std::string ncOutputFileName = manager->get_output_root() + output_name + ".nc"; - if(mpi_rank == 0){ - catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::replace); - if(!catchmentNcFile){ - LOG("Catchment output netcdf file creation failed: " + ncOutputFileName, LogLevel::FATAL); - throw std::runtime_error("Catchment output netcdf file creation failed: " + ncOutputFileName); - } - //Add dimension and coordinate variable for time - //TO DO: create a separate function if this action is initiated from outside the class. - //TO DO: convert seconds epoch to minutes or days? - int num_timesteps = sim_time_->get_total_output_times(); - auto time_dim = catchmentNcFile->addDim("time", num_timesteps); - auto time_var = catchmentNcFile->addVar("time", NC_INT, time_dim); - time_var.putAtt("units", "Seconds since 1970-01-01 00:00:00"); - time_var.putAtt("calendar", "gregorian"); - std::vector time_epoch_seconds(num_timesteps); - time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); - for(int time_index = 1; time_index < num_timesteps; time_index++) - { - sim_time_->advance_timestep(); - time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); - } - time_var.putVar(time_epoch_seconds.data()); - - //Add dimension and coordinate variable for catchments - //TO DO: create a separate function if this action is initiated from outside the class. - auto catchments_dim = catchmentNcFile->addDim("catchments", catchments.size()); - auto catchments_var = catchmentNcFile->addVar("catchments", NC_STRING, catchments_dim); - catchments_var.putAtt("Catchment ID", "Catchment identifier in input"); - int item_index = 0; - std::vector index; - index.resize(1); - for (auto const& catchment : catchments) + + //Check if there is a need to create a netcdf file (whether output variables are mentioned in realization) + if (create_ncfile()){ + try{ + catchments.reserve(manager_->get_size()); + for (auto const& formulation_info : manager->get_all_formulations()) { - index[0] = item_index; - catchments_var.putVar(index,catchment); - item_index++; + std::string catchm = formulation_info.first; + catchments.push_back(catchm); } + + #if NGEN_WITH_MPI + if (mpi_num_procs > 1){ + std::vector all_catchments = catchments; + all_catchments = parallel::gather_strings(catchments, mpi_rank, mpi_num_procs); + if (mpi_rank == 0){ + std::sort(all_catchments.begin(), all_catchments.end()); + all_catchments.erase( + std::unique(all_catchments.begin(), all_catchments.end()), + all_catchments.end() + ); + } + catchments = parallel::broadcast_strings(all_catchments, mpi_rank, mpi_num_procs); + } + #endif + std::string ncOutputFileName = manager->get_output_root() + output_name + ".nc"; + if(mpi_rank == 0){ + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::replace); + if(!catchmentNcFile){ + LOG("Catchment output netcdf file creation failed: " + ncOutputFileName, LogLevel::FATAL); + throw std::runtime_error("Catchment output netcdf file creation failed: " + ncOutputFileName); + } + //Add dimension and coordinate variable for time + //TO DO: create a separate function if this action is initiated from outside the class. + //TO DO: convert seconds epoch to minutes or days? + int num_timesteps = sim_time_->get_total_output_times(); + auto time_dim = catchmentNcFile->addDim("time", num_timesteps); + auto time_var = catchmentNcFile->addVar("time", NC_INT, time_dim); + time_var.putAtt("units", "Seconds since 1970-01-01 00:00:00"); + time_var.putAtt("calendar", "gregorian"); + std::vector time_epoch_seconds(num_timesteps); + time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); + for(int time_index = 1; time_index < num_timesteps; time_index++) + { + sim_time_->advance_timestep(); + time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + } + time_var.putVar(time_epoch_seconds.data()); - //Add output data variables information such as headers, variable names, units to netcdf - //TO DO: change scope of this function if this is initiated from outside the class. - add_output_variable_info_from_formulation(); + //Add dimension and coordinate variable for catchments + //TO DO: create a separate function if this action is initiated from outside the class. + auto catchments_dim = catchmentNcFile->addDim("catchments", catchments.size()); + auto catchments_var = catchmentNcFile->addVar("catchments", NC_STRING, catchments_dim); + catchments_var.putAtt("Catchment ID", "Catchment identifier in input"); + int item_index = 0; + std::vector index; + index.resize(1); + for (auto const& catchment : catchments) + { + index[0] = item_index; + catchments_var.putVar(index,catchment); + item_index++; + } + + //Add output data variables information such as headers, variable names, units to netcdf + //TO DO: change scope of this function if this is initiated from outside the class. + add_output_variable_info_from_formulation(); + + //Add global attributes + catchmentNcFile->putAtt("title", "NextGen Catchment Output Data"); + catchmentNcFile->putAtt("description", "NetCDF file containing catchment-level output data from NextGen simulation"); + catchmentNcFile->putAtt("institution", "NOAA"); + } - //Add global attributes - catchmentNcFile->putAtt("title", "NextGen Catchment Output Data"); - catchmentNcFile->putAtt("description", "NetCDF file containing catchment-level output data from NextGen simulation"); - catchmentNcFile->putAtt("institution", "NOAA"); - } - - #if NGEN_WITH_MPI - if (mpi_num_procs > 1) MPI_Barrier(MPI_COMM_WORLD); - #endif + #if NGEN_WITH_MPI + if (mpi_num_procs > 1) MPI_Barrier(MPI_COMM_WORLD); + #endif - if (mpi_rank != 0){ - catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::write); - retrieve_output_variables_mpi(); + if (mpi_rank != 0){ + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::write); + retrieve_output_variables_mpi(); + } + } + catch (const netCDF::exceptions::NcException& e){ + LOG(std::string("Error in catchments NetCDF initiation: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in catchments NetCDF initiation: ") + e.what()); } } - catch (const netCDF::exceptions::NcException& e){ - LOG(std::string("Error in catchments NetCDF initiation: ") + e.what(), LogLevel::FATAL); - throw std::runtime_error(std::string("Error in catchments NetCDF initiation: ") + e.what()); + else{ + LOG("No output variables/headers information provided in the realization config. A NetCDF file for the catchment output will not be created.", LogLevel::INFO); } } @@ -121,8 +127,6 @@ void NetCDFCreator::add_output_variable_info_from_formulation() nc_output_variables[index].putAtt("_FillValue", NC_DOUBLE, -1.0); //TO DO: Change to another value, if recommended. nc_output_variables[index].putAtt("missing_value", NC_DOUBLE, -2.0); //TO DO: Change to another value, if recommended. } - }else{ - LOG("No output variables/headers information provided in the realization config. No output variables written to NetCDF.", LogLevel::WARNING); } } @@ -139,8 +143,6 @@ void NetCDFCreator::retrieve_output_variables_mpi() for(int index = 0; index < output_headers.size(); index ++){ nc_output_variables[index] = catchmentNcFile->getVar(output_headers[index]); } - }else{ - LOG("No output variables/headers information provided in the realization config. No output variables written to NetCDF.", LogLevel::WARNING); } } @@ -180,7 +182,15 @@ std::vector NetCDFCreator::string_split(std::string str, char delimiter) return res; } -netCDF::NcFile& NetCDFCreator::GetNcFile(){ +bool NetCDFCreator::create_ncfile() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + return (r_c->get_output_header_count() > 0) ? true : false; +} + +netCDF::NcFile& NetCDFCreator::get_ncfile(){ return *catchmentNcFile; } diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 648675691c..d3fa2ca9d3 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -46,10 +46,12 @@ void ngen::Layer::update_models(boost::span catchment_outflows, r_c->get_output_line_for_timestep(output_time_index)+"\n"; r_c->write_output(output); - //capture all the output values for this timestep to write to netcdf + //capture all the output values for this timestep to write to netcdf in non-MPI run. #if NGEN_WITH_NETCDF + #if !NGEN_WITH_MPI catchment_output_values[id] = r_c->get_output_line_for_timestep(output_time_index); - #endif + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF } //TODO put this somewhere else. For now, just trying to ensure we get m^3/s into nexus output double area; diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index dd4c7efc35..19e0d05bde 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -95,11 +95,10 @@ void NgenSimulation::advance_models_one_output_step() simulation_step_ ); // assume update_models() calls time->advance_timestep() - // After updating the layer, get the output data for that timestep and write to netcdf + // After updating the layer, get the output data for that timestep and write to netcdf. This is currently + //set up only for a non-MPI run. #if NGEN_WITH_NETCDF - #if NGEN_WITH_MPI - //do nothing? - #else + #if !NGEN_WITH_MPI std::map catchment_output_vals = layer->get_catchment_output_data_for_timestep(); nc_writer_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); #endif //NGEN_WITH_MPI diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 588dd8f2ed..c8fb7ce8ef 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -471,7 +471,7 @@ ngen_add_test( NGEN_WITH_NETCDF ) -########################## NgenSimulation Class Tests +########################## NetCDFCreator Class Tests ngen_add_test( test_netcdf_creator OBJECTS diff --git a/test/core/NetCDFCreatorTest.cpp b/test/core/NetCDFCreatorTest.cpp index f062ad4237..f99323dca0 100644 --- a/test/core/NetCDFCreatorTest.cpp +++ b/test/core/NetCDFCreatorTest.cpp @@ -206,7 +206,7 @@ class NetCDFCreatorTest : public ::testing::Test { TEST_F(NetCDFCreatorTest, TestCatchmentIdentifiers) { nc_creator = std::make_unique(manager_,"catchment_test", *sim_time_, 0, 1); - netCDF::NcFile& ncFile = nc_creator->GetNcFile(); + netCDF::NcFile& ncFile = nc_creator->get_ncfile(); netCDF::NcVar catchments_var = ncFile.getVar("catchments"); std::vector nc_dims = catchments_var.getDims(); size_t len = nc_dims[0].getSize(); @@ -251,7 +251,7 @@ TEST_F(NetCDFCreatorTest, TestOutputValues) std::vectoroutput_headers = r_c->get_output_header_fields(); std::vectoroutput_units = r_c->get_output_variable_units(); - netCDF::NcFile& ncFile = nc_creator->GetNcFile(); //get a handle to the netcdf file object + netCDF::NcFile& ncFile = nc_creator->get_ncfile(); //get a handle to the netcdf file object //find the index for "cat-52" and retrieve the values for the output variables double catchment_output_nc;