diff --git a/CMakeLists.txt b/CMakeLists.txt index ecf309e715..d3125a1b04 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,6 +355,27 @@ if(NGEN_WITH_SQLITE) target_include_directories(partitionGenerator PUBLIC AFTER "${NGEN_INC_DIR}/geopackage") endif() +if(NGEN_WITH_NETCDF) + add_library(NetCDFCreator STATIC "${NGEN_SRC_DIR}/NetCDFCreator.cpp") + target_include_directories(NetCDFCreator PUBLIC + ${NGEN_INC_DIR} + ${NGEN_INC_DIR}/core + ${NGEN_INC_DIR}/core/catchment + ${NGEN_INC_DIR}/realizations/catchment + ${NGEN_INC_DIR}/simulation_time + ${NGEN_INC_DIR}/utilities + ${NGEN_INC_DIR}/geojson + ${NGEN_INC_DIR}/forcing + ${NGEN_INC_DIR}/core/mediator + ${Python_INCLUDE_DIRS} + ${pybind11_INCLUDE_DIR} + ${MPI_CXX_INCLUDE_DIRS} + ) + target_link_libraries(NetCDFCreator PUBLIC NGen::config_header) + target_link_libraries(NetCDFCreator PRIVATE NetCDF) + target_link_libraries(ngen PRIVATE NetCDFCreator) +endif() + target_link_libraries(partitionGenerator PUBLIC NGen::core NGen::geojson) if(NGEN_WITH_SQLITE) target_link_libraries(partitionGenerator PUBLIC NGen::geopackage) diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index 5c3c4481fa..c7e84a4db9 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -9,6 +9,7 @@ #include "State_Exception.hpp" #include "geojson/FeatureBuilder.hpp" #include +#include namespace hy_features { @@ -110,6 +111,8 @@ namespace ngen std::unordered_map &nexus_indexes, int current_step); + std::map get_catchment_output_data_for_timestep(); + protected: const LayerDescription description; @@ -121,6 +124,7 @@ namespace ngen //TODO is this really required at the top level? or can this be moved to SurfaceLayer? const geojson::GeoJSON catchment_data; long output_time_index; + std::map catchment_output_values; }; } diff --git a/include/core/NetCDFCreator.hpp b/include/core/NetCDFCreator.hpp new file mode 100644 index 0000000000..e5a1fe425a --- /dev/null +++ b/include/core/NetCDFCreator.hpp @@ -0,0 +1,47 @@ +#ifndef NGEN_NETCDF_CREATOR_HPP +#define NGEN_NETCDF_CREATOR_HPP + +#include + +#if NGEN_WITH_NETCDF +#include +#include +#include +#include + +namespace netCDF { + class NcVar; + class NcFile; +} +class NetCDFCreator +{ +public: + NetCDFCreator(std::shared_ptr manager, + const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs); + NetCDFCreator() = delete; + ~NetCDFCreator(); + + void write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values); + + netCDF::NcFile& get_ncfile(); + +protected: + void add_output_variable_info_from_formulation(); + + void retrieve_output_variables_mpi(); + + std::vector string_split(std::string str, char delimiter); + + bool create_ncfile(); + + void close_ncfile(); + +private: + std::shared_ptr catchmentNcFile; + std::shared_ptr manager_; + std::shared_ptr sim_time_; + std::vector catchments; + std::vector nc_output_variables; +}; +#endif // NGEN_WITH_NETCDF +#endif // NGEN_NETCDF_CREATOR_HPP \ No newline at end of file diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index 00e5ef49eb..ba61a1fc88 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -4,8 +4,13 @@ #include #include +#include #include +#if NGEN_WITH_NETCDF + #include +#endif + namespace hy_features { class HY_Features; @@ -59,6 +64,8 @@ class NgenSimulation size_t get_num_output_times() const; std::string get_timestamp_for_step(int step) const; + void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs); + private: void advance_models_one_output_step(); @@ -81,6 +88,9 @@ class NgenSimulation // Serialization template will be defined and instantiated in the .cpp file template void serialize(Archive& ar); + + //Pointer to netcdfcreator to write simulation output per timestep. + std::unique_ptr nc_writer_; }; #endif diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index f2a13074e8..2adaa2dbab 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -180,6 +180,15 @@ namespace realization { return REQUIRED_PARAMETERS; } + /** + * Get the units for the output variables organized as a vector of strings. + * + * @return The units for the output variables organized as a vector. + */ + const std::vector &get_output_variable_units() const { + return output_variable_units; + } + virtual bool is_bmi_input_variable(const std::string &var_name) const = 0; /** @@ -238,6 +247,10 @@ namespace realization { output_header_fields = output_headers; } + void set_output_variable_units(const std::vector &output_units) { + output_variable_units = output_units; + } + /** * Set the names of variables in formulation output. * @@ -264,6 +277,11 @@ namespace realization { * the BMI module output variables accessible to the instance. */ std::vector output_variable_names; + /** + * Output units corresponding to the variables output by the realization, as defined in + * `output_variables`. + */ + std::vector output_variable_units; /** The degree of precision in output values when converting to text. */ int output_precision; diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index df6599f2a5..be34ca6aef 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -232,6 +232,10 @@ namespace realization { return this->formulations.at(id); } + std::map> get_all_formulations() const { + return this->formulations; + } + std::shared_ptr get_domain_formulation(long id) const { return this->domain_formulations.at(id); } diff --git a/src/NGen.cpp b/src/NGen.cpp index c554ef751a..75019f4fdf 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -11,6 +11,10 @@ #include #include +#if NGEN_WITH_NETCDF +#include +#endif + #if NGEN_WITH_SQLITE3 #include #endif @@ -712,12 +716,19 @@ int main(int argc, char* argv[]) { std::move(nexus_indexes), mpi_rank, mpi_num_procs); - + #if NGEN_WITH_NETCDF + #if NGEN_WITH_MPI + LOG("Under MPI mode, a NetCDF file for catchment output values is not created due to limitations in NetCDFCxx4 library.", LogLevel::INFO); + #else + simulation->create_netcdf_writer(manager, "catchment_output", mpi_rank, mpi_num_procs); //create a NetCDF file only for Non-MPI run. + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF auto time_done_init = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); - + simulation->run_catchments(); + #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); diff --git a/src/NetCDFCreator.cpp b/src/NetCDFCreator.cpp new file mode 100644 index 0000000000..e8cd56c391 --- /dev/null +++ b/src/NetCDFCreator.cpp @@ -0,0 +1,208 @@ +#include + +#if NGEN_WITH_NETCDF +#include +#include +#include + +#if NGEN_WITH_MPI +#include +#include "parallel_utils.h" +#endif + +NetCDFCreator::NetCDFCreator(std::shared_ptr manager, + const std::string& output_name,Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs) +{ + manager_ = manager; + sim_time_ = std::make_shared(sim_time); + + //Check if there is a need to create a netcdf file (whether output variables are mentioned in realization) + if (create_ncfile()){ + try{ + catchments.reserve(manager_->get_size()); + for (auto const& formulation_info : manager->get_all_formulations()) + { + std::string catchm = formulation_info.first; + catchments.push_back(catchm); + } + + #if NGEN_WITH_MPI + if (mpi_num_procs > 1){ + std::vector all_catchments = catchments; + all_catchments = parallel::gather_strings(catchments, mpi_rank, mpi_num_procs); + if (mpi_rank == 0){ + std::sort(all_catchments.begin(), all_catchments.end()); + all_catchments.erase( + std::unique(all_catchments.begin(), all_catchments.end()), + all_catchments.end() + ); + } + catchments = parallel::broadcast_strings(all_catchments, mpi_rank, mpi_num_procs); + } + #endif + std::string ncOutputFileName = manager->get_output_root() + output_name + ".nc"; + if(mpi_rank == 0){ + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::replace); + if(!catchmentNcFile){ + LOG("Catchment output netcdf file creation failed: " + ncOutputFileName, LogLevel::FATAL); + throw std::runtime_error("Catchment output netcdf file creation failed: " + ncOutputFileName); + } + //Add dimension and coordinate variable for time + //TO DO: create a separate function if this action is initiated from outside the class. + //TO DO: convert seconds epoch to minutes or days? + int num_timesteps = sim_time_->get_total_output_times(); + auto time_dim = catchmentNcFile->addDim("time", num_timesteps); + auto time_var = catchmentNcFile->addVar("time", NC_INT, time_dim); + time_var.putAtt("units", "Seconds since 1970-01-01 00:00:00"); + time_var.putAtt("calendar", "gregorian"); + std::vector time_epoch_seconds(num_timesteps); + time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); + for(int time_index = 1; time_index < num_timesteps; time_index++) + { + sim_time_->advance_timestep(); + time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + } + time_var.putVar(time_epoch_seconds.data()); + + //Add dimension and coordinate variable for catchments + //TO DO: create a separate function if this action is initiated from outside the class. + auto catchments_dim = catchmentNcFile->addDim("catchments", catchments.size()); + auto catchments_var = catchmentNcFile->addVar("catchments", NC_STRING, catchments_dim); + catchments_var.putAtt("Catchment ID", "Catchment identifier in input"); + int item_index = 0; + std::vector index; + index.resize(1); + for (auto const& catchment : catchments) + { + index[0] = item_index; + catchments_var.putVar(index,catchment); + item_index++; + } + + //Add output data variables information such as headers, variable names, units to netcdf + //TO DO: change scope of this function if this is initiated from outside the class. + add_output_variable_info_from_formulation(); + + //Add global attributes + catchmentNcFile->putAtt("title", "NextGen Catchment Output Data"); + catchmentNcFile->putAtt("description", "NetCDF file containing catchment-level output data from NextGen simulation"); + catchmentNcFile->putAtt("institution", "NOAA"); + } + + #if NGEN_WITH_MPI + if (mpi_num_procs > 1) MPI_Barrier(MPI_COMM_WORLD); + #endif + + if (mpi_rank != 0){ + catchmentNcFile = std::make_shared(ncOutputFileName, netCDF::NcFile::write); + retrieve_output_variables_mpi(); + } + } + catch (const netCDF::exceptions::NcException& e){ + LOG(std::string("Error in catchments NetCDF initiation: ") + e.what(), LogLevel::FATAL); + throw std::runtime_error(std::string("Error in catchments NetCDF initiation: ") + e.what()); + } + } + else{ + LOG("No output variables/headers information provided in the realization config. A NetCDF file for the catchment output will not be created.", LogLevel::INFO); + } +} + +void NetCDFCreator::add_output_variable_info_from_formulation() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + std::vectoroutput_units = r_c->get_output_variable_units(); + nc_output_variables.resize(output_variables.size()); + + std::vector dims = {catchmentNcFile->getDim("time"), catchmentNcFile->getDim("catchments")}; + for(int index = 0; index < output_variables.size(); index ++){ + nc_output_variables[index] = catchmentNcFile->addVar(output_headers[index], NC_DOUBLE, dims); + nc_output_variables[index].putAtt("variable name", output_variables[index]); + nc_output_variables[index].putAtt("variable units", output_units[index]); + nc_output_variables[index].putAtt("_FillValue", NC_DOUBLE, -1.0); //TO DO: Change to another value, if recommended. + nc_output_variables[index].putAtt("missing_value", NC_DOUBLE, -2.0); //TO DO: Change to another value, if recommended. + } + } +} + +void NetCDFCreator::retrieve_output_variables_mpi() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_headers = r_c->get_output_header_fields(); + nc_output_variables.resize(output_headers.size()); + + std::vector dims = {catchmentNcFile->getDim("time"), catchmentNcFile->getDim("catchments")}; + for(int index = 0; index < output_headers.size(); index ++){ + nc_output_variables[index] = catchmentNcFile->getVar(output_headers[index]); + } + } +} + +void NetCDFCreator::write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values) +{ + for (auto const& catchment_val : catchment_output_values) + { + std::string catchment_id = catchment_val.first; + + //iterate through catchment dimension to find the index of the catchment for writing. + //also split the comma separated outputs string to a vector of double values + std::vector catchment_output; + std::vector count = {1, 1}; + for (size_t c_index = 0; c_index < catchments.size(); ++c_index) + { + if(catchments[c_index] == catchment_id){ + catchment_output = string_split(catchment_val.second, ','); + std::vector start = {time_index, c_index}; + for(int var_index = 0; var_index < nc_output_variables.size(); ++var_index) + { + nc_output_variables[var_index].putVar(start, count, &catchment_output[var_index]); + } + break; + } + } + } +} + +std::vector NetCDFCreator::string_split(std::string str, char delimiter) +{ + std::stringstream ss(str); + std::vector res; + std::string token; + while (getline(ss, token, delimiter)) { //will return full string if no comma (or single output variable). + res.push_back(std::stod(token)); + } + return res; +} + +bool NetCDFCreator::create_ncfile() +{ + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + return (r_c->get_output_header_count() > 0) ? true : false; +} + +netCDF::NcFile& NetCDFCreator::get_ncfile(){ + return *catchmentNcFile; +} + +void NetCDFCreator::close_ncfile(){ + if (catchmentNcFile != nullptr) { + catchmentNcFile->close(); + } + catchmentNcFile = nullptr; +} + +NetCDFCreator::~NetCDFCreator() +{ + close_ncfile(); +} +#endif // NGEN_WITH_NETCDF \ No newline at end of file diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 56d6506be6..d3fa2ca9d3 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -45,6 +45,13 @@ void ngen::Layer::update_models(boost::span catchment_outflows, std::string output = std::to_string(output_time_index)+","+current_timestamp+","+ r_c->get_output_line_for_timestep(output_time_index)+"\n"; r_c->write_output(output); + + //capture all the output values for this timestep to write to netcdf in non-MPI run. + #if NGEN_WITH_NETCDF + #if !NGEN_WITH_MPI + catchment_output_values[id] = r_c->get_output_line_for_timestep(output_time_index); + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF } //TODO put this somewhere else. For now, just trying to ensure we get m^3/s into nexus output double area; @@ -83,3 +90,7 @@ void ngen::Layer::update_models(boost::span catchment_outflows, simulation_time.advance_timestep(); } } + +std::map ngen::Layer::get_catchment_output_data_for_timestep(){ + return catchment_output_values; +} diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index c5cce69103..19e0d05bde 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -21,7 +21,7 @@ NgenSimulation::NgenSimulation( std::unordered_map nexus_indexes, int mpi_rank, int mpi_num_procs - ) + ) : simulation_step_(0) , sim_time_(std::make_shared(sim_time)) , layers_(std::move(layers)) @@ -40,7 +40,6 @@ void NgenSimulation::run_catchments() { // Now loop some time, iterate catchments, do stuff for total number of output times auto num_times = get_num_output_times(); - for (; simulation_step_ < num_times; simulation_step_++) { // Make room for this output step's results catchment_outflows_.resize(catchment_outflows_.size() + catchment_indexes_.size(), 0.0); @@ -95,6 +94,16 @@ void NgenSimulation::advance_models_one_output_step() nexus_indexes_, simulation_step_ ); // assume update_models() calls time->advance_timestep() + + // After updating the layer, get the output data for that timestep and write to netcdf. This is currently + //set up only for a non-MPI run. + #if NGEN_WITH_NETCDF + #if !NGEN_WITH_MPI + std::map catchment_output_vals = layer->get_catchment_output_data_for_timestep(); + nc_writer_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); + #endif //NGEN_WITH_MPI + #endif //NGEN_WITH_NETCDF + prev_layer_time = layer_next_time; } else { layer_min_next_time = prev_layer_time = layer->current_timestep_epoch_time(); @@ -269,3 +278,8 @@ void NgenSimulation::serialize(Archive& ar) { ar & nexus_indexes_; ar & nexus_downstream_flows_; } + +void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs) +{ + this->nc_writer_ = std::make_unique(manager,nc_output_file_name,*sim_time_, mpi_rank, mpi_num_procs_); +} diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 7c541e2d50..46eacc7cd0 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -600,6 +600,7 @@ namespace realization { output_var_units[i] = get_provider_units_for_variable(names[i]); } } + set_output_variable_units(output_var_units); //check if output variable indices (for vector variables) are specified in config. If not, default to zero (first index). if(output_var_indices.size() == 0){ diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 145b7dfb73..f1ab5d7afb 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -233,6 +233,7 @@ void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap proper output_var_units[i] = get_provider_units_for_variable(names[i]); } } + set_output_variable_units(output_var_units); //check if output variable indices (for vector variables) are specified in config. If not, default to zero (first index). if(output_var_indices.size() == 0){ @@ -413,7 +414,6 @@ std::string Bmi_Multi_Formulation::get_output_line_for_timestep(int timestep, st if (timestep != (next_time_step_index - 1)) { throw std::invalid_argument("Only current time step valid when getting multi-module BMI formulation output"); } - // Start by first checking whether we are just using the last module's values if (is_out_vars_from_last_mod) { // The default behavior, which means we either diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 623dd7a481..c8fb7ce8ef 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -61,6 +61,7 @@ endif() add_executable(test_all) target_link_libraries(test_all PRIVATE gtest gtest_main) +target_link_libraries(test_all PRIVATE NetCDFCreator) set_target_properties(test_all PROPERTIES FOLDER test) target_include_directories(test_all PRIVATE ${CMAKE_CURRENT_LIST_DIR}/bmi) @@ -446,7 +447,12 @@ ngen_add_test( NGen::geojson NGen::logging NGen::core_mediator + NGen::realizations_catchment + NGen::forcing NGen::ngen_bmi + NetCDFCreator + REQUIRES + NGEN_WITH_NETCDF ) ########################### Netcdf Forcing Tests @@ -465,6 +471,26 @@ ngen_add_test( NGEN_WITH_NETCDF ) +########################## NetCDFCreator Class Tests +ngen_add_test( + test_netcdf_creator + OBJECTS + core/NetCDFCreatorTest.cpp + LIBRARIES + NGen::core + NGen::geopackage + NGen::geojson + NGen::logging + NGen::core_catchment + NGen::core_mediator + NGen::realizations_catchment + NGen::forcing + NGen::ngen_bmi + NetCDFCreator + REQUIRES + NGEN_WITH_NETCDF +) + ########################## Primary Combined Unit Test Target ngen_add_test( test_unit diff --git a/test/core/NetCDFCreatorTest.cpp b/test/core/NetCDFCreatorTest.cpp new file mode 100644 index 0000000000..f99323dca0 --- /dev/null +++ b/test/core/NetCDFCreatorTest.cpp @@ -0,0 +1,293 @@ +#include + + +#if NGEN_WITH_NETCDF +#include "gtest/gtest.h" +#include +#include +#include "Bmi_Testing_Util.hpp" +#include "FileChecker.h" +#include +#include +#include +#include + +class NetCDFCreatorTest : public ::testing::Test { + protected: + std::stringstream stream_; + std::shared_ptr manager_; + geojson::GeoJSON fabric = std::make_shared(); + std::shared_ptr sim_time_; + std::map> calculated_results; + std::unique_ptr nc_creator; + + typedef struct tm time_type; + std::shared_ptr start_date_time; + std::shared_ptr end_date_time; + + void SetUp() override { + SetupFormulationManager(); + } + + void TearDown() override { + + } + + //Construct a Formulation Manager and Simulation Time objects + void SetupFormulationManager() { + stream_ << fix_paths(example_config); + boost::property_tree::ptree realization_config; + boost::property_tree::json_parser::read_json(stream_, realization_config); + manager_ = std::make_shared(realization_config); + + auto possible_simulation_time = realization_config.get_child_optional("time"); + if (!possible_simulation_time) { + std::string throw_msg; throw_msg.assign("ERROR: No simulation time period defined."); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); + } + + auto simulation_time_config = realization::config::Time(*possible_simulation_time).make_params(); + sim_time_ = std::make_shared(simulation_time_config); + + std::ostream* raw_pointer = &std::cout; + std::shared_ptr s_ptr(raw_pointer, [](void*) {}); + utils::StreamHandler catchment_output(s_ptr); + + this->add_feature("cat-52"); + this->add_feature("cat-67"); + manager_->read(simulation_time_config, fabric, catchment_output); + } + + std::string fix_paths(std::string json) + { + std::vector forcing_paths = { + "./data/forcing/cat-52_2015-12-01 00_00_00_2015-12-30 23_00_00.csv", + "./data/forcing/cat-67_2015-12-01 00_00_00_2015-12-30 23_00_00.csv", + }; + std::vector v = {}; + for(unsigned int i = 0; i < path_options.size(); i++){ + v.push_back( path_options[i] + "data/forcing" ); + } + std::string dir = utils::FileChecker::find_first_readable(v); + if(dir != ""){ + std::string remove = "\"./data/forcing/\""; + std::string replace = "\""+dir+"\""; + boost::replace_all(json, remove , replace); + } + + //BMI_CPP_INIT_DIR_PATH + replace_paths(json, "{{BMI_CPP_INIT_DIR_PATH}}", "data/bmi/test_bmi_cpp"); + //EXTERN_DIR_PATH + replace_paths(json, "{{EXTERN_LIB_DIR_PATH}}", "extern/test_bmi_cpp/cmake_build/"); + + for (unsigned int i = 0; i < forcing_paths.size(); i++) { + if(json.find(forcing_paths[i]) == std::string::npos){ + continue; + } + std::vector v = {}; + for (unsigned int j = 0; j < path_options.size(); j++) { + v.push_back(path_options[j] + forcing_paths[i]); + } + std::string right_path = utils::FileChecker::find_first_readable(v); + if(right_path != forcing_paths[i]){ + std::cerr<<"Replacing "< v{path_options.size()}; + for(unsigned int i = 0; i < path_options.size(); i++) + v[i] = path_options[i] + replacement; + + const std::string dir = utils::FileChecker::find_first_readable(v); + if (dir == "") return; + + boost::replace_all(input, pattern, dir); + } + + void add_feature(std::string id) + { + geojson::three_dimensional_coordinates three_dimensions { + { + {1.0, 2.0}, + {3.0, 4.0}, + {5.0, 6.0} + }, + { + {7.0, 8.0}, + {9.0, 10.0}, + {11.0, 12.0} + } + }; + std::vector bounding_box{1.0, 2.0}; + geojson::PropertyMap properties{}; + + geojson::Feature feature = std::make_shared(geojson::PolygonFeature( + geojson::polygon(three_dimensions), + id, + properties + )); + + fabric->add_feature(feature); + } + + + std::vector path_options = { + "", + "../", + "../../", + "./test/", + "../test/", + "../../test/" + + }; + + const std::string example_config = + "{" + " \"global\": {" + " \"formulations\": [" + " {" + " \"name\": \"bmi_multi\"," + " \"params\": {" + " \"model_type_name\": \"bmi_multi_c++\"," + " \"forcing_file\": \"\"," + " \"init_config\": \"\"," + " \"allow_exceed_end_time\": true," + " \"main_output_variable\": \"OUTPUT_VAR_4\"," + " \"uses_forcing_file\": false," + " \"modules\": [" + " {" + " \"name\": \"bmi_c++\"," + " \"params\": {" + " \"model_type_name\": \"test_bmi_c++\"," + " \"library_file\": \"{{EXTERN_LIB_DIR_PATH}}" BMI_TEST_CPP_LIB_NAME "\"," + " \"init_config\": \"{{BMI_CPP_INIT_DIR_PATH}}/test_bmi_cpp_config_2.txt\"," + " \"allow_exceed_end_time\": true," + " \"main_output_variable\": \"OUTPUT_VAR_4\"," + " \"uses_forcing_file\": false," + " \"model_params\": {" + " \"MODEL_VAR_1\": {" + " \"source\": \"hydrofabric\"," + " \"from\": \"val\"" + " }," + " \"MODEL_VAR_2\": {" + " \"source\": \"hydrofabric\"" + " }" + " }," + " \"" BMI_REALIZATION_CFG_PARAM_OPT__VAR_STD_NAMES "\": {" + " \"INPUT_VAR_1\": \"APCP_surface\"," + " \"INPUT_VAR_2\": \"APCP_surface\"" + " }" + " }" + " }" + " ]" + " }" + " }" + " ]," + " \"forcing\": { " + " \"file_pattern\": \".*{{id}}.*.csv\"," + " \"path\": \"./data/forcing/\"," + " \"provider\": \"CsvPerFeature\"" + " }" + " }," + " \"time\": {" + " \"start_time\": \"2015-12-01 00:00:00\"," + " \"end_time\": \"2015-12-30 23:00:00\"," + " \"output_interval\": 3600" + " }" + "}"; +}; + +TEST_F(NetCDFCreatorTest, TestCatchmentIdentifiers) +{ + nc_creator = std::make_unique(manager_,"catchment_test", *sim_time_, 0, 1); + netCDF::NcFile& ncFile = nc_creator->get_ncfile(); + netCDF::NcVar catchments_var = ncFile.getVar("catchments"); + std::vector nc_dims = catchments_var.getDims(); + size_t len = nc_dims[0].getSize(); + int item_index = 0; + std::vector index; + index.resize(1); + std::vector catchments; + std::string catchment; + for(size_t i = 0; i < len; ++i){ + index[0] = item_index; + catchments_var.getVar(index, &catchment); + catchments.push_back(catchment.c_str()); + item_index++; + } + //delete the netcdf file that was created once the information is obtained. + ncFile.close(); + std::string ncOutputFileName = manager_->get_output_root() + "catchment_test.nc"; + const char* nc_filename = ncOutputFileName.c_str(); + int result = std::remove(nc_filename); + + ASSERT_EQ(catchments.size(), 2); + ASSERT_TRUE(std::find(catchments.begin(), catchments.end(), "cat-52") != catchments.end()); + ASSERT_TRUE(std::find(catchments.begin(), catchments.end(), "cat-67") != catchments.end()); +} + +TEST_F(NetCDFCreatorTest, TestOutputValues) +{ + nc_creator = std::make_unique(manager_,"catchment_test", *sim_time_, 0, 1); //create netcdf file or replace this with an existing file? + + //write outputs to netcdf at time index 0 for cat-52 + std::map catchment_output_values; + auto c_form = std::dynamic_pointer_cast(manager_->get_formulation("cat-52")); + double resp = c_form->get_response(0, 3600); + std::string out_resp = c_form->get_output_line_for_timestep(0); + catchment_output_values["cat-52"] = out_resp; + nc_creator->write_simulations_response_from_formulation(0, catchment_output_values); + + + auto r_c = std::dynamic_pointer_cast(manager_->get_formulation("cat-52")); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + std::vectoroutput_units = r_c->get_output_variable_units(); + + netCDF::NcFile& ncFile = nc_creator->get_ncfile(); //get a handle to the netcdf file object + + //find the index for "cat-52" and retrieve the values for the output variables + double catchment_output_nc; + std::string output_str; + netCDF::NcVar catchments_var = ncFile.getVar("catchments"); + std::vector nc_dims = catchments_var.getDims(); + size_t len = nc_dims[0].getSize(); + size_t catchment_index = 0; + std::vector index; + index.resize(1); + std::vector catchments; + std::string catchment; + for(size_t i = 0; i < len; ++i){ + index[0] = catchment_index; + catchments_var.getVar(index, &catchment); + std::string catchm = catchment.c_str(); + if(catchm == "cat-52"){ + //use the catchment_index to query the value from the netcdf file and write it to a string. + std::vector start = {0, catchment_index}; + std::vector count = {1, 1}; + for(int var_index = 0; var_index < output_variables.size(); var_index ++){ + netCDF::NcVar out_var = ncFile.getVar(output_headers[var_index]); + out_var.getVar(start, count, &catchment_output_nc); + output_str += (output_str.empty() ? "" : ",") + std::to_string(catchment_output_nc); + } + } + catchment_index++; + } + + //delete the netcdf file that was created once the information is obtained. + ncFile.close(); + std::string ncOutputFileName = manager_->get_output_root() + "catchment_test.nc"; + const char* nc_filename = ncOutputFileName.c_str(); + int result = std::remove(nc_filename); + + ASSERT_EQ(out_resp, output_str); + } +} +#endif