Read New NHF Schema (NGWPC-9422)

CMakeLists.txt

@@ -372,7 +372,6 @@ target_link_libraries(ngen
         NGen::parallel
         NGen::state_save_restore
         NGen::bmi_protocols
-        NGen::state_save_restore
 )
 
 if(NGEN_WITH_SQLITE)

Dockerfile

@@ -7,10 +7,10 @@ ARG ORG=ngwpc
 ARG NGEN_FORCING_IMAGE_TAG=latest
 ARG NGEN_FORCING_IMAGE=ghcr.io/${ORG}/ngen-bmi-forcing:${NGEN_FORCING_IMAGE_TAG}
 
-FROM ${NGEN_FORCING_IMAGE} AS base
+#FROM ${NGEN_FORCING_IMAGE} AS base
 
 # Uncomment when building locally
-#FROM ngen-bmi-forcing AS base
+FROM ngen-bmi-forcing AS base
 
 # OCI Metadata Arguments
 ARG NGEN_FORCING_IMAGE
@@ -471,6 +471,11 @@ RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \
     cd extern/topoflow-glacier; \
     pip install .
 
+RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \
+    set -eux; \
+    cd extern/topoflow-glacier; \
+    pip install .
+
 RUN set -eux && \
     mkdir --parents /ngencerf/data/ngen-run-logs/ && \
     mkdir --parents /ngen-app/bin/ && \

include/core/nexus/HY_PointHydroNexusRemote.hpp

@@ -72,7 +72,6 @@ class HY_PointHydroNexusRemote : public HY_PointHydroNexus
 		communication_type get_communicator_type() { return type; }
 
     private:
-        void post_receives();
         void process_communications();
 
         int world_rank;

include/realizations/catchment/Bmi_Module_Formulation.hpp

@@ -7,8 +7,6 @@
 #include "Bmi_Adapter.hpp"
 #include <DataProvider.hpp>
 #include "bmi_utilities.hpp"
-
-#include <boost/core/span.hpp>
 #include "bmi/protocols.hpp"
 
 using data_access::MEAN;
@@ -59,11 +57,6 @@ namespace realization {
 
         /**
          * Get the collection of forcing output property names this instance can provide.
-         *
-         * This is part of the @ref ForcingProvider interface.  This interface must be implemented for items of this
-         * type to be usable as "forcing" providers for situations when some other object needs to receive as an input
-         * (i.e., one of its forcings) a data property output from this object.
-         *
          * For this type, this is the collection of BMI output variables, plus any aliases included in the formulation
          * config's output variable mapping.
          *

include/realizations/catchment/Bmi_Multi_Formulation.hpp

@@ -627,7 +627,7 @@ namespace realization {
 
             // Since this is a nested formulation, support usage of the '{{id}}' syntax for init config file paths.
             Catchment_Formulation::config_pattern_substitution(properties, BMI_REALIZATION_CFG_PARAM_REQ__INIT_CONFIG,
-                                                               "{{id}}", id);
+                                                               "{{id}}", Catchment_Formulation::config_pattern_id_replacement(id));
 
             // Call create_formulation to perform the rest of the typical initialization steps for the formulation.
             mod->create_formulation(properties);

include/realizations/catchment/Catchment_Formulation.hpp

@@ -32,6 +32,12 @@ namespace realization {
             static void config_pattern_substitution(geojson::PropertyMap &properties, const std::string &key,
                                                     const std::string &pattern, const std::string &replacement);
 
+            /**Remove leading non-numeric characters from the ID string.
+             * 
+             * This may be needed to correct NGEN adding an identifying prefix to the ID with system file names without the prefix.
+             */ 
+            static std::string config_pattern_id_replacement(const std::string &id);
+
             /**
              * Get a header line appropriate for a file made up of entries from this type's implementation of
              * ``get_output_line_for_timestep``.

include/realizations/catchment/Formulation_Manager.hpp

@@ -170,13 +170,18 @@ namespace realization {
                     for (std::pair<std::string, boost::property_tree::ptree> catchment_config : *possible_catchment_configs) {
                         ss.str(""); ss << "Processing catchment: " << catchment_config.first << std::endl;
                         LOG(ss.str(), LogLevel::DEBUG);
+                        // ensure catchment's ID starts with "cat-" so it can be found in the fabric
+                        std::string catchment_id = catchment_config.first;
+                        if (strncmp(catchment_id.c_str(), "cat-", 4) != 0) {
+                            catchment_id = "cat-" + catchment_id;
+                        }
 
-                        int catchment_index = fabric->find(catchment_config.first);
+                        int catchment_index = fabric->find(catchment_id);
                         if (catchment_index == -1) {
                             #ifndef NGEN_QUIET
                                 ss.str("");
                                 ss << "Formulation_Manager::read: Cannot create formulation for catchment "
-                                   << catchment_config.first
+                                   << catchment_id
                                    << " that isn't identified in the hydrofabric or requested subset" << std::endl;
                                 LOG(ss.str(), LogLevel::WARNING);
                             #endif
@@ -200,7 +205,7 @@ namespace realization {
                         this->add_formulation(
                             this->construct_formulation_from_config(
                                 simulation_time_config,
-                                catchment_config.first,
+                                catchment_id,
                                 catchment_formulation,
                                 output_stream
                             )
@@ -553,7 +558,7 @@ namespace realization {
                                 global_copy.formulation.parameters,
                                 BMI_REALIZATION_CFG_PARAM_REQ__INIT_CONFIG,
                                 "{{id}}",
-                                identifier
+                                Catchment_Formulation::config_pattern_id_replacement(identifier)
                             );
                         } else {
                             ss.str(""); ss << "init_config is present but empty for identifier: " << identifier << std::endl;
@@ -665,7 +670,9 @@ namespace realization {
 
                 // Replace {{id}} if present
                 if (id_index != std::string::npos) {
-                    filepattern = filepattern.replace(id_index, sizeof("{{id}}") - 1, identifier);
+                    // account generate the regex to search for the ID with or without a prefix
+                    std::string pattern_id = Catchment_Formulation::config_pattern_id_replacement(identifier);
+                    filepattern = filepattern.replace(id_index, sizeof("{{id}}") - 1, pattern_id);
                 }
 
                 // Compile the file pattern as a regex

src/NGen.cpp

@@ -448,11 +448,12 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) {
 #if NGEN_WITH_SQLITE3
         try {
             nexus_collection = ngen::geopackage::read(nexusDataFile, "nexus", nexus_subset_ids);
-        } catch (...) {
+        } catch (std::exception &e) {
             // Handle all exceptions
             std::string msg = "Geopackage error occurred reading nexuses: " + nexusDataFile;
             LOG(msg,LogLevel::FATAL);
-            throw std::runtime_error(msg);
+            LOG(LogLevel::FATAL, e.what());
+            throw;
         }
 #else
         LOG(LogLevel::FATAL, "SQLite3 support required to read GeoPackage files.");
@@ -480,11 +481,12 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) {
         try {
         catchment_collection =
             ngen::geopackage::read(catchmentDataFile, "divides", catchment_subset_ids);
-        } catch (...) {
+        } catch (std::exception &e) {
             // Handle all exceptions
             std::string msg = "Geopackage error occurred reading divides: " + catchmentDataFile;
             LOG(msg,LogLevel::FATAL);
-            throw std::runtime_error(msg);
+            LOG(LogLevel::FATAL, e.what());
+            throw;
         }
 
 #else