feat: Error reporting at the end of the simulatiuon

.github/workflows/ci_tests.yml

@@ -150,47 +150,47 @@ jobs:
       fail-fast : false
       matrix:
         include:
-          - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3)
-            CMAKE_BUILD_TYPE: Release
-            DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9
-            BUILD_SHARED_LIBS: ON
-            ENABLE_HYPRE: OFF
-            ENABLE_TRILINOS: ON
-            GEOS_ENABLE_BOUNDS_CHECK: ON
-            HOST_CONFIG: /spack-generated.cmake
-
-          - name: Ubuntu debug (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces
-            BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests"
-            CMAKE_BUILD_TYPE: Debug
-            DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10
-            BUILD_SHARED_LIBS: ON
-            ENABLE_HYPRE: OFF
-            ENABLE_TRILINOS: ON
-            GEOS_ENABLE_BOUNDS_CHECK: ON
-            RUNS_ON: streak2
-            NPROC: 8
-            DOCKER_RUN_ARGS: "--cpus=8 --memory=256g --runtime=nvidia -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.crt:/certs/ca-bundle.crt:ro"
-
-            HOST_CONFIG: /spack-generated.cmake
-
-          - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces
-            CMAKE_BUILD_TYPE: Release
-            DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10
-            BUILD_SHARED_LIBS: ON
-            ENABLE_HYPRE: OFF
-            ENABLE_TRILINOS: ON
-            GEOS_ENABLE_BOUNDS_CHECK: ON
-            HOST_CONFIG: /spack-generated.cmake
-
-          - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2)
-            CMAKE_BUILD_TYPE: Release
-            DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11
-            ENABLE_HYPRE: ON
-            ENABLE_TRILINOS: OFF
-            BUILD_SHARED_LIBS: ON
-            GEOS_ENABLE_BOUNDS_CHECK: OFF
-            GCP_BUCKET: geosx/ubuntu22.04-gcc11
-            HOST_CONFIG: /spack-generated.cmake
+          # - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3)
+          #   CMAKE_BUILD_TYPE: Release
+          #   DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9
+          #   BUILD_SHARED_LIBS: ON
+          #   ENABLE_HYPRE: OFF
+          #   ENABLE_TRILINOS: ON
+          #   GEOS_ENABLE_BOUNDS_CHECK: ON
+          #   HOST_CONFIG: /spack-generated.cmake
+
+          # - name: Ubuntu debug (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces
+          #   BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests"
+          #   CMAKE_BUILD_TYPE: Debug
+          #   DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10
+          #   BUILD_SHARED_LIBS: ON
+          #   ENABLE_HYPRE: OFF
+          #   ENABLE_TRILINOS: ON
+          #   GEOS_ENABLE_BOUNDS_CHECK: ON
+          #   RUNS_ON: streak2
+          #   NPROC: 8
+          #   DOCKER_RUN_ARGS: "--cpus=8 --memory=256g --runtime=nvidia -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.crt:/certs/ca-bundle.crt:ro"
+
+          #   HOST_CONFIG: /spack-generated.cmake
+
+          # - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces
+          #   CMAKE_BUILD_TYPE: Release
+          #   DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10
+          #   BUILD_SHARED_LIBS: ON
+          #   ENABLE_HYPRE: OFF
+          #   ENABLE_TRILINOS: ON
+          #   GEOS_ENABLE_BOUNDS_CHECK: OFF
+          #   HOST_CONFIG: /spack-generated.cmake
+
+          # - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2)
+          #   CMAKE_BUILD_TYPE: Release
+          #   DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11
+          #   ENABLE_HYPRE: ON
+          #   ENABLE_TRILINOS: OFF
+          #   BUILD_SHARED_LIBS: ON
+          #   GEOS_ENABLE_BOUNDS_CHECK: OFF
+          #   GCP_BUCKET: geosx/ubuntu22.04-gcc11
+          #   HOST_CONFIG: /spack-generated.cmake
 
           - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2)
             CMAKE_BUILD_TYPE: Release

src/coreComponents/common/CMakeLists.txt

@@ -39,10 +39,13 @@ set( common_headers
      GEOS_RAJA_Interface.hpp
      GeosxMacros.hpp
      MemoryInfos.hpp
+     logger/DiagnosticMessage.hpp
+     logger/GeosExceptions.hpp
      logger/GeosExceptions.hpp
      logger/Logger.hpp
      logger/ErrorHandling.hpp
      logger/ExternalErrorHandler.hpp
+     logger/LogHistory.hpp
      MpiWrapper.hpp
      Path.hpp
      Span.hpp
@@ -72,6 +75,7 @@ endif( )
 # Specify all sources
 #
 set( common_sources
+     format/EnumStrings.cpp
      format/table/TableData.cpp
      format/table/TableFormatter.cpp
      format/table/TableLayout.cpp
@@ -82,6 +86,7 @@ set( common_sources
      logger/Logger.cpp
      logger/ErrorHandling.cpp
      logger/ExternalErrorHandler.cpp
+     logger/LogHistory.cpp
      BufferAllocator.cpp
      MemoryInfos.cpp
      MpiWrapper.cpp

src/coreComponents/common/MpiWrapper.cpp

@@ -462,6 +462,26 @@ int MpiWrapper::nodeCommSize()
   return nodeCommSize;
 }
 
+void MpiWrapper::gatherStringOnRank0( string_view rankStr, std::function< void(string_view) > && func )
+{
+  std::vector< buffer_unit_type > localbuffer;
+  localbuffer.reserve( rankStr.size());
+  localbuffer.insert( localbuffer.end(), rankStr.begin(), rankStr.end());
+  auto [globalLogRecords, counts, offsets] =
+    MpiWrapper::gatherBufferRank0< std::vector< buffer_unit_type > >( localbuffer );
+  if( MpiWrapper::commRank() == 0 )
+  {
+    for( integer rankId = 0; rankId < MpiWrapper::commSize(); ++rankId )
+    {
+      if( counts[rankId] > 0 )
+      {
+        func( string( globalLogRecords.begin() + offsets[rankId],
+                      globalLogRecords.begin() + offsets[rankId]+ counts[rankId] ) );
+      }
+    }
+  }
+}
+
 namespace internal
 {
 

src/coreComponents/common/MpiWrapper.hpp

@@ -331,6 +331,88 @@ struct MpiWrapper
    */
   static int nodeCommSize();
 
+  /**
+   * @brief Structure holding the result from all the gather operation
+   * @tparam CONTAINER The container type holding the data.
+   * The underlying storage in CONTAINER must be contiguous
+   */
+  template< typename CONTAINER >
+  struct GatherResult
+  {
+    // Collected data who must be trivially copyable
+    CONTAINER data;
+    // Number of elements per row
+    stdVector< integer > counts;
+    // Starting index for each row in 'data'
+    stdVector< integer > offsets;
+  };
+
+/**
+ * @brief Gather buffers of varying sizes from all ranks to rank 0.
+ * @tparam CONTAINER The container type holding the data.
+ * @tparam VALUE_T The trivially copyable underlying data type (deduced automatically).
+ * @param localBuffer The local buffer to be gathered on rank 0.
+ * @return A struct containing:
+ * - 'data': all the gathered data on rank 0
+ * - 'counts': number of elements for each rank
+ * - 'offsets': starting index for each rank in 'data'
+ */
+  template<
+    typename CONTAINER,
+    typename VALUE_T = typename CONTAINER::value_type,
+    typename = std::enable_if_t<
+      std::is_trivially_copyable_v< VALUE_T > &&
+      std::is_same_v< decltype(std::declval< CONTAINER >().data()), VALUE_T * > &&
+      std::is_same_v< decltype(std::declval< CONTAINER >().size()), std::size_t >
+      >
+    >
+  static GatherResult< CONTAINER >
+  gatherBufferRank0( CONTAINER const & localBuffer )
+  {
+    integer const numRanks = MpiWrapper::commSize();
+    integer const numLocalValues  = static_cast< integer >(localBuffer.size());
+
+    GatherResult< CONTAINER > gatherResult;
+
+    if( MpiWrapper::commRank() == 0 )
+    {
+      gatherResult.counts.resize( numRanks );
+      gatherResult.offsets.resize( numRanks );
+    }
+
+
+    MpiWrapper::gather( &numLocalValues, 1, gatherResult.counts.data(), 1, 0 );
+
+    if( MpiWrapper::commRank() == 0 )
+    {
+      integer totalSize = 0;
+      for( integer i = 0; i < numRanks; ++i )
+      {
+        gatherResult.offsets[i] = totalSize;
+        totalSize += gatherResult.counts[i];
+      }
+      gatherResult.data.resize( totalSize );
+    }
+
+    MpiWrapper::gatherv( localBuffer.data(),
+                         numLocalValues,
+                         gatherResult.data.data(),
+                         gatherResult.counts.data(),
+                         gatherResult.offsets.data(),
+                         0 );
+
+    return gatherResult;
+  }
+
+  /**
+   * @brief Gather srting from all ranks to rank 0
+   * @tparam FUNC Callable type invoked as void(string_view) for each non-empty rank string.
+   * @param str The local string to send from the calling rank.
+   * @param func Callback invoked on rank 0 for each non-empty received string.
+   */
+  static void gatherStringOnRank0( string_view str,
+                                   std::function< void(string_view) > && func );
+
   /**
    * @brief Strongly typed wrapper around MPI_Allgather.
    * @tparam T_SEND The pointer type for \p sendbuf

src/coreComponents/common/format/EnumStrings.cpp

@@ -0,0 +1,42 @@
+/*
+ * ------------------------------------------------------------------------------------------------------------
+ * SPDX-License-Identifier: LGPL-2.1-only
+ *
+ * Copyright (c) 2016-2024 Lawrence Livermore National Security LLC
+ * Copyright (c) 2018-2024 TotalEnergies
+ * Copyright (c) 2018-2024 The Board of Trustees of the Leland Stanford Junior University
+ * Copyright (c) 2023-2024 Chevron
+ * Copyright (c) 2019-     GEOS/GEOSX Contributors
+ * All rights reserved
+ *
+ * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details.
+ * ------------------------------------------------------------------------------------------------------------
+ */
+
+/**
+ * @file EnumStrings.cpp
+ */
+
+ #include "common/format/EnumStrings.hpp"
+ #include "common/logger/Logger.hpp"
+
+namespace geos
+{
+
+void internal::EnumErrorMessageToString( size_t index,
+                                         string_view typeName,
+                                         std::size_t size )
+{
+  GEOS_THROW( "Invalid value " << index << " of type " << typeName<< ". Valid range is 0.." << size - 1,
+              InputError );
+}
+
+void internal::EnumErrorMessageFromString( string_view s,
+                                           string_view typeName,
+                                           string_view concat )
+{
+  GEOS_THROW( "Invalid value '" << s << "' of type " << typeName << ". Valid options are: " << concat,
+              InputError );
+}
+
+}

src/coreComponents/common/format/EnumStrings.hpp

@@ -28,7 +28,6 @@
 #include "common/format/StringUtilities.hpp"
 // #include "codingUtilities/RTTypes.hpp"
 #include "common/DataTypes.hpp"
-#include "common/logger/Logger.hpp"
 #include "common/format/Format.hpp"
 
 #include <iostream>
@@ -40,6 +39,25 @@ namespace geos
 
 namespace internal
 {
+
+/**
+ * @brief Raise a throw error in case of bad use of enum::toString
+ * @param index The index of the enum causing the error
+ * @param typeName The enum name
+ * @param size The size of the enum
+ */
+void EnumErrorMessageToString( size_t index, string_view typeName, std::size_t size );
+
+/**
+ * @brief Raise a throw error in case of bad use of enum::FromString
+ * @param s The string that not belonging to the enum
+ * @param typeName The enum name
+ * @param size The string concatenation of the enum
+ */
+void EnumErrorMessageFromString( string_view s,
+                                 string_view typeName,
+                                 string_view concat );
+
 /**
  * @brief Simple compile-time variadic function that counts its arguments.
  * @tparam ARGS variadic pack of argument types
@@ -152,12 +170,10 @@ struct EnumStrings
     auto const & strings = get();
     std::size_t size = std::distance( std::begin( strings ), std::end( strings ) );
     base_type const index = static_cast< base_type >( e );
-    GEOS_THROW_IF( index >= LvArray::integerConversion< base_type >( size ),
-                   GEOS_FMT( "Invalid value {} of type {}. Valid range is 0..{}",
-                             index,
-                             getEnumTypeNameString( enum_type{} ),
-                             size - 1 ),
-                   InputError );
+    if( index >= LvArray::integerConversion< base_type >( size ))
+    {
+      internal::EnumErrorMessageToString( index, getEnumTypeNameString( enum_type{} ), size - 1 );
+    }
     return strings[ index ];
   }
 
@@ -170,12 +186,10 @@ struct EnumStrings
   {
     auto const & strings = get();
     auto const it = std::find( std::begin( strings ), std::end( strings ), s );
-    GEOS_THROW_IF( it == std::end( strings ),
-                   GEOS_FMT( "Invalid value '{}' of type {}. Valid options are: {}",
-                             s,
-                             getEnumTypeNameString( enum_type{} ),
-                             concat( ", " ) ),
-                   InputError );
+    if( it == std::end( strings ))
+    {
+      internal::EnumErrorMessageFromString( s, getEnumTypeNameString( enum_type{} ), EnumStrings::concat( ", " ));
+    }
     enum_type const e = static_cast< enum_type >( LvArray::integerConversion< base_type >( std::distance( std::begin( strings ), it ) ) );
     return e;
   }

src/coreComponents/common/format/LogPart.cpp

@@ -18,18 +18,22 @@
 
 #include "LogPart.hpp"
 #include "common/format/StringUtilities.hpp"
+#include "common/logger/ErrorHandling.hpp"
 #include <algorithm>
 
 using namespace geos::stringutilities;
 namespace geos
 {
 
-LogPart::LogPart( string_view logPartTitle, bool enableOutput )
+LogPart::LogPart( string_view logpartName, bool enableOutput )
 {
-  m_formattedStartDescription.m_title = logPartTitle;
-  m_formattedEndDescription.m_title = GEOS_FMT( "{}{}", m_prefixEndTitle, logPartTitle );
+  m_formattedStartDescription.m_title = logpartName;
+  m_formattedEndDescription.m_title = GEOS_FMT( "{}{}", m_prefixEndTitle, logpartName );
 
   m_enableOutput = enableOutput;
+
+  ErrorLogger::global().setCurrentLogPart( std::string( logpartName ) );
+
 }
 
 void LogPart::addDescription( string_view description )

src/coreComponents/common/format/LogPart.hpp

@@ -19,9 +19,8 @@
 #ifndef GEOS_COMMON_FORMAT_LOGPART_HPP
 #define GEOS_COMMON_FORMAT_LOGPART_HPP
 
-#include "common/DataTypes.hpp"
-#include "common/format/Format.hpp"
 #include "common/format/StringUtilities.hpp"
+#include "common/format/Format.hpp"
 
 namespace geos
 {