fix: Infinite hang when MPI process crashes

src/coreComponents/common/MpiWrapper.cpp

@@ -155,6 +155,7 @@ void internal::ManagedResources::finalize()
 void MpiWrapper::finalize()
 {
 #ifdef GEOS_USE_MPI
+  MpiWrapper::commFree( MPI_COMM_GEOS );
   internal::getManagedResources().finalize();
   MPI_CHECK_ERROR( MPI_Finalize() );
 #endif

src/coreComponents/common/MpiWrapper.hpp

@@ -196,7 +196,7 @@ struct MpiWrapper
   static int init( int * argc, char * * * argv );
 
   /**
-   * @brief Free MPI managed resources, then call MPI_Finalize().
+   * @brief Finalize the MPI environment and free MPI-managed resources.
    * Please note that once called, MPI functions, communicators and resources can no longer be used.
    */
   static void finalize();

src/coreComponents/common/initializeEnvironment.cpp

@@ -19,7 +19,6 @@
 #include "Path.hpp"
 #include "LvArray/src/system.hpp"
 #include "common/LifoStorageCommon.hpp"
-#include "common/MemoryInfos.hpp"
 #include "logger/ErrorHandling.hpp"
 #include "logger/ExternalErrorHandler.hpp"
 #include <umpire/TypedAllocator.hpp>
@@ -192,10 +191,12 @@ void setupMPI( int argc, char * argv[] )
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-void finalizeMPI()
+void finalizeMPI( bool inError )
 {
-  MpiWrapper::commFree( MPI_COMM_GEOS );
-  MpiWrapper::finalize();
+  if( !inError )
+  {
+    MpiWrapper::finalize();
+  }
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -330,13 +331,12 @@ void setupEnvironment( int argc, char * argv[] )
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-void cleanupEnvironment()
+void cleanupEnvironment( bool inError )
 {
-  MemoryLogging::getInstance().memoryStatsReport();
   LvArray::system::resetSignalHandling();
   finalizeLogger();
   finalizeCaliper();
-  finalizeMPI();
+  finalizeMPI( inError );
 }
 
 } // namespace geos

src/coreComponents/common/initializeEnvironment.hpp

@@ -135,8 +135,10 @@ void setupMPI( int argc, char * argv[] );
 
 /**
  * @brief Finalize MPI.
+ * @param inError inError Indicate if the simulation end with an exception.
+ * By default set to false
  */
-void finalizeMPI();
+void finalizeMPI( bool inError = false );
 
 /**
  * @brief Setup CUDA
@@ -153,8 +155,9 @@ void setupEnvironment( int argc, char * argv[] );
 
 /**
  * @brief Cleanup/finalize the environment.
+ * @param inError indicate if an exception occurred
  */
-void cleanupEnvironment();
+void cleanupEnvironment( bool inError = false );
 
 #if defined( GEOS_USE_CALIPER )
 

src/coreComponents/mainInterface/initialization.cpp

@@ -309,10 +309,10 @@ std::unique_ptr< CommandLineOptions > basicSetup( int argc, char * argv[], bool
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-void basicCleanup()
+void basicCleanup( bool inError )
 {
   finalizeLAI();
-  cleanupEnvironment();
+  cleanupEnvironment( inError );
 }
 
 

src/coreComponents/mainInterface/initialization.hpp

@@ -42,8 +42,9 @@ std::unique_ptr< CommandLineOptions > basicSetup( int argc, char * argv[], bool
 
 /**
  * @brief Perform the basic GEOSX cleanup.
+ * @param inError indicate if an exception occurred
  */
-void basicCleanup();
+void basicCleanup( bool inError = false );
 
 
 

src/coreComponents/mesh/unitTests/testGraphColoringMPI.cpp

@@ -162,7 +162,6 @@ int main( int argc, char * *argv )
   int result = RUN_ALL_TESTS();
 
   // Finalize MPI
-  MpiWrapper::commFree( MPI_COMM_GEOS );
   MpiWrapper::finalize();
 
   return result;

src/main/main.cpp

@@ -16,6 +16,7 @@
 // Source includes
 #include "common/logger/ErrorHandling.hpp"
 #include "common/logger/Logger.hpp"
+#include "common/MemoryInfos.hpp"
 #include "common/TimingMacros.hpp"
 #include "common/Units.hpp"
 #include "mainInterface/initialization.hpp"
@@ -56,7 +57,9 @@ int main( int argc, char *argv[] )
       runTime = state.getRunTime();
     }
 
-    basicCleanup();
+    MemoryLogging::getInstance().memoryStatsReport();
+
+    basicCleanup( false );
 
     std::chrono::system_clock::time_point endTime = std::chrono::system_clock::now();
     std::chrono::system_clock::duration totalTime = endTime - startTime;
@@ -71,14 +74,13 @@ int main( int argc, char *argv[] )
   // A NotAnError is thrown if "-h" or "--help" option is used.
   catch( NotAnError const & )
   {
-    basicCleanup();
+    basicCleanup( false );
     return 0;
   }
   catch( geos::Exception & e )
   { // GEOS generated exceptions management
     ErrorLogger::global().flushCurrentExceptionMessage();
-    basicCleanup();
-    // lvarray error handler is just program termination
+    basicCleanup( true );
     LvArray::system::callErrorHandler();
   }
   catch( std::exception const & e )
@@ -88,8 +90,7 @@ int main( int argc, char *argv[] )
                                            ::geos::logger::internal::g_rank )
                                            .addCallStackInfo( LvArray::system::stackTrace( true ) )
                                            .getDiagnosticMsg());
-    basicCleanup();
-    // lvarray error handler is just program termination
+    basicCleanup( true );
     LvArray::system::callErrorHandler();
   }
   return 0;