Merge Pull Request trilinos#12076 from trilinos/Trilinos/master_merge…

…_20230721_175808

Automatically Merged using Trilinos Master Merge AutoTester
PR Title: Trilinos Master Merge PR Generator: Auto PR created to promote from master_merge_20230721_175808 branch to master
PR Author: trilinos-autotester

cmake/ctest/drivers/geminga/TrilinosCTestDriverCore.geminga.gcc-cuda.cmake

@@ -126,7 +126,6 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
 
       ### PACKAGE CONFIGURATION ###
           "-DKokkos_ENABLE_CUDA:BOOL=ON"
-          "-DKokkos_ENABLE_CUDA_UVM:BOOL=ON"
           "-DKokkos_ENABLE_CUDA_LAMBDA:BOOL=ON"
           "-DKokkos_ARCH_KEPLER35:BOOL=ON"
           "-DTrilinos_ENABLE_Epetra:BOOL=OFF"

packages/framework/ini-files/config-specs.ini

@@ -1136,6 +1136,9 @@ use ATS2-RUN-SERIAL-TESTS
 [RHEL7]
 use COMMON
 
+[RHEL8]
+use COMMON
+
 [RHEL7_POST]
 
 [RHEL7_TEST_DISABLES|INTEL]
@@ -1577,6 +1580,9 @@ use RHEL7_COMPILER|INTEL
 opt-set-cmake-var TPL_BLAS_LIBRARIES   STRING : -mkl
 opt-set-cmake-var TPL_LAPACK_LIBRARIES STRING : -mkl
 
+[RHEL8_COMPILER|CUDA]
+use RHEL7_COMPILER|CUDA
+
 [RHEL7_SEMS_LIB-TYPE|STATIC]
 use LIB-TYPE|STATIC
 # Use .so's rather than .a's here due to missing HDF5 symbols
@@ -1586,6 +1592,12 @@ opt-set-cmake-var TPL_HDF5_LIBRARIES STRING : ${SEMS_HDF5_LIBRARY_PATH|ENV}/libh
 use LIB-TYPE|SHARED
 opt-set-cmake-var TPL_HDF5_LIBRARIES STRING : ${SEMS_HDF5_LIBRARY_PATH|ENV}/libhdf5_hl.so;${SEMS_HDF5_LIBRARY_PATH|ENV}/libhdf5.so;${SEMS_ZLIB_LIBRARY_PATH|ENV}/libz.so;-ldl
 
+[RHEL8_SEMS_LIB-TYPE|STATIC]
+use LIB-TYPE|STATIC
+
+[RHEL8_SEMS_LIB-TYPE|SHARED]
+use LIB-TYPE|SHARED
+
 [RHEL7_SEMS_V2_LIB-TYPE|STATIC]
 use LIB-TYPE|STATIC
 opt-set-cmake-var TPL_HDF5_LIBRARIES STRING : ${HDF5_LIB|ENV}/libhdf5_hl.a;${HDF5_LIB|ENV}/libhdf5.a;${ZLIB_LIB|ENV}/libz.a;-ldl
@@ -1974,6 +1986,7 @@ opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING       : -
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                BOOL         : ON
 opt-set-cmake-var CMAKE_CXX_EXTENSIONS                          BOOL         : OFF
 opt-set-cmake-var Teko_DISABLE_LSCSTABALIZED_TPETRA_ALPAH_INV_D BOOL         : ON
+opt-set-cmake-var ROL_test_algorithm_TypeP_CompareTypeU_MPI_1_DISABLE BOOL         : ON
 opt-set-cmake-var CMAKE_CXX_FLAGS                               STRING       : -fno-strict-aliasing -Wall -Wno-clobbered -Wno-vla -Wno-pragmas -Wno-unknown-pragmas -Wno-parentheses -Wno-unused-local-typedefs -Wno-literal-suffix -Wno-deprecated-declarations -Wno-misleading-indentation -Wno-int-in-bool-context -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-inline -Wno-nonnull-compare -Wno-address -Werror -Werror=shadow -DTRILINOS_HIDE_DEPRECATED_HEADER_WARNINGS
 
 use GCC_OPENMP_PACKAGE_SPECIFIC_WARNING_FLAGS
@@ -2302,6 +2315,115 @@ opt-set-cmake-var CMAKE_CXX_FLAGS                          STRING FORCE : -Wall
 #  MPI issue (TRILFRAME-552)
 opt-set-cmake-var ROL_example_PinT_parabolic-control_AugmentedSystem_test_MPI_2_DISABLE BOOL FORCE : ON
 
+[rhel8_sems-cuda-11.4.2-sems-gnu-10.1.0-sems-openmpi-4.1.4_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_no-uvm_deprecated-on_no-package-enables]
+# uses sems-v2 modules
+use RHEL8_COMPILER|CUDA
+use NODE-TYPE|CUDA
+use BUILD-TYPE|RELEASE
+use RHEL8_SEMS_LIB-TYPE|STATIC
+use KOKKOS-ARCH|VOLTA70
+use USE-ASAN|NO
+use USE-FPIC|NO
+use USE-MPI|YES
+use USE-PT|NO
+use RHEL8
+use USE-COMPLEX|YES
+use USE-RDC|NO
+use USE-UVM|NO
+use USE-DEPRECATED|YES
+use PACKAGE-ENABLES|NO-PACKAGE-ENABLES
+use COMMON_SPACK_TPLS
+use SEMS_COMMON_CUDA_11
+
+# TPL ENABLE/DISABLE settings
+opt-set-cmake-var TPL_ENABLE_BLAS BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_BinUtils BOOL FORCE : OFF
+opt-set-cmake-var TPL_ENABLE_Boost BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_CGNS BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_CUDA BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_CUSPARSE BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_DLlib BOOL FORCE : OFF
+opt-set-cmake-var TPL_ENABLE_HDF5 BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_HWLOC BOOL FORCE : OFF
+opt-set-cmake-var TPL_ENABLE_LAPACK BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_METIS BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_Matio BOOL FORCE : OFF
+opt-set-cmake-var TPL_ENABLE_MPI BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_Netcdf BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_ParMETIS BOOL FORCE : OFF
+opt-set-cmake-var TPL_ENABLE_Pthread BOOL FORCE : ON
+opt-set-cmake-var TPL_ENABLE_Scotch BOOL FORCE : OFF
+opt-set-cmake-var TPL_ENABLE_SuperLU BOOL FORCE : OFF
+opt-set-cmake-var TPL_ENABLE_SuperLUDist BOOL FORCE : OFF
+opt-set-cmake-var TPL_ENABLE_Zlib BOOL FORCE : ON
+
+#TPL_*_LIBRARIES
+# see https://github.com/trilinos/Trilinos/issues/11109#issuecomment-1272146298
+opt-set-cmake-var TPL_BLAS_LIBRARIES STRING FORCE : /lib64/libblas.so.3
+opt-set-cmake-var TPL_BoostLib_LIBRARIES STRING FORCE : ${BOOST_LIB|ENV}/libboost_program_options.a;${BOOST_LIB|ENV}/libboost_system.a
+opt-set-cmake-var TPL_Boost_LIBRARIES STRING FORCE : ${BOOST_LIB|ENV}/libboost_program_options.a;${BOOST_LIB|ENV}/libboost_system.a
+opt-set-cmake-var TPL_DLlib_LIBRARIES FILEPATH FORCE : "-ldl"
+opt-set-cmake-var TPL_HDF5_LIBRARIES STRING FORCE : ${HDF5_LIB|ENV}/libhdf5_hl.so;${HDF5_LIB|ENV}/libhdf5.a;${ZLIB_LIB|ENV}/libz.a;-ldl
+# see https://github.com/trilinos/Trilinos/issues/11109#issuecomment-1272146298
+opt-set-cmake-var TPL_LAPACK_LIBRARIES STRING FORCE : /lib64/liblapack.so.3
+opt-set-cmake-var TPL_METIS_LIBRARIES STRING FORCE : ${METIS_LIB|ENV}/libmetis.so
+opt-set-cmake-var TPL_Netcdf_LIBRARIES STRING FORCE : -L${NETCDF_C_ROOT|ENV}/lib64;${NETCDF_C_ROOT|ENV}/lib/libnetcdf.a;${PARALLEL_NETCDF_ROOT|ENV}/lib/libpnetcdf.a;${TPL_HDF5_LIBRARIES|CMAKE}
+
+#TPL_[INCLUDE|LIBRARY]_DIRS
+opt-set-cmake-var Netcdf_INCLUDE_DIRS STRING FORCE : ${NETCDF_C_INC|ENV}
+opt-set-cmake-var ParMETIS_INCLUDE_DIRS STRING FORCE : ${PARMETIS_INC|ENV}
+opt-set-cmake-var ParMETIS_LIBRARY_DIRS STRING FORCE : ${PARMETIS_LIB|ENV}
+opt-set-cmake-var Scotch_INCLUDE_DIRS STRING FORCE : ${SCOTCH_INC|ENV}
+opt-set-cmake-var Scotch_LIBRARY_DIRS STRING FORCE : ${SCOTCH_LIB|ENV}
+opt-set-cmake-var SuperLU_INCLUDE_DIRS STRING FORCE : ${SUPERLU_INC|ENV}
+opt-set-cmake-var SuperLU_LIBRARY_DIRS STRING FORCE : ${SUPERLU_LIB|ENV}
+
+#CXX Settings
+opt-set-cmake-var CMAKE_CXX_FLAGS STRING : -fPIC -Wall -Warray-bounds -Wchar-subscripts -Wcomment -Wenum-compare -Wformat -Wuninitialized -Wmaybe-uninitialized -Wmain -Wnarrowing -Wnonnull -Wparentheses -Wreorder -Wreturn-type -Wsign-compare -Wsequence-point -Wtrigraphs -Wunused-function -Wunused-but-set-variable -Wunused-variable -Wwrite-strings
+
+#Package Options
+opt-set-cmake-var EpetraExt_ENABLE_HDF5 BOOL FORCE : OFF
+opt-set-cmake-var Kokkos_ENABLE_CUDA BOOL FORCE : ON
+opt-set-cmake-var Kokkos_ENABLE_CUDA_LAMBDA BOOL FORCE : ON
+opt-set-cmake-var Kokkos_ENABLE_CXX11_DISPATCH_LAMBDA BOOL FORCE : ON
+#opt-set-cmake-var Kokkos_ENABLE_Debug_Bounds_Check BOOL FORCE : ON
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS STRING : --bind-to;none
+opt-set-cmake-var Panzer_FADTYPE STRING FORCE : "Sacado::Fad::DFad<RealType>"
+opt-set-cmake-var Phalanx_KOKKOS_DEVICE_TYPE STRING FORCE : CUDA
+opt-set-cmake-var Sacado_ENABLE_HIERARCHICAL_DFAD BOOL FORCE : ON
+opt-set-cmake-var Tpetra_INST_SERIAL BOOL FORCE : ON
+opt-set-cmake-var Zoltan_ENABLE_Scotch BOOL FORCE : OFF
+opt-set-cmake-var Trilinos_ENABLE_TrilinosFrameworkTests BOOL FORCE : OFF
+opt-set-cmake-var Trilinos_ENABLE_TrilinosBuildStats BOOL FORCE : OFF
+
+# Test failures as of 11-28-22
+opt-set-cmake-var PanzerAdaptersSTK_tQuad8ToQuad4Factory_MPI_2_DISABLE BOOL : ON
+opt-set-cmake-var PanzerAdaptersSTK_tQuadraticToLinearMeshFactory_MPI_2_DISABLE BOOL : ON
+opt-set-cmake-var Pliris_vector_random_MPI_3_DISABLE BOOL : ON
+opt-set-cmake-var Pliris_vector_random_MPI_4_DISABLE BOOL : ON
+opt-set-cmake-var ROL_example_PDE-OPT_navier-stokes_example_01_MPI_4_DISABLE BOOL : ON
+opt-set-cmake-var ROL_example_PDE-OPT_nonlinear-elliptic_example_01_MPI_4_DISABLE BOOL : ON
+opt-set-cmake-var ROL_example_PDE-OPT_nonlinear-elliptic_example_02_MPI_4_DISABLE BOOL : ON
+opt-set-cmake-var ROL_example_PDE-OPT_obstacle_example_01_MPI_4_DISABLE BOOL : ON
+opt-set-cmake-var ROL_example_PDE-OPT_topo-opt_poisson_example_01_MPI_4_DISABLE BOOL : ON
+opt-set-cmake-var ROL_test_elementwise_TpetraMultiVector_MPI_4_DISABLE BOOL : ON
+opt-set-cmake-var STKUnit_tests_stk_mesh_unit_tests_MPI_4_DISABLE BOOL : ON
+# This was failing fairly reliably and Nate said it should be okay to disable (https://github.com/trilinos/Trilinos/issues/11678)
+opt-set-cmake-var Kokkos_CoreUnitTest_CudaTimingBased_MPI_1_DISABLE BOOL : ON
+
+# This is temporarily disabled because it seems to be particularly sensitive to the spack-built
+#  MPI issue (TRILFRAME-552)
+opt-set-cmake-var ROL_example_PinT_parabolic-control_AugmentedSystem_test_MPI_2_DISABLE BOOL FORCE : ON
+
+use RHEL7_POST
+
+use CUDA11-RUN-SERIAL-TESTS
+
+[rhel8_sems-cuda-11.4.2-sems-gnu-10.1.0-sems-openmpi-4.1.4_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_no-uvm_deprecated-on_all]
+# uses sems-v2 modules
+use rhel8_sems-cuda-11.4.2-sems-gnu-10.1.0-sems-openmpi-4.1.4_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_no-uvm_deprecated-on_no-package-enables
+use PACKAGE-ENABLES|ALL
+
 [rhel7_ascdo-gnu-10.3.0-serial_debug_shared_no-kokkos-arch_no-asan_no-complex_no-fpic_no-mpi_no-pt_no-rdc_no-uvm_deprecated-on_no-package-enables]
 # uses asc-do modules
 use NODE-TYPE|SERIAL
@@ -2750,4 +2872,4 @@ use USE-UVM|NO
 use USE-DEPRECATED|YES
 
 opt-set-cmake-var Trilinos_ENABLE_Teuchos BOOL : ON
-opt-set-cmake-var Trilinos_ENABLE_ALL_PACKAGES BOOL FORCE : OFF
+opt-set-cmake-var Trilinos_ENABLE_ALL_PACKAGES BOOL FORCE : OFF

packages/framework/ini-files/supported-envs.ini

@@ -173,6 +173,7 @@ gnu
 
 [rhel8]
 oneapi-intelmpi
+sems-cuda-11.4.2-sems-gnu-10.1.0-sems-openmpi-4.1.4
 
 [ats2]
 cuda-11.2.152-gnu-8.3.1-spmpi-rolling

packages/muelu/CMakeLists.txt

@@ -176,16 +176,6 @@ ELSE()
   SET(MUELU_VALGRIND_TESTING NO)
 ENDIF()
 
-# Profiling
-TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_PROFILING
-  HAVE_${PACKAGE_NAME_UC}_PROFILING
-  "Enable detailed timing and memory profiling of MueLu."
-  OFF)
-
-IF (${PACKAGE_NAME}_ENABLE_PROFILING)
-  MESSAGE(STATUS "Enabling MueLu profiling code")
-ENDIF()
-
 # Profiling (expert users only. There is no TPL for Google Perftools)
 TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_GOOGLE_PERFTOOLS
   HAVE_${PACKAGE_NAME_UC}_GOOGLE_PERFTOOLS

packages/muelu/cmake/MueLu_config.hpp.in

@@ -13,8 +13,6 @@
 
 #cmakedefine HAVE_MUELU_EXPLICIT_INSTANTIATION
 
-#cmakedefine HAVE_MUELU_PROFILING
-
 #cmakedefine HAVE_MUELU_GOOGLE_PERFTOOLS
 
 #cmakedefine HAVE_MUELU_EXPERIMENTAL

packages/muelu/src/Utils/MueLu_Memory.cpp

@@ -115,7 +115,6 @@ namespace MueLu {
 #endif
     }
 
-#ifdef HAVE_MUELU_PROFILING
     void ReportTimeAndMemory(Teuchos::Time const &timer, Teuchos::Comm<int> const &Comm)
     {
       double maxTime=0,minTime=0,avgTime=0;
@@ -138,12 +137,6 @@ namespace MueLu {
         std::cout << "&&&" << timer.name() << " " << MemUtils::PrintMemoryUsage() << std::endl;
       }
     } //ReportTimeAndMemory
-#else
-    void ReportTimeAndMemory(Teuchos::Time const &/* timer */, Teuchos::Comm<int> const &/* Comm */)
-    {
-      return;
-    }
-#endif
 
   } //namespace MemUtils
 

packages/muelu/src/Utils/MueLu_Memory.hpp

@@ -47,6 +47,8 @@
 #define MUELU_MEMORY_HPP
 
 #include <string>
+#include "Teuchos_Time.hpp"
+#include "Teuchos_Comm.hpp"
 
 namespace Teuchos { class Time; }
 namespace Teuchos { template <typename Ordinal> class Comm; }

packages/muelu/src/Utils/MueLu_Monitor.cpp

@@ -43,7 +43,7 @@ namespace MueLu {
         levelTimeMonitor_ = rcp(new TimeMonitor(object, object.ShortClassName() + ": " + msg +
             " (total, level=" + Teuchos::Utils::toString(levelID) + ")", timerLevel));
       levelTimeMonitorExclusive_ = rcp(new MutuallyExclusiveTimeMonitor<Level>(object, object.ShortClassName() +
-          MUELU_TIMER_AS_STRING + ": " + msg + " (level=" + Teuchos::Utils::toString(levelID) + ")", timerLevel));
+            ": " + msg + " (level=" + Teuchos::Utils::toString(levelID) + ")", timerLevel));
     }
   }
 
@@ -57,7 +57,7 @@ namespace MueLu {
         levelTimeMonitor_ = rcp(new TimeMonitor(object, label+object.ShortClassName() + ": " +  msg +
             " (total, level=" + Teuchos::Utils::toString(level.GetLevelID()) + ")", timerLevel));
       levelTimeMonitorExclusive_ = rcp(new MutuallyExclusiveTimeMonitor<Level>(object, label+object.ShortClassName() +
-          MUELU_TIMER_AS_STRING + ": " + msg + " (level=" + Teuchos::Utils::toString(level.GetLevelID()) + ")", timerLevel));
+            ": " + msg + " (level=" + Teuchos::Utils::toString(level.GetLevelID()) + ")", timerLevel));
     }
   }
 

packages/muelu/src/Utils/MueLu_Monitor.hpp

@@ -178,13 +178,6 @@ namespace MueLu {
     TimeMonitor timerMonitor_;
   };
 
-//convert integer timer number to string
-#ifdef HAVE_MUELU_PROFILING
-#define MUELU_TIMER_AS_STRING static_cast<std::ostringstream*>( &(std::ostringstream() << " " << timerIdentifier_++) )->str()
-#else
-#define MUELU_TIMER_AS_STRING
-#endif
-
   //---------------------------------------------------------------------------------------------------
 
   /*! @class FactoryMonitor

packages/phalanx/src/Phalanx_Kokkos_Tools_CheckStreams.cpp

@@ -0,0 +1,66 @@
+#include "Phalanx_Kokkos_Tools_CheckStreams.hpp"
+#include "Phalanx_KokkosDeviceTypes.hpp"
+#include "Kokkos_Core.hpp"
+#include "Teuchos_Assert.hpp"
+#include <limits>
+
+// **********************************
+// Ideally, we would like to also check allocations, deallocations,
+// deep_copy, create_mirror, create_mirror_view, and
+// create_mirror_view_and_copy as well. The kokkos tools will need to
+// be modified to pass in the device id for these functions. For now
+// we can only check parallel_* and fencing.
+// **********************************
+
+// Lambdas can only be converted to function pointers if they do not capture.
+// Using a global non-static variable in an unnamed namespace to "capture" the
+// device id.
+#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP)
+namespace {
+  uint32_t phalanx_default_stream_device_id = std::numeric_limits<uint32_t>::max();
+
+  void phalanx_kt_parallel_x_callback(char const *label, uint32_t device_id,
+                                      uint64_t * /*kernel_id*/)
+  {
+    TEUCHOS_TEST_FOR_EXCEPTION(device_id == phalanx_default_stream_device_id,
+                               std::runtime_error,
+                               "\"ERROR: the kernel \"" << label
+                               << "\" with device id=" << device_id
+                               << " is the same as the default stream id="
+                             << phalanx_default_stream_device_id);
+  }
+
+  void phalanx_kt_fence_callback(char const *label, uint32_t device_id,
+                                 uint64_t * /*fence_id*/)
+  {
+    TEUCHOS_TEST_FOR_EXCEPTION(device_id == phalanx_default_stream_device_id,
+                               std::runtime_error,
+                               "\"ERROR: the fence \"" << label
+                               << "\" with device id=" << device_id
+                               << " is the same as the default stream id="
+                               << phalanx_default_stream_device_id);
+  }
+}
+#endif
+
+void PHX::set_enforce_no_default_stream_use()
+{
+#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP)
+  phalanx_default_stream_device_id = Kokkos::Tools::Experimental::device_id(PHX::Device());
+
+  Kokkos::Tools::Experimental::set_begin_parallel_for_callback(phalanx_kt_parallel_x_callback);
+  Kokkos::Tools::Experimental::set_begin_parallel_reduce_callback(phalanx_kt_parallel_x_callback);
+  Kokkos::Tools::Experimental::set_begin_parallel_scan_callback(phalanx_kt_parallel_x_callback);
+  Kokkos::Tools::Experimental::set_begin_fence_callback(phalanx_kt_fence_callback);
+#endif
+}
+
+void PHX::unset_enforce_no_default_stream_use()
+{
+#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP)
+  Kokkos::Tools::Experimental::set_begin_parallel_for_callback(nullptr);
+  Kokkos::Tools::Experimental::set_begin_parallel_reduce_callback(nullptr);
+  Kokkos::Tools::Experimental::set_begin_parallel_scan_callback(nullptr);
+  Kokkos::Tools::Experimental::set_begin_fence_callback(nullptr);
+#endif
+}

packages/phalanx/src/Phalanx_Kokkos_Tools_CheckStreams.hpp

@@ -0,0 +1,15 @@
+#ifndef PHALANX_KOKKOS_TOOLS_VERIFYSTREAM_HPP
+#define PHALANX_KOKKOS_TOOLS_VERIFYSTREAM_HPP
+
+namespace PHX {
+  /// Function that sets kokkos-tools callbacks to make sure that the
+  /// default stream is not being using in any kokkos calls. Checks
+  /// are only active for cuda and hip backends as the id for the
+  /// serial backend is always the same.
+  void set_enforce_no_default_stream_use();
+
+  /// Function that unsets kokkos-tools callbacks.
+  void unset_enforce_no_default_stream_use();
+}
+
+#endif