Minor changes

JanVogelsang · Dec 12, 2024 · c7228bd · c7228bd
1 parent 68549ac
commit c7228bd
Show file tree

Hide file tree

Showing 18 changed files with 164 additions and 207 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -68,6 +68,7 @@ set( with-models OFF CACHE STRING "The models to include as a semicolon-separate
 set( tics_per_ms "1000.0" CACHE STRING "Specify elementary unit of time [default=1000 tics per ms]." )
 set( tics_per_step "100" CACHE STRING "Specify resolution [default=100 tics per step]." )
 set( with-detailed-timers OFF CACHE STRING "Build with detailed internal time measurements [default=OFF]. Detailed timers can affect the performance." )
+set( with-mpi-sync-timer OFF CACHE STRING "Build with mpi synchronization barrier and timer [default=OFF]. Can affect the performance." )
 set( with-threaded-timers ON CACHE STRING "Build with one internal timer per thread [default=ON]. Multi-threaded timers can affect the performance." )
 set( target-bits-split "standard" CACHE STRING "Split of the 64-bit target neuron identifier type [default='standard']. 'standard' is recommended for most users. If running on more than 262144 MPI processes or more than 512 threads, change to 'hpc'." )
 
@@ -145,6 +146,7 @@ nest_process_with_openmp()
 nest_process_with_mpi()
 nest_process_with_detailed_timers()
 nest_process_with_threaded_timers()
+nest_process_with_mpi_sync_timer()
 nest_process_with_libneurosim()
 nest_process_with_music()
 nest_process_with_sionlib()

diff --git a/cmake/ProcessOptions.cmake b/cmake/ProcessOptions.cmake
@@ -469,6 +469,13 @@ function( NEST_PROCESS_WITH_THREADED_TIMERS )
   endif ()
 endfunction()
 
+function( NEST_PROCESS_WITH_MPI_SYNC_TIMER )
+  set( MPI_SYNC_TIMER OFF PARENT_SCOPE )
+  if ( ${with-mpi-sync-timer} STREQUAL "ON" )
+    set( MPI_SYNC_TIMER ON PARENT_SCOPE )
+  endif ()
+endfunction()
+
 function( NEST_PROCESS_WITH_LIBNEUROSIM )
   # Find libneurosim
   set( HAVE_LIBNEUROSIM OFF PARENT_SCOPE )

diff --git a/doc/htmldoc/developer_space/guidelines/coding_guidelines_cpp.rst b/doc/htmldoc/developer_space/guidelines/coding_guidelines_cpp.rst
@@ -556,7 +556,7 @@ For example, the ``stopwatch.h`` file could look like:
     }
 
     inline nest::Stopwatch::timestamp_t
-    nest::Stopwatch::elapsed_timestamp() const
+    nest::Stopwatch::elapsed_us() const
     {
     #ifndef DISABLE_TIMING
       if ( isRunning() )
@@ -622,7 +622,7 @@ For example, the ``stopwatch.h`` file could look like:
     }
 
     inline nest::Stopwatch::timestamp_t
-    nest::Stopwatch::get_timestamp()
+    nest::Stopwatch::get_current_time()
     {
       // works with:
       // * hambach (Linux 2.6.32 x86_64)

diff --git a/doc/htmldoc/installation/cmake_options.rst b/doc/htmldoc/installation/cmake_options.rst
@@ -202,6 +202,9 @@ NEST properties
 +-----------------------------------------------+----------------------------------------------------------------+
 | ``-Dwith-detailed-timers=[OFF|ON]``           | Build with detailed internal time measurements [default=OFF].  |
 |                                               | Detailed timers can affect the performance.                    |
++----------------------------------------------------------------------------------------------------------------+
+| ``-Dwith-mpi-sync-timer=[OFF|ON]``            | Build with mpi synchronization barrier and timer [default=OFF].|
+|                                               | Can affect the performance.                                    |
 +-----------------------------------------------+----------------------------------------------------------------+
 | ``-Dtarget-bits-split=['standard'|'hpc']``    | Split of the 64-bit target neuron identifier type              |
 |                                               | [default='standard']. 'standard' is recommended for most users.|

diff --git a/doc/htmldoc/nest_behavior/built-in_timers.rst b/doc/htmldoc/nest_behavior/built-in_timers.rst
@@ -107,3 +107,7 @@ available as kernel attributes:
 |                                |buffers of the corresponding      |                                  |
 |                                |postsynaptic neurons              |                                  |
 +--------------------------------+----------------------------------+----------------------------------+
+
+
+with-mpi-sync-timer
+with-threaded-timers
diff --git a/nestkernel/connection_manager.cpp b/nestkernel/connection_manager.cpp
@@ -1801,9 +1801,9 @@ nest::ConnectionManager::collect_compressed_spike_data( const size_t tid )
     } // of omp single; implicit barrier
 
     source_table_.collect_compressible_sources( tid );
-    kernel().get_omp_synchronization_stopwatch().start();
+    kernel().get_omp_synchronization_construction_stopwatch().start();
 #pragma omp barrier
-    kernel().get_omp_synchronization_stopwatch().stop();
+    kernel().get_omp_synchronization_construction_stopwatch().stop();
 #pragma omp single
     {
       source_table_.fill_compressed_spike_data( compressed_spike_data_ );

diff --git a/nestkernel/event_delivery_manager.cpp b/nestkernel/event_delivery_manager.cpp
@@ -416,12 +416,12 @@ EventDeliveryManager::gather_spike_data_( std::vector< SpikeDataT >& send_buffer
     set_end_marker_( send_buffer_position, send_buffer, local_max_spikes_per_rank );
 
     sw_collocate_spike_data_.stop();
-#if defined( HAVE_MPI ) && defined( TIMER_DETAILED )
+    sw_communicate_spike_data_.start();
+#ifdef MPI_SYNC_TIMER
     kernel().get_mpi_synchronization_stopwatch().start();
     kernel().mpi_manager.synchronize();
     kernel().get_mpi_synchronization_stopwatch().stop();
 #endif
-    sw_communicate_spike_data_.start();
 
     // Given that we templatize by plain vs offgrid, this if should not be necessary, but ...
     if ( off_grid_spiking_ )
@@ -797,9 +797,9 @@ EventDeliveryManager::gather_target_data( const size_t tid )
         resize_send_recv_buffers_target_data();
       }
     } // of omp master; (no barrier)
-    kernel().get_omp_synchronization_stopwatch().start();
+    kernel().get_omp_synchronization_construction_stopwatch().start();
 #pragma omp barrier
-    kernel().get_omp_synchronization_stopwatch().stop();
+    kernel().get_omp_synchronization_construction_stopwatch().stop();
 
     kernel().connection_manager.restore_source_table_entry_point( tid );
 
@@ -814,18 +814,13 @@ EventDeliveryManager::gather_target_data( const size_t tid )
       set_complete_marker_target_data_( assigned_ranks, send_buffer_position );
     }
     kernel().connection_manager.save_source_table_entry_point( tid );
-    kernel().get_omp_synchronization_stopwatch().start();
+    kernel().get_omp_synchronization_construction_stopwatch().start();
 #pragma omp barrier
-    kernel().get_omp_synchronization_stopwatch().stop();
+    kernel().get_omp_synchronization_construction_stopwatch().stop();
     kernel().connection_manager.clean_source_table( tid );
 
 #pragma omp master
     {
-#if defined( HAVE_MPI ) && defined( TIMER_DETAILED )
-      kernel().get_mpi_synchronization_stopwatch().start();
-      kernel().mpi_manager.synchronize();
-      kernel().get_mpi_synchronization_stopwatch().stop();
-#endif
       sw_communicate_target_data_.start();
       kernel().mpi_manager.communicate_target_data_Alltoall( send_buffer_target_data_, recv_buffer_target_data_ );
       sw_communicate_target_data_.stop();
@@ -874,9 +869,9 @@ EventDeliveryManager::gather_target_data_compressed( const size_t tid )
         resize_send_recv_buffers_target_data();
       }
     } // of omp master; no barrier
-    kernel().get_omp_synchronization_stopwatch().start();
+    kernel().get_omp_synchronization_construction_stopwatch().start();
 #pragma omp barrier
-    kernel().get_omp_synchronization_stopwatch().stop();
+    kernel().get_omp_synchronization_construction_stopwatch().stop();
 
     TargetSendBufferPosition send_buffer_position(
       assigned_ranks, kernel().mpi_manager.get_send_recv_count_target_data_per_rank() );
@@ -891,17 +886,12 @@ EventDeliveryManager::gather_target_data_compressed( const size_t tid )
       set_complete_marker_target_data_( assigned_ranks, send_buffer_position );
     }
 
-    kernel().get_omp_synchronization_stopwatch().start();
+    kernel().get_omp_synchronization_construction_stopwatch().start();
 #pragma omp barrier
-    kernel().get_omp_synchronization_stopwatch().stop();
+    kernel().get_omp_synchronization_construction_stopwatch().stop();
 
 #pragma omp master
     {
-#if defined( HAVE_MPI ) && defined( TIMER_DETAILED )
-      kernel().get_mpi_synchronization_stopwatch().start();
-      kernel().mpi_manager.synchronize();
-      kernel().get_mpi_synchronization_stopwatch().stop();
-#endif
       sw_communicate_target_data_.start();
       kernel().mpi_manager.communicate_target_data_Alltoall( send_buffer_target_data_, recv_buffer_target_data_ );
       sw_communicate_target_data_.stop();
@@ -921,9 +911,9 @@ EventDeliveryManager::gather_target_data_compressed( const size_t tid )
       {
         buffer_size_target_data_has_changed_ = kernel().mpi_manager.increase_buffer_size_target_data();
       } // of omp master (no barrier)
-      kernel().get_omp_synchronization_stopwatch().start();
+      kernel().get_omp_synchronization_construction_stopwatch().start();
 #pragma omp barrier
-      kernel().get_omp_synchronization_stopwatch().stop();
+      kernel().get_omp_synchronization_construction_stopwatch().stop();
     }
 
   } // of while

diff --git a/nestkernel/event_delivery_manager.h b/nestkernel/event_delivery_manager.h
@@ -468,7 +468,6 @@ class EventDeliveryManager : public ManagerInterface
 
   // private stop watches for benchmarking purposes
   // (intended for internal core developers, not for use in the public API)
-  // TODO JV: Make sure DETAILED_TIMERS is only ever used in stopwatch.h
   Stopwatch< StopwatchVerbosity::Detailed, StopwatchType::MasterOnly > sw_collocate_spike_data_;
   Stopwatch< StopwatchVerbosity::Detailed, StopwatchType::MasterOnly > sw_communicate_spike_data_;
   Stopwatch< StopwatchVerbosity::Detailed, StopwatchType::MasterOnly > sw_communicate_target_data_;

diff --git a/nestkernel/kernel_manager.cpp b/nestkernel/kernel_manager.cpp
@@ -93,7 +93,8 @@ nest::KernelManager::initialize()
     manager->initialize( /* adjust_number_of_threads_or_rng_only */ false );
   }
 
-  sw_omp_synchronization_.reset();
+  sw_omp_synchronization_construction_.reset();
+  sw_omp_synchronization_simulation_.reset();
   sw_mpi_synchronization_.reset();
 
   ++fingerprint_;
@@ -110,7 +111,8 @@ nest::KernelManager::prepare()
     manager->prepare();
   }
 
-  sw_omp_synchronization_.reset();
+  sw_omp_synchronization_construction_.reset();
+  sw_omp_synchronization_simulation_.reset();
   sw_mpi_synchronization_.reset();
 }
 
@@ -177,7 +179,8 @@ nest::KernelManager::change_number_of_threads( size_t new_num_threads )
   kernel().event_delivery_manager.reset_timers_for_preparation();
   kernel().event_delivery_manager.reset_timers_for_dynamics();
 
-  sw_omp_synchronization_.reset();
+  sw_omp_synchronization_construction_.reset();
+  sw_omp_synchronization_simulation_.reset();
   sw_mpi_synchronization_.reset();
 }
 
@@ -202,7 +205,8 @@ nest::KernelManager::get_status( DictionaryDatum& dict )
     manager->get_status( dict );
   }
 
-  sw_omp_synchronization_.output_timer( dict, names::time_omp_synchronization );
+  sw_omp_synchronization_construction_.output_timer( dict, names::time_omp_synchronization_construction );
+  sw_omp_synchronization_simulation_.output_timer( dict, names::time_omp_synchronization_simulation );
   sw_mpi_synchronization_.output_timer( dict, names::time_mpi_synchronization );
 }
 

diff --git a/nestkernel/kernel_manager.h b/nestkernel/kernel_manager.h
@@ -288,10 +288,18 @@ class KernelManager
   NodeManager node_manager;
   /**@}*/
 
+  //! Get the stopwatch to measure the time each thread is idle during network construction.
   Stopwatch< StopwatchVerbosity::Detailed, StopwatchType::Threaded >&
-  get_omp_synchronization_stopwatch()
+  get_omp_synchronization_construction_stopwatch()
   {
-    return sw_omp_synchronization_;
+    return sw_omp_synchronization_construction_;
+  }
+
+  //! Get the stopwatch to measure the time each thread is idle during simulation.
+  Stopwatch< StopwatchVerbosity::Detailed, StopwatchType::Threaded >&
+  get_omp_synchronization_simulation_stopwatch()
+  {
+    return sw_omp_synchronization_simulation_;
   }
 
   Stopwatch< StopwatchVerbosity::Detailed, StopwatchType::MasterOnly >&
@@ -307,7 +315,8 @@ class KernelManager
   bool initialized_;   //!< true if the kernel is initialized
   std::ofstream dump_; //!< for FULL_LOGGING output
 
-  Stopwatch< StopwatchVerbosity::Detailed, StopwatchType::Threaded > sw_omp_synchronization_;
+  Stopwatch< StopwatchVerbosity::Detailed, StopwatchType::Threaded > sw_omp_synchronization_construction_;
+  Stopwatch< StopwatchVerbosity::Detailed, StopwatchType::Threaded > sw_omp_synchronization_simulation_;
   Stopwatch< StopwatchVerbosity::Detailed, StopwatchType::MasterOnly > sw_mpi_synchronization_;
 };
 

diff --git a/nestkernel/nest_names.cpp b/nestkernel/nest_names.cpp
@@ -606,7 +606,8 @@ const Name time_deliver_spike_data( "time_deliver_spike_data" );
 const Name time_gather_secondary_data( "time_gather_secondary_data" );
 const Name time_gather_spike_data( "time_gather_spike_data" );
 const Name time_gather_target_data( "time_gather_target_data" );
-const Name time_omp_synchronization( "time_omp_synchronization" );
+const Name time_omp_synchronization_construction( "time_omp_synchronization_construction" );
+const Name time_omp_synchronization_simulation( "time_omp_synchronization_simulation" );
 const Name time_mpi_synchronization( "time_mpi_synchronization" );
 const Name time_in_steps( "time_in_steps" );
 const Name time_simulate( "time_simulate" );

diff --git a/nestkernel/nest_names.h b/nestkernel/nest_names.h
@@ -634,7 +634,8 @@ extern const Name time_deliver_spike_data;
 extern const Name time_gather_secondary_data;
 extern const Name time_gather_spike_data;
 extern const Name time_gather_target_data;
-extern const Name time_omp_synchronization;
+extern const Name time_omp_synchronization_construction;
+extern const Name time_omp_synchronization_simulation;
 extern const Name time_mpi_synchronization;
 extern const Name time_in_steps;
 extern const Name time_simulate;

diff --git a/nestkernel/per_thread_bool_indicator.cpp b/nestkernel/per_thread_bool_indicator.cpp
@@ -65,7 +65,7 @@ PerThreadBoolIndicator::initialize( const size_t num_threads, const bool status
 bool
 PerThreadBoolIndicator::all_false() const
 {
-  kernel().get_omp_synchronization_stopwatch().start();
+  kernel().get_omp_synchronization_construction_stopwatch().start();
 // We need two barriers here to ensure that no thread can continue and change the result
 // before all threads have determined the result.
 #pragma omp barrier
@@ -74,42 +74,42 @@ PerThreadBoolIndicator::all_false() const
   bool ret = ( are_true_ == 0 );
 #pragma omp barrier
 
-  kernel().get_omp_synchronization_stopwatch().stop();
+  kernel().get_omp_synchronization_construction_stopwatch().stop();
   return ret;
 }
 
 bool
 PerThreadBoolIndicator::all_true() const
 {
-  kernel().get_omp_synchronization_stopwatch().start();
+  kernel().get_omp_synchronization_construction_stopwatch().start();
 #pragma omp barrier
   bool ret = ( are_true_ == size_ );
 #pragma omp barrier
-  kernel().get_omp_synchronization_stopwatch().stop();
+  kernel().get_omp_synchronization_construction_stopwatch().stop();
   return ret;
 }
 
 bool
 PerThreadBoolIndicator::any_false() const
 {
-  kernel().get_omp_synchronization_stopwatch().start();
+  kernel().get_omp_synchronization_construction_stopwatch().start();
 #pragma omp barrier
   bool ret = ( are_true_ < size_ );
 #pragma omp barrier
 
-  kernel().get_omp_synchronization_stopwatch().stop();
+  kernel().get_omp_synchronization_construction_stopwatch().stop();
   return ret;
 }
 
 bool
 PerThreadBoolIndicator::any_true() const
 {
-  kernel().get_omp_synchronization_stopwatch().start();
+  kernel().get_omp_synchronization_construction_stopwatch().start();
 #pragma omp barrier
   bool ret = ( are_true_ > 0 );
 #pragma omp barrier
 
-  kernel().get_omp_synchronization_stopwatch().stop();
+  kernel().get_omp_synchronization_construction_stopwatch().stop();
   return ret;
 }