cluster: fix tso fallback due raft cluster did not stop tso service (#…

…8837) (#8838)

close #8836

Signed-off-by: lhy1024 <admin@liudos.us>

Co-authored-by: lhy1024 <admin@liudos.us>

server/cluster/cluster.go

@@ -491,6 +491,8 @@ func (c *RaftCluster) startTSOJobsIfNeeded() error {
 			log.Error("failed to initialize the global TSO allocator", errs.ZapError(err))
 			return err
 		}
+	} else {
+		log.Warn("the global TSO allocator is already initialized")
 	}
 	return nil
 }
@@ -855,6 +857,15 @@ func (c *RaftCluster) runReplicationMode() {
 // Stop stops the cluster.
 func (c *RaftCluster) Stop() {
 	c.Lock()
+	// We need to try to stop tso jobs whatever the cluster is running or not.
+	// Because we need to call checkTSOService as soon as possible while the cluster is starting,
+	// which makes the cluster may not be running but the tso job has been started.
+	// For example, the cluster meets an error when starting, such as cluster is not bootstrapped.
+	// In this case, the `running` in `RaftCluster` is false, but the tso job has been started.
+	// Ref: https://github.com/tikv/pd/issues/8836
+	if err := c.stopTSOJobsIfNeeded(); err != nil {
+		log.Error("failed to stop tso jobs", errs.ZapError(err))
+	}
 	if !c.running {
 		c.Unlock()
 		return
@@ -864,9 +875,6 @@ func (c *RaftCluster) Stop() {
 	if !c.IsServiceIndependent(constant.SchedulingServiceName) {
 		c.stopSchedulingJobs()
 	}
-	if err := c.stopTSOJobsIfNeeded(); err != nil {
-		log.Error("failed to stop tso jobs", errs.ZapError(err))
-	}
 	c.heartbeatRunner.Stop()
 	c.miscRunner.Stop()
 	c.logRunner.Stop()

tests/integrations/client/client_test.go

@@ -197,11 +197,11 @@ func TestLeaderTransferAndMoveCluster(t *testing.T) {
 	oldServers := cluster.GetServers()
 	oldLeaderName := cluster.WaitLeader()
 	for range 3 {
+		time.Sleep(5 * time.Second)
 		newPD, err := cluster.Join(ctx)
 		re.NoError(err)
 		re.NoError(newPD.Run())
 		oldLeaderName = cluster.WaitLeader()
-		time.Sleep(5 * time.Second)
 	}
 
 	// ABCDEF->DEF