Delay reporting of blockchain not in sync to avoid false monitoring alerts

Author: martinboehm

api/worker.go

@@ -1625,12 +1625,12 @@ func (w *Worker) GetBalanceHistory(address string, fromTimestamp, toTimestamp in
 
 func (w *Worker) waitForBackendSync() {
 	// wait a short time if blockbook is synchronizing with backend
-	inSync, _, _ := w.is.GetSyncState()
+	inSync, _, _, _ := w.is.GetSyncState()
 	count := 30
 	for !inSync && count > 0 {
 		time.Sleep(time.Millisecond * 100)
 		count--
-		inSync, _, _ = w.is.GetSyncState()
+		inSync, _, _, _ = w.is.GetSyncState()
 	}
 }
 
@@ -2262,9 +2262,15 @@ func nonZeroTime(t time.Time) *time.Time {
 
 // GetSystemInfo returns information about system
 func (w *Worker) GetSystemInfo(internal bool) (*SystemInfo, error) {
-	start := time.Now()
+	start := time.Now().UTC()
 	vi := common.GetVersionInfo()
-	inSync, bestHeight, lastBlockTime := w.is.GetSyncState()
+	inSync, bestHeight, lastBlockTime, startSync := w.is.GetSyncState()
+	if !inSync && !w.is.InitialSync {
+		// if less than 5 seconds into syncing, return inSync=true to avoid short time not in sync reports that confuse monitoring
+		if startSync.Add(5 * time.Second).After(start) {
+			inSync = true
+		}
+	}
 	inSyncMempool, lastMempoolTime, mempoolSize := w.is.GetMempoolSyncState()
 	ci, err := w.chain.GetChainInfo()
 	var backendError string

common/internalstate.go

@@ -69,6 +69,7 @@ type InternalState struct {
 	InitialSync    bool      `json:"initialSync"`
 	IsSynchronized bool      `json:"isSynchronized"`
 	BestHeight     uint32    `json:"bestHeight"`
+	StartSync      time.Time `json:"-"`
 	LastSync       time.Time `json:"lastSync"`
 	BlockTimes     []uint32  `json:"-"`
 	AvgBlockPeriod uint32    `json:"-"`
@@ -96,6 +97,7 @@ type InternalState struct {
 func (is *InternalState) StartedSync() {
 	is.mux.Lock()
 	defer is.mux.Unlock()
+	is.StartSync = time.Now().UTC()
 	is.IsSynchronized = false
 }
 
@@ -105,15 +107,15 @@ func (is *InternalState) FinishedSync(bestHeight uint32) {
 	defer is.mux.Unlock()
 	is.IsSynchronized = true
 	is.BestHeight = bestHeight
-	is.LastSync = time.Now()
+	is.LastSync = time.Now().UTC()
 }
 
 // UpdateBestHeight sets new best height, without changing IsSynchronized flag
 func (is *InternalState) UpdateBestHeight(bestHeight uint32) {
 	is.mux.Lock()
 	defer is.mux.Unlock()
 	is.BestHeight = bestHeight
-	is.LastSync = time.Now()
+	is.LastSync = time.Now().UTC()
 }
 
 // FinishedSyncNoChange marks end of synchronization in case no index update was necessary, it does not update lastSync time
@@ -124,10 +126,10 @@ func (is *InternalState) FinishedSyncNoChange() {
 }
 
 // GetSyncState gets the state of synchronization
-func (is *InternalState) GetSyncState() (bool, uint32, time.Time) {
+func (is *InternalState) GetSyncState() (bool, uint32, time.Time, time.Time) {
 	is.mux.Lock()
 	defer is.mux.Unlock()
-	return is.IsSynchronized, is.BestHeight, is.LastSync
+	return is.IsSynchronized, is.BestHeight, is.LastSync, is.StartSync
 }
 
 // StartedMempoolSync signals start of mempool synchronization

db/txcache.go

@@ -46,7 +46,7 @@ func (c *TxCache) GetTransaction(txid string) (*bchain.Tx, int, error) {
 		}
 		if tx != nil {
 			// number of confirmations is not stored in cache, they change all the time
-			_, bestheight, _ := c.is.GetSyncState()
+			_, bestheight, _, _ := c.is.GetSyncState()
 			tx.Confirmations = bestheight - h + 1
 			c.metrics.TxCacheEfficiency.With(common.Labels{"status": "hit"}).Inc()
 			return tx, int(h), nil

fiat/fiat_rates.go

@@ -135,7 +135,7 @@ func (rd *RatesDownloader) Run() error {
 						} else {
 							glog.Info("FiatRatesDownloader: UpdateHistoricalTokenTickers finished")
 							if is != nil {
-								is.HistoricalTokenFiatRatesTime = time.Now()
+								is.HistoricalTokenFiatRatesTime = time.Now().UTC()
 							}
 						}
 					}()

server/socketio.go

@@ -584,7 +584,7 @@ type resultGetInfo struct {
 }
 
 func (s *SocketIoServer) getInfo() (res resultGetInfo, err error) {
-	_, height, _ := s.is.GetSyncState()
+	_, height, _, _ := s.is.GetSyncState()
 	res.Result.Blocks = int(height)
 	res.Result.Testnet = s.chain.IsTestnet()
 	res.Result.Network = s.chain.GetNetworkName()