From e271e22f7c1fae676e2fce6ab002b358694d9450 Mon Sep 17 00:00:00 2001
From: Christian Haudum <christian.haudum@gmail.com>
Date: Wed, 29 Jan 2025 09:53:27 +0100
Subject: [PATCH] fix(blooms): Initialize bloom gateway client only once
 (#15994)

The bloom gateway client is used both in the bloom builder and in the index gateways. When running Loki in SSD mode, both services are part of the backend target, and therefore the client is initialised twice, leading to duplicate metrics registration and a subsequent panic.

This commit extracts the initialisation of the bloom gateway client into a separate service that is started once and becomes a dependency of both bloom builder and index gateway.

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>
---
 pkg/loki/loki.go    |  4 +++-
 pkg/loki/modules.go | 30 +++++++++++++++---------------
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go
index 994afa94ccb99..d8c9be61f52f5 100644
--- a/pkg/loki/loki.go
+++ b/pkg/loki/loki.go
@@ -370,6 +370,7 @@ type Loki struct {
 	ingesterQuerier           *querier.IngesterQuerier
 	Store                     storage.Store
 	BloomStore                bloomshipper.Store
+	bloomGatewayClient        bloomgateway.Client
 	tableManager              *index.TableManager
 	frontend                  Frontend
 	ruler                     *base_ruler.Ruler
@@ -694,6 +695,7 @@ func (t *Loki) setupModuleManager() error {
 	mm.RegisterModule(IndexGatewayRing, t.initIndexGatewayRing, modules.UserInvisibleModule)
 	mm.RegisterModule(IndexGatewayInterceptors, t.initIndexGatewayInterceptors, modules.UserInvisibleModule)
 	mm.RegisterModule(BloomGateway, t.initBloomGateway)
+	mm.RegisterModule(BloomGatewayClient, t.initBloomGatewayClient)
 	mm.RegisterModule(QueryScheduler, t.initQueryScheduler)
 	mm.RegisterModule(QuerySchedulerRing, t.initQuerySchedulerRing, modules.UserInvisibleModule)
 	mm.RegisterModule(Analytics, t.initAnalytics, modules.UserInvisibleModule)
@@ -732,7 +734,7 @@ func (t *Loki) setupModuleManager() error {
 		BloomGateway:             {Server, BloomStore, Analytics},
 		BloomPlanner:             {Server, BloomStore, Analytics, Store},
 		BloomBuilder:             {Server, BloomStore, Analytics, Store},
-		BloomStore:               {IndexGatewayRing},
+		BloomStore:               {IndexGatewayRing, BloomGatewayClient},
 		PatternRingClient:        {Server, MemberlistKV, Analytics},
 		PatternIngesterTee:       {Server, MemberlistKV, Analytics, PatternRingClient},
 		PatternIngester:          {Server, MemberlistKV, Analytics, PatternRingClient, PatternIngesterTee, Overrides},
diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go
index 58d3e77a74c4b..5dd548f5d7b28 100644
--- a/pkg/loki/modules.go
+++ b/pkg/loki/modules.go
@@ -128,6 +128,7 @@ const (
 	IndexGatewayInterceptors = "index-gateway-interceptors"
 	BloomStore               = "bloom-store"
 	BloomGateway             = "bloom-gateway"
+	BloomGatewayClient       = "bloom-gateway-client"
 	BloomPlanner             = "bloom-planner"
 	BloomBuilder             = "bloom-builder"
 	QueryScheduler           = "query-scheduler"
@@ -1549,16 +1550,12 @@ func (t *Loki) initIndexGateway() (services.Service, error) {
 
 	var bloomQuerier indexgateway.BloomQuerier
 	if t.Cfg.BloomGateway.Enabled {
-		bloomGatewayClient, err := bloomgateway.NewClient(t.Cfg.BloomGateway.Client, prometheus.DefaultRegisterer, logger)
-		if err != nil {
-			return nil, err
-		}
 		resolver := bloomgateway.NewBlockResolver(t.BloomStore, logger)
 		querierCfg := bloomgateway.QuerierConfig{
 			BuildTableOffset: t.Cfg.BloomBuild.Planner.MinTableOffset,
 			BuildInterval:    t.Cfg.BloomBuild.Planner.PlanningInterval,
 		}
-		bloomQuerier = bloomgateway.NewQuerier(bloomGatewayClient, querierCfg, t.Overrides, resolver, prometheus.DefaultRegisterer, logger)
+		bloomQuerier = bloomgateway.NewQuerier(t.bloomGatewayClient, querierCfg, t.Overrides, resolver, prometheus.DefaultRegisterer, logger)
 	}
 
 	gateway, err := indexgateway.NewIndexGateway(t.Cfg.IndexGateway, t.Overrides, logger, prometheus.DefaultRegisterer, t.Store, indexClients, bloomQuerier)
@@ -1653,6 +1650,18 @@ func (t *Loki) initBloomPlanner() (services.Service, error) {
 	return p, nil
 }
 
+func (t *Loki) initBloomGatewayClient() (services.Service, error) {
+	var err error
+	if t.Cfg.BloomGateway.Enabled {
+		logger := log.With(util_log.Logger, "component", "bloom-gateway-client")
+		t.bloomGatewayClient, err = bloomgateway.NewClient(t.Cfg.BloomGateway.Client, prometheus.DefaultRegisterer, logger)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return nil, nil
+}
+
 func (t *Loki) initBloomBuilder() (services.Service, error) {
 	if !t.Cfg.BloomBuild.Enabled {
 		return nil, nil
@@ -1670,15 +1679,6 @@ func (t *Loki) initBloomBuilder() (services.Service, error) {
 		ringManager = t.indexGatewayRingManager
 	}
 
-	var bloomGatewayClient bloomgateway.Client
-	if t.Cfg.BloomGateway.Enabled {
-		var err error
-		bloomGatewayClient, err = bloomgateway.NewClient(t.Cfg.BloomGateway.Client, prometheus.DefaultRegisterer, logger)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	return builder.New(
 		t.Cfg.BloomBuild.Builder,
 		t.Overrides,
@@ -1687,7 +1687,7 @@ func (t *Loki) initBloomBuilder() (services.Service, error) {
 		t.ClientMetrics,
 		t.Store,
 		t.BloomStore,
-		bloomGatewayClient,
+		t.bloomGatewayClient,
 		logger,
 		prometheus.DefaultRegisterer,
 		ringManager,