Merge branch 'main' into fix-action-names-in-main

.gitignore

@@ -27,8 +27,8 @@ cmd/querytee/querytee
 dlv
 rootfs/
 dist
-coverage.txt
-test_results.txt
+*coverage.txt
+*test_results.txt
 .DS_Store
 .aws-sam
 .idea

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 ##### Enhancements
 
+* [11814](https://github.com/grafana/loki/pull/11814) **kavirajk**: feat: Support split align and caching for instant metric query results
 * [11851](https://github.com/grafana/loki/pull/11851) **elcomtik**: Helm: Allow the definition of resources for GrafanaAgent pods.
 * [11819](https://github.com/grafana/loki/pull/11819) **jburnham**: Ruler: Add the ability to disable the `X-Scope-OrgId` tenant identification header in remote write requests.
 * [11633](https://github.com/grafana/loki/pull/11633) **cyriltovena**: Add profiling integrations to tracing instrumentation.
@@ -57,6 +58,7 @@
 * [11499](https://github.com/grafana/loki/pull/11284) **jmichalek132** Config: Adds `frontend.log-query-request-headers` to enable logging of request headers in query logs.
 * [11817](https://github.com/grafana/loki/pull/11817) **ashwanthgoli** Ruler: Add support for filtering results of `/prometheus/api/v1/rules` endpoint by rule_name, rule_group, file and type.
 * [11897](https://github.com/grafana/loki/pull/11897) **ashwanthgoli** Metadata: Introduces a separate split interval of `split_recent_metadata_queries_by_interval` for `recent_metadata_query_window` to help with caching recent metadata query results.
+* [11970](https://github.com/grafana/loki/pull/11897) **masslessparticle** Ksonnet: Introduces memory limits to the compactor configuration to avoid unbounded memory usage.
 
 ##### Fixes
 * [11074](https://github.com/grafana/loki/pull/11074) **hainenber** Fix panic in lambda-promtail due to mishandling of empty DROP_LABELS env var.
@@ -69,7 +71,7 @@
 * [11657](https://github.com/grafana/loki/pull/11657) **ashwanthgoli** Log results cache: compose empty response based on the request being served to avoid returning incorrect limit or direction.
 * [11587](https://github.com/grafana/loki/pull/11587) **trevorwhitney** Fix semantics of label parsing logic of metrics and logs queries. Both only parse the first label if multiple extractions into the same label are requested.
 * [11776](https://github.com/grafana/loki/pull/11776) **ashwanthgoli** Background Cache: Fixes a bug that is causing the background queue size to be incremented twice for each enqueued item.
-* [11921](https://github.com/grafana/loki/pull/11921) **paul1r**: Parsing: String array elements were not being parsed correctly in JSON processing 
+* [11921](https://github.com/grafana/loki/pull/11921) **paul1r**: Parsing: String array elements were not being parsed correctly in JSON processing
 
 ##### Changes
 

cmd/loki/loki-local-with-memcached.yaml

@@ -22,6 +22,17 @@ query_range:
   cache_results: true
   cache_volume_results: true
   cache_series_results: true
+  cache_instant_metric_results: true
+  instant_metric_query_split_align: true
+  instant_metric_results_cache:
+    cache:
+      default_validity: 12h
+      memcached_client:
+        consistent_hash: true
+        addresses: "dns+localhost:11211"
+        max_idle_conns: 16
+        timeout: 500ms
+        update_interval: 1m
   series_results_cache:
     cache:
       default_validity: 12h

docs/sources/configure/_index.md

@@ -886,6 +886,28 @@ volume_results_cache:
   # CLI flag: -frontend.volume-results-cache.compression
   [compression: <string> | default = ""]
 
+# Cache instant metric query results.
+# CLI flag: -querier.cache-instant-metric-results
+[cache_instant_metric_results: <boolean> | default = false]
+
+# If a cache config is not specified and cache_instant_metric_results is true,
+# the config for the results cache is used.
+instant_metric_results_cache:
+  # The cache block configures the cache backend.
+  # The CLI flags prefix for this block configuration is:
+  # frontend.instant-metric-results-cache
+  [cache: <cache_config>]
+
+  # Use compression in cache. The default is an empty value '', which disables
+  # compression. Supported values are: 'snappy' and ''.
+  # CLI flag: -frontend.instant-metric-results-cache.compression
+  [compression: <string> | default = ""]
+
+# Whether to align the splits of instant metric query with splitByInterval and
+# query's exec time. Useful when instant_metric_cache is enabled
+# CLI flag: -querier.instant-metric-query-split-align
+[instant_metric_query_split_align: <boolean> | default = false]
+
 # Cache series query results.
 # CLI flag: -querier.cache-series-results
 [cache_series_results: <boolean> | default = false]
@@ -2935,6 +2957,13 @@ The `limits_config` block configures global and per-tenant limits in Loki.
 # CLI flag: -experimental.querier.recent-metadata-query-window
 [recent_metadata_query_window: <duration> | default = 0s]
 
+# Split instant metric queries by a time interval and execute in parallel. The
+# value 0 disables splitting instant metric queries by time. This also
+# determines how cache keys are chosen when instant metric query result caching
+# is enabled.
+# CLI flag: -querier.split-instant-metric-queries-by-interval
+[split_instant_metric_queries_by_interval: <duration> | default = 1h]
+
 # Interval to use for time-based splitting when a request is within the
 # `query_ingesters_within` window; defaults to `split-queries-by-interval` by
 # setting to 0.
@@ -3155,7 +3184,7 @@ shard_streams:
 
 # Skip factor for the n-grams created when computing blooms from log lines.
 # CLI flag: -bloom-compactor.ngram-skip
-[bloom_ngram_skip: <int> | default = 0]
+[bloom_ngram_skip: <int> | default = 1]
 
 # Scalable Bloom Filter desired false-positive rate.
 # CLI flag: -bloom-compactor.false-positive-rate
@@ -4403,6 +4432,7 @@ The cache block configures the cache backend. The supported CLI flags `<prefix>`
 - `bloom.metas-cache`
 - `frontend`
 - `frontend.index-stats-results-cache`
+- `frontend.instant-metric-results-cache`
 - `frontend.label-results-cache`
 - `frontend.series-results-cache`
 - `frontend.volume-results-cache`

docs/sources/release-notes/cadence.md

@@ -8,7 +8,7 @@ weight: 1
 
 ## Stable Releases
 
-Loki releases (this includes [Promtail](/clients/promtail), [Loki Canary](/operations/loki-canary/), etc) use the following
+Loki releases (this includes [Promtail](https://grafana.com/docs/loki/<LOKI_VERSION>/send-data/promtail/), [Loki Canary](https://grafana.com/docs/loki/<LOKI_VERSION>/operations/loki-canary/), etc.) use the following
 naming scheme: `MAJOR`.`MINOR`.`PATCH`.
 
 - `MAJOR` (roughly once a year): these releases include large new features and possible backwards-compatibility breaks.
@@ -18,14 +18,14 @@ naming scheme: `MAJOR`.`MINOR`.`PATCH`.
 {{% admonition type="note" %}}
 While our naming scheme resembles [Semantic Versioning](https://semver.org/), at this time we do not strictly follow its
 guidelines to the letter. Our goal is to provide regular releases that are as stable as possible, and we take backwards-compatibility
-seriously. As with any software, always read the [release notes](/release-notes) and the [upgrade guide](/upgrading) whenever
+seriously. As with any software, always read the [release notes](https://grafana.com/docs/loki/<LOKI_VERSION>/release-notes/) and the [upgrade guide](https://grafana.com/docs/loki/<LOKI_VERSION>/setup/upgrade/) whenever
 choosing a new version of Loki to install.
 {{% /admonition %}}
 
 New releases are based of a [weekly release](#weekly-releases) which we have vetted for stability over a number of weeks.
 
 We strongly recommend keeping up-to-date with patch releases as they are released. We post updates of new releases in the `#loki` channel
-of our [Slack community](/community/getting-in-touch).
+of our [Slack community](https://grafana.com/docs/loki/<LOKI_VERSION>/community/getting-in-touch/).
 
 You can find all of our releases [on GitHub](https://github.com/grafana/loki/releases) and on [Docker Hub](https://hub.docker.com/r/grafana/loki).
 

go.mod

@@ -118,7 +118,7 @@ require (
 	github.com/DmitriyVTitov/size v1.5.0
 	github.com/IBM/go-sdk-core/v5 v5.13.1
 	github.com/IBM/ibm-cos-sdk-go v1.10.0
-	github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc
+	github.com/axiomhq/hyperloglog v0.0.0-20240124082744-24bca3a5b39b
 	github.com/d4l3k/messagediff v1.2.1
 	github.com/efficientgo/core v1.0.0-rc.2
 	github.com/fsnotify/fsnotify v1.6.0

go.sum

@@ -368,8 +368,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 h1:xsOtPAvHqhvQvBza5ohaUcfq1Lce
 github.com/aws/aws-sdk-go-v2/service/sts v1.16.1/go.mod h1:Aq2/Qggh2oemSfyHH+EO4UBbgWG6zFCXLHYI4ILTY7w=
 github.com/aws/smithy-go v1.11.1 h1:IQ+lPZVkSM3FRtyaDox41R8YS6iwPMYIreejOgPW49g=
 github.com/aws/smithy-go v1.11.1/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
-github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc h1:Keo7wQ7UODUaHcEi7ltENhbAK2VgZjfat6mLy03tQzo=
-github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc/go.mod h1:k08r+Yj1PRAmuayFiRK6MYuR5Ve4IuZtTfxErMIh0+c=
+github.com/axiomhq/hyperloglog v0.0.0-20240124082744-24bca3a5b39b h1:F3yMzKumBUQ6Fn0sYI1YQ16vQRucpZOfBQ9HXWl5+XI=
+github.com/axiomhq/hyperloglog v0.0.0-20240124082744-24bca3a5b39b/go.mod h1:k08r+Yj1PRAmuayFiRK6MYuR5Ve4IuZtTfxErMIh0+c=
 github.com/baidubce/bce-sdk-go v0.9.141 h1:EV5BH5lfymIGPSmYDo9xYdsVlvWAW6nFeiA6t929zBE=
 github.com/baidubce/bce-sdk-go v0.9.141/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=

operator/CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+- [11964](https://github.com/grafana/loki/pull/11964) **xperimental**: Provide Azure region for managed credentials using environment variable
 - [11920](https://github.com/grafana/loki/pull/11920) **xperimental**: Refactor handling of credentials in managed-auth mode
 - [11869](https://github.com/grafana/loki/pull/11869) **periklis**: Add support for running with Google Workload Identity
 - [11868](https://github.com/grafana/loki/pull/11868) **xperimental**: Integrate support for OpenShift-managed credentials in Azure

operator/internal/config/managed_auth.go

@@ -26,6 +26,7 @@ func discoverManagedAuthConfig() *ManagedAuthConfig {
 	clientID := os.Getenv("CLIENTID")
 	tenantID := os.Getenv("TENANTID")
 	subscriptionID := os.Getenv("SUBSCRIPTIONID")
+	region := os.Getenv("REGION")
 
 	switch {
 	case roleARN != "":
@@ -40,6 +41,7 @@ func discoverManagedAuthConfig() *ManagedAuthConfig {
 				ClientID:       clientID,
 				SubscriptionID: subscriptionID,
 				TenantID:       tenantID,
+				Region:         region,
 			},
 		}
 	}

operator/internal/handlers/credentialsrequest_create.go → operator/internal/handlers/credentialsrequest.go

@@ -2,12 +2,10 @@ package handlers
 
 import (
 	"context"
-	"errors"
 	"fmt"
 
 	"github.com/ViaQ/logerr/v2/kverrors"
 	"github.com/go-logr/logr"
-	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -19,11 +17,8 @@ import (
 	"github.com/grafana/loki/operator/internal/external/k8s"
 	"github.com/grafana/loki/operator/internal/manifests"
 	"github.com/grafana/loki/operator/internal/manifests/openshift"
-	"github.com/grafana/loki/operator/internal/manifests/storage"
 )
 
-var errAzureNoRegion = errors.New("can not create CredentialsRequest: missing secret field: region")
-
 // CreateCredentialsRequest creates a new CredentialsRequest resource for a Lokistack
 // to request a cloud credentials Secret resource from the OpenShift cloud-credentials-operator.
 func CreateCredentialsRequest(ctx context.Context, log logr.Logger, scheme *runtime.Scheme, managedAuth *config.ManagedAuthConfig, k k8s.Client, req ctrl.Request) error {
@@ -39,32 +34,6 @@ func CreateCredentialsRequest(ctx context.Context, log logr.Logger, scheme *runt
 		return kverrors.Wrap(err, "failed to lookup LokiStack", "name", req.String())
 	}
 
-	if managedAuth.Azure != nil && managedAuth.Azure.Region == "" {
-		// Managed environment for Azure does not provide Region, but we need this for the CredentialsRequest.
-		// This looks like an oversight when creating the UI in OpenShift, but for now we need to pull this data
-		// from somewhere else -> the Azure Storage Secret
-		storageSecretName := client.ObjectKey{
-			Namespace: stack.Namespace,
-			Name:      stack.Spec.Storage.Secret.Name,
-		}
-		storageSecret := &corev1.Secret{}
-		if err := k.Get(ctx, storageSecretName, storageSecret); err != nil {
-			if apierrors.IsNotFound(err) {
-				// Skip this error here as it will be picked up by the LokiStack handler instead
-				ll.Error(err, "could not find secret for LokiStack", "name", req.String())
-				return nil
-			}
-			return err
-		}
-
-		region := storageSecret.Data[storage.KeyAzureRegion]
-		if len(region) == 0 {
-			return errAzureNoRegion
-		}
-
-		managedAuth.Azure.Region = string(region)
-	}
-
 	opts := openshift.Options{
 		BuildOpts: openshift.BuildOptions{
 			LokiStackName:      stack.Name,

operator/internal/handlers/credentialsrequest_create_test.go → operator/internal/handlers/credentialsrequest_test.go

@@ -6,7 +6,6 @@ import (
 
 	cloudcredentialv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1"
 	"github.com/stretchr/testify/require"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -19,7 +18,7 @@ import (
 	"github.com/grafana/loki/operator/internal/external/k8s/k8sfakes"
 )
 
-func credentialsRequestFakeClient(cr *cloudcredentialv1.CredentialsRequest, lokistack *lokiv1.LokiStack, secret *corev1.Secret) *k8sfakes.FakeClient {
+func credentialsRequestFakeClient(cr *cloudcredentialv1.CredentialsRequest, lokistack *lokiv1.LokiStack) *k8sfakes.FakeClient {
 	k := &k8sfakes.FakeClient{}
 	k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error {
 		switch object.(type) {
@@ -33,11 +32,6 @@ func credentialsRequestFakeClient(cr *cloudcredentialv1.CredentialsRequest, loki
 				return errors.NewNotFound(schema.GroupResource{}, name.Name)
 			}
 			k.SetClientObject(object, lokistack)
-		case *corev1.Secret:
-			if secret == nil {
-				return errors.NewNotFound(schema.GroupResource{}, name.Name)
-			}
-			k.SetClientObject(object, secret)
 		}
 		return nil
 	}
@@ -58,7 +52,7 @@ func TestCreateCredentialsRequest_CreateNewResource(t *testing.T) {
 		},
 	}
 
-	k := credentialsRequestFakeClient(nil, lokistack, nil)
+	k := credentialsRequestFakeClient(nil, lokistack)
 	req := ctrl.Request{
 		NamespacedName: client.ObjectKey{Name: "my-stack", Namespace: "ns"},
 	}
@@ -89,13 +83,8 @@ func TestCreateCredentialsRequest_CreateNewResourceAzure(t *testing.T) {
 			Namespace: "ns",
 		},
 	}
-	secret := &corev1.Secret{
-		Data: map[string][]byte{
-			"region": []byte(wantRegion),
-		},
-	}
 
-	k := credentialsRequestFakeClient(nil, lokistack, secret)
+	k := credentialsRequestFakeClient(nil, lokistack)
 	req := ctrl.Request{
 		NamespacedName: client.ObjectKey{Name: "my-stack", Namespace: "ns"},
 	}
@@ -105,6 +94,7 @@ func TestCreateCredentialsRequest_CreateNewResourceAzure(t *testing.T) {
 			ClientID:       "test-client-id",
 			SubscriptionID: "test-tenant-id",
 			TenantID:       "test-subscription-id",
+			Region:         "test-region",
 		},
 	}
 
@@ -122,47 +112,6 @@ func TestCreateCredentialsRequest_CreateNewResourceAzure(t *testing.T) {
 	require.Equal(t, wantRegion, providerSpec.AzureRegion)
 }
 
-func TestCreateCredentialsRequest_CreateNewResourceAzure_Errors(t *testing.T) {
-	lokistack := &lokiv1.LokiStack{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "my-stack",
-			Namespace: "ns",
-		},
-	}
-	req := ctrl.Request{
-		NamespacedName: client.ObjectKey{Name: "my-stack", Namespace: "ns"},
-	}
-
-	tt := []struct {
-		secret    *corev1.Secret
-		wantError string
-	}{
-		{
-			secret:    &corev1.Secret{},
-			wantError: errAzureNoRegion.Error(),
-		},
-	}
-
-	for _, tc := range tt {
-		tc := tc
-		t.Run(tc.wantError, func(t *testing.T) {
-			t.Parallel()
-
-			managedAuth := &config.ManagedAuthConfig{
-				Azure: &config.AzureEnvironment{
-					ClientID:       "test-client-id",
-					SubscriptionID: "test-tenant-id",
-					TenantID:       "test-subscription-id",
-				},
-			}
-			k := credentialsRequestFakeClient(nil, lokistack, tc.secret)
-
-			err := CreateCredentialsRequest(context.Background(), logger, scheme, managedAuth, k, req)
-			require.EqualError(t, err, tc.wantError)
-		})
-	}
-}
-
 func TestCreateCredentialsRequest_DoNothing_WhenCredentialsRequestExist(t *testing.T) {
 	req := ctrl.Request{
 		NamespacedName: client.ObjectKey{Name: "my-stack", Namespace: "ns"},
@@ -187,7 +136,7 @@ func TestCreateCredentialsRequest_DoNothing_WhenCredentialsRequestExist(t *testi
 		},
 	}
 
-	k := credentialsRequestFakeClient(cr, lokistack, nil)
+	k := credentialsRequestFakeClient(cr, lokistack)
 
 	err := CreateCredentialsRequest(context.Background(), logger, scheme, managedAuth, k, req)
 	require.NoError(t, err)

operator/internal/handlers/internal/storage/secrets.go

@@ -182,18 +182,11 @@ func extractAzureConfigSecret(s *corev1.Secret, fg configv1.FeatureGates) (*stor
 	// Extract and validate optional fields
 	endpointSuffix := s.Data[storage.KeyAzureStorageEndpointSuffix]
 	audience := s.Data[storage.KeyAzureAudience]
-	region := s.Data[storage.KeyAzureRegion]
 
 	if !workloadIdentity && len(audience) > 0 {
 		return nil, fmt.Errorf("%w: %s", errSecretFieldNotAllowed, storage.KeyAzureAudience)
 	}
 
-	if fg.OpenShift.ManagedAuthEnv {
-		if len(region) == 0 {
-			return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureRegion)
-		}
-	}
-
 	return &storage.AzureStorageConfig{
 		Env:              string(env),
 		Container:        string(container),