Simplifying managing system and metadata discovery labels. Adding ear…

…ly code for relabling configuration support to prevent need of modifying existent grafana dashboards.

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Factories/DiscoveryTargetBuilder.cs

@@ -89,8 +89,41 @@ public DiscoveryTargetBuilder WithoutLabels(ICollection<DiscoveryLabel> labels)
 
     public ICollection<DiscoveryLabel> Labels => _discoveryTarget.Labels;
 
+    /// <summary>
+    /// Adds system labels to the target - these are inferred from most valuable information
+    /// </summary>
+    internal DiscoveryTargetBuilder WithSystemLabels()
+    {
+        // Store system labels
+        List<DiscoverySystemLabel> systemLabels =
+        [
+            new EcsClusterDiscoverySystemLabel(_discoveryTarget),
+            new EcsServiceDiscoverySystemLabel(_discoveryTarget),
+            new EcsTaskDiscoverySystemLabel(_discoveryTarget),
+            new EcsTaskDefinitionDiscoverySystemLabel(_discoveryTarget),
+            new CloudMapServiceDiscoverySystemLabel(_discoveryTarget),
+            new CloudMapServiceInstanceDiscoverySystemLabel(_discoveryTarget),
+            new CloudMapServiceTypeDiscoverySystemLabel(_discoveryTarget)
+        ];
+
+        // Cast system labels to DiscoveryLabel
+        var labels = systemLabels.Select(DiscoveryLabel (l) => l).ToList();
+
+        WithLabels(labels);
+
+        // Cast system labels to DiscoveryMetadataLabel and provide these values as metadata to allow relabelling in Prometheus
+        labels = systemLabels.Select(DiscoveryLabel (l) => new DiscoveryMetadataLabel(l)).ToList();
+
+        WithLabels(labels);
+
+        return this;
+    }
+
     public DiscoveryTarget Build()
     {
+        // Add system labels
+        WithSystemLabels();
+
         return _discoveryTarget;
     }
 }

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabel.cs

@@ -8,17 +8,17 @@ public class DiscoveryLabel : IEquatable<DiscoveryLabel>
     /// <summary>
     /// Label name
     /// </summary>
-    public required string Name { get; init; }
+    public string Name { get; init; } = null!;
 
     /// <summary>
     /// Label value
     /// </summary>
-    public required string Value { get; init; }
+    public string Value { get; init; } = null!;
 
     /// <summary>
     /// Label priority
     /// </summary>
-    public required DiscoveryLabelPriority Priority { get; init; }
+    public DiscoveryLabelPriority Priority { get; init; }
 
     public bool Equals(DiscoveryLabel? other)
     {
@@ -49,11 +49,13 @@ public override bool Equals(object? obj)
             return true;
         }
 
-        return obj.GetType() == GetType() && Equals((DiscoveryLabel) obj);
+        return obj.GetType() == GetType() && Equals((DiscoveryLabel)obj);
     }
 
     public override int GetHashCode()
     {
         return HashCode.Combine(Name, Value);
     }
+
+    public bool HasValue => !string.IsNullOrWhiteSpace(Value);
 }

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabelPriority.cs

@@ -5,6 +5,17 @@ namespace Apptality.CloudMapEcsPrometheusDiscovery.Discovery.Models;
 /// </summary>
 public enum DiscoveryLabelPriority
 {
+    /// <summary>
+    /// System labels are of the highest priority, intended for internal use,
+    /// and will take precedence over other labels.
+    /// </summary>
+    System = 0,
+    /// <summary>
+    /// Metadata labels are of the second-highest priority and will take precedence over other labels.
+    /// These used by Prometheus to group targets.
+    /// </summary>
+    Metadata = 1,
+
     /// <summary>
     /// Labels inferred from tags resolved from the ECS Tasks are of the highest priority,
     /// and will take precedence over other tags resolved from parent service, CloudMap service,

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoverySystemLabels.cs

@@ -0,0 +1,75 @@
+namespace Apptality.CloudMapEcsPrometheusDiscovery.Discovery.Models;
+
+/// <summary>
+/// System labels directly inferred from the AWS resource properties
+/// </summary>
+internal abstract class DiscoverySystemLabel : DiscoveryLabel
+{
+    const string LabelPrefix = "_sys_";
+    public string OriginalName { get; init; }
+
+    protected DiscoverySystemLabel(string name, string value)
+    {
+        OriginalName = name;
+        Name = $"{LabelPrefix}{name}";
+        Value = value;
+        Priority = DiscoveryLabelPriority.System;
+    }
+}
+
+/// <summary>
+/// Label for metadata information. These then can be used for relabelling in Prometheus
+/// </summary>
+/// <remarks>
+/// Read more at:
+/// <ul>
+/// <li><a href="https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config">Prometheus Documentation</a></li>
+/// <li><a href="https://www.robustperception.io/life-of-a-label/">Life of a Label</a></li>
+/// </ul>
+/// </remarks>
+internal class DiscoveryMetadataLabel : DiscoveryLabel
+{
+    const string LabelPrefix = "__meta_";
+
+    private DiscoveryMetadataLabel(string value)
+    {
+        Value = value;
+        Priority = DiscoveryLabelPriority.Metadata;
+    }
+
+    public DiscoveryMetadataLabel(string name, string value) : this(value)
+    {
+        Name = $"{LabelPrefix}{name}";
+    }
+
+    public DiscoveryMetadataLabel(DiscoveryLabel label) : this(label.Value)
+    {
+        Name = $"{LabelPrefix}{label.Name}";
+    }
+
+    public DiscoveryMetadataLabel(DiscoverySystemLabel label) : this(label.Value)
+    {
+        Name = $"{LabelPrefix}{label.OriginalName}";
+    }
+}
+
+internal class EcsClusterDiscoverySystemLabel(DiscoveryTarget value)
+    : DiscoverySystemLabel("ecs_cluster", value.EcsCluster);
+
+internal class EcsServiceDiscoverySystemLabel(DiscoveryTarget value)
+    : DiscoverySystemLabel("ecs_service", value.EcsService);
+
+internal class EcsTaskDiscoverySystemLabel(DiscoveryTarget value)
+    : DiscoverySystemLabel("ecs_task", value.EcsTaskArn);
+
+internal class EcsTaskDefinitionDiscoverySystemLabel(DiscoveryTarget value)
+    : DiscoverySystemLabel("ecs_task_definition", value.EcsTaskDefinitionArn);
+
+internal class CloudMapServiceDiscoverySystemLabel(DiscoveryTarget value)
+    : DiscoverySystemLabel("cloudmap_service_name", value.CloudMapServiceName);
+
+internal class CloudMapServiceInstanceDiscoverySystemLabel(DiscoveryTarget value)
+    : DiscoverySystemLabel("cloudmap_service_instance_id", value.CloudMapServiceInstanceId);
+
+internal class CloudMapServiceTypeDiscoverySystemLabel(DiscoveryTarget value)
+    : DiscoverySystemLabel("cloudmap_service_type", value.ServiceType?.ToString() ?? "");

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Options/DiscoveryOptions.cs

@@ -138,8 +138,8 @@ public sealed class DiscoveryOptions
     /// </summary>
     /// <remarks>
     /// This prefix has a special meaning and is used to identify the metrics path, port, and name
-    /// for the service scrape configuration (in a single ecs service you may have multiple
-    /// containers listening on different ports, possibly having a different path you need to scrape from).
+    /// for the service scrape configuration. In a single ecs service, you may have multiple
+    /// containers listening on different ports, possibly having a different path you need to scrape from.
     /// <para>
     /// 'PATH', 'PORT', 'NAME' will be appended to the prefix to identify
     /// metrics path, port, and scrape configuration names respectively.
@@ -174,7 +174,7 @@ public sealed class DiscoveryOptions
     /// </para>
     /// If a path is not provided, default path "/metrics" will be used.
     /// <br />
-    /// If a name is not provided, the service name will be omitted from labels ('scrape_cfg_name' label).
+    /// If a name is not provided, the service name will be omitted from labels ('scrape_target_name' label).
     /// <br />
     /// This directly affects the Prometheus http sd scrape configuration:
     /// <code>
@@ -184,7 +184,7 @@ public sealed class DiscoveryOptions
     ///    ],
     ///    "labels": {
     ///      "__metrics_path__": "${METRICS_PATH}",
-    ///      "scrape_cfg_name": "${METRICS_NAME}",
+    ///      "scrape_target_name": "${METRICS_NAME}",
     ///      ...
     ///     }
     /// }]
@@ -193,4 +193,70 @@ public sealed class DiscoveryOptions
     [Required]
     [RegularExpression(@"^[a-zA-Z]{1}[\w-]+[_]{1}$")]
     public string MetricsPathPortTagPrefix { get; set; } = "METRICS_";
+
+    /// <summary>
+    /// Semicolon separated string of relabel configurations to apply to the scrape configuration result.
+    /// This allows using simple syntax to create new labels with existing values,
+    /// or to combine multiple pre-calculated labels into a single label.
+    /// </summary>
+    /// <remarks>
+    /// The intended use case is to create new labels based on existing labels, so that you don't have to
+    /// modify your Grafana Dashboards or alerting rules when you change the way you tag your resources.
+    /// <br />
+    /// Relabel configurations are applied to the scrape configuration before it is returned to the client.
+    /// You can only operate using labels that are already present in the target.
+    /// <br />
+    /// Example:
+    /// Consider the following response of discovery target for configuration without relabeling:
+    /// <code>
+    ///  {
+    ///        "targets": ["10.200.10.200:8080"],
+    ///        "labels": {
+    ///            "__metrics_path__": "/metrics",
+    ///            "instance": "10.200.10.200",
+    ///            "scrape_target_name": "app",
+    ///            "ecs_cluster": "my-cluster",
+    ///            "ecs_service": "my-service",
+    ///            ...
+    ///        }
+    ///    },
+    /// </code>
+    /// By applying the following relabel configuration:
+    /// <code>
+    /// "service_and_cluster={{ecs_cluster}}-{{ecs_service}};"
+    /// </code>
+    /// This will create a new label 'service_and_cluster' with the value of 'ecs_cluster' and 'ecs_service' labels combined:
+    /// <code>
+    ///  {
+    ///        "targets": ["10.200.10.200:8080"],
+    ///        "labels": {
+    ///            "__metrics_path__": "/metrics",
+    ///            "instance": "10.200.10.200",
+    ///            "scrape_target_name": "app",
+    ///            "ecs_cluster": "my-cluster",
+    ///            "ecs_service": "my-service",
+    ///            "service_and_cluster": "my-cluster-my-service",
+    ///            ...
+    ///        }
+    ///    },
+    /// </code>
+    /// <br />
+    /// If your intent is to only 'rename' the existing label, you can use the following syntax:
+    /// <code>
+    /// "cluster={{ecs_cluster}};"
+    /// </code>
+    /// This will create new label 'cluster' with the value of 'ecs_cluster' label.
+    /// <br />
+    /// IMPORTANT:
+    /// <ul>
+    /// <li>1) If amy of the replacement tokens are not found in the source labels, it will not be replaced at all.</li>
+    /// <li>2) You can replace existing labels ("ecs_cluster=my-{{ecs_cluster}};").</li>
+    /// <li>3) You can have multiple relabel configurations for the same label. They will be applied in order defined ("a=1;a={{a}}-2;" will result in a="1-2").</li>
+    /// </ul>
+    /// <br/>
+    /// To have a fall-backs, use 'ExtraPrometheusLabels' option. These have the lowest priority,
+    /// and are only added to the response if the label is not already present in the target
+    /// (matching tag is not present in any of the original resources).
+    /// </remarks>
+    public string RelabelConfigurations { get; set; } = string.Empty;
 }

src/Apptality.CloudMapEcsPrometheusDiscovery/Prometheus/PrometheusResponseFactory.cs

@@ -36,24 +36,20 @@ public static PrometheusResponse Create(ICollection<DiscoveryTarget> discoveryTa
                 // Add scrape configuration specific labele, if provided
                 if (!string.IsNullOrWhiteSpace(scrapeConfiguration.Name))
                 {
-                    staticConfig.Labels.Add("scrape_cfg_name", scrapeConfiguration.Name);
+                    staticConfig.Labels.Add("scrape_target_name", scrapeConfiguration.Name);
                 }
 
-                // Add ECS specific labels
-                staticConfig.Labels.AddLabelWithValue("ecs_cluster", discoveryTarget.EcsCluster, true);
-                staticConfig.Labels.AddLabelWithValue("ecs_service", discoveryTarget.EcsService, true);
-                staticConfig.Labels.AddLabelWithValue("ecs_task", discoveryTarget.EcsTaskArn);
-                staticConfig.Labels.AddLabelWithValue("ecs_task_definition_arn", discoveryTarget.EcsTaskDefinitionArn);
-
-                // Add optional CloudMap labels, when not empty
-                staticConfig.Labels.AddLabelWithValue("cloudmap_service_name", discoveryTarget.CloudMapServiceName, true);
-                staticConfig.Labels.AddLabelWithValue("cloudmap_service_instance_id", discoveryTarget.CloudMapServiceInstanceId);
-                staticConfig.Labels.AddLabelWithValue("cloudmap_service_type", discoveryTarget.ServiceType?.ToString() ?? "");
-
-                // Now add all the labels from the target
+                // Stage labels in a dictionary to sort them before adding to the static config
+                var interimLabels = new Dictionary<string, string>();
                 foreach (var discoveryLabel in discoveryTarget.Labels)
                 {
-                    staticConfig.Labels.AddLabelWithValue(discoveryLabel.Name, discoveryLabel.Value);
+                    interimLabels.AddLabelWithValue(discoveryLabel.Name, discoveryLabel.Value);
+                }
+
+                // Now sort the labels and add them to the static config
+                foreach (var entry in interimLabels.OrderBy(d=>d.Key.ToLower()))
+                {
+                    staticConfig.Labels.Add(entry.Key, entry.Value);
                 }
 
                 response.Add(staticConfig);
@@ -69,21 +65,16 @@ public static PrometheusResponse Create(ICollection<DiscoveryTarget> discoveryTa
     internal static void AddLabelWithValue(
         this Dictionary<string, string> labels,
         string labelName,
-        string labelValue,
-        bool isMetaLabel = false
+        string labelValue
     )
     {
         if (string.IsNullOrWhiteSpace(labelValue)) return;
 
         var validLabelName = labelName.ToValidPrometheusLabelName();
-        if (labels.TryAdd(validLabelName, labelValue) && isMetaLabel)
+        if (!labels.TryAdd(validLabelName, labelValue))
         {
-            // Add meta-label if it was added successfully
-            labels.TryAdd($"__meta_{validLabelName}", labelValue);
-            return;
+            Log.Debug("Failed to add label {LabelName} with value {LabelValue}", validLabelName, labelValue);
         }
-
-        Log.Debug("Failed to add label {LabelName} with value {LabelValue}", validLabelName, labelValue);
     }
 
     /// <summary>

src/Apptality.CloudMapEcsPrometheusDiscovery/appsettings.json

@@ -99,7 +99,7 @@
     // * METRICS_PORT = 9001 (default path "/metrics" will be used, no configuration name will be added)
     //
     // If a path is not provided, default path "/metrics" will be used.
-    // If a name is not provided, the service name will be omitted from labels ('scrape_cfg_name' label).
+    // If a name is not provided, the service name will be omitted from labels ('scrape_target_name' label).
     //
     // This directly affects the Prometheus http sd scrape configuration:
     // [{
@@ -108,7 +108,7 @@
     //    ],
     //    "labels": {
     //      "__metrics_path__": "${METRICS_PATH}",
-    //      "scrape_cfg_name": "${METRICS_NAME}",
+    //      "scrape_target_name": "${METRICS_NAME}",
     //      ...
     //     }
     // }]