Adding support for re-labeling. This should avoid the need for develo…

…pers to have breaking changes in Grafana dashboards.

README.MD

@@ -37,17 +37,18 @@ Below is a high level overview of what application is capable of. For the full l
 * **Scalable and Flexible**: Built to be scalable and flexible, adapting to changes in the service infrastructure
   * Allows having any number of port-metrics pairs per each running ECS Task (useful when your ECS Task Definition defines multiple containers, which also expose ports and metrics endpoint). See `MetricsPathPortTagPrefix` configuration property for more details.
   * Allows supplying static set of labels to be added to every Prometheus target (see `ExtraPrometheusLabels` configuration property)
+  * Supports rich mechanism for re-labling, so you don't have to fix your Grafana dashboards. See `RelabelConfigurations` configuration property for more details.
   * Exposes health check endpoint: `/health`
 
-Only one of `EcsClusters` or `CloudMapNamespaces` can be provided for application to work. If you would like to include both [Service Connect](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/service-connect.html) and [Service Discovery](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/service-discovery.html) targets, `CloudMapNamespaces` must be supplied (with or without `EcsClusters`). When **both parameters** are supplied, the end result is only an intersection of targets that exist in ECS clusters supplied and CloudMap namespaces provided.
+At least one of `EcsClusters` or `CloudMapNamespaces` must be provided for application to work. If you would like to include both [Service Connect](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/service-connect.html) and [Service Discovery](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/service-discovery.html) targets, `CloudMapNamespaces` must be supplied (with or without `EcsClusters`). When **both parameters** are supplied, the end result is only an intersection of targets that exist in ECS clusters supplied and CloudMap namespaces provided.
 
 > **Permissions**: IAM permissions are required to discover ECS clusters (`ecs:Get*`, `ecs:List*`, `ecs:Describe*`) and CloudMap namespaces (`servicediscovery:Get*`, `servicediscovery:List*`, `servicediscovery:Discover*`, `route53:Get*`)
 
 ## Usage Example
 
 > 🚧 Documentation is still in progress, it'll be finished in next week or so.
 
-If anyone is willing to give this a trial - please refer to [appsettings.json](src/Apptality.CloudMapEcsPrometheusDiscovery/appsettings.json) for complete list of supported configuration options.
+You can also refer to [appsettings.json](src/Apptality.CloudMapEcsPrometheusDiscovery/appsettings.json) for supported configuration parameters.
 
 Example run command:
 

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Factories/DiscoveryTargetBuilder.cs

@@ -10,6 +10,10 @@ public class DiscoveryTargetBuilder
 {
     private readonly DiscoveryTarget _discoveryTarget = new();
 
+    private readonly List<RelabelConfiguration> _relabelConfigurations = [];
+
+    public ICollection<DiscoveryLabel> Labels => _discoveryTarget.Labels;
+
     public DiscoveryTargetBuilder WithCloudMapServiceName(string cloudMapServiceName)
     {
         _discoveryTarget.CloudMapServiceName = cloudMapServiceName;
@@ -81,6 +85,7 @@ public DiscoveryTargetBuilder WithLabels(ICollection<DiscoveryLabel> labels)
                 // Remove the existing label if it's less important
                 _discoveryTarget.Labels.Remove(existingLabel);
             }
+
             // Add the new label
             _discoveryTarget.Labels.Add(label);
         }
@@ -90,16 +95,31 @@ public DiscoveryTargetBuilder WithLabels(ICollection<DiscoveryLabel> labels)
 
     public DiscoveryTargetBuilder WithoutLabels(ICollection<DiscoveryLabel> labels)
     {
-        _discoveryTarget.Labels.RemoveAll(l => labels.Select(lb => lb.Name).Contains(l.Name));
+        _discoveryTarget.Labels.RemoveAll(l => labels.Select(lb => lb.Name.ToLower()).Contains(l.Name.ToLower()));
         return this;
     }
 
-    public ICollection<DiscoveryLabel> Labels => _discoveryTarget.Labels;
+    public DiscoveryTargetBuilder WithRelabelConfigurations(ICollection<RelabelConfiguration> relabelConfigurations)
+    {
+        _relabelConfigurations.AddRange(relabelConfigurations);
+        return this;
+    }
+
+    public DiscoveryTarget Build()
+    {
+        // Add system labels
+        ApplySystemLabels();
+
+        // Apply re-label configurations
+        ApplyRelabelConfigurations();
+
+        return _discoveryTarget;
+    }
 
     /// <summary>
     /// Adds system labels to the target - these are inferred from most valuable information
     /// </summary>
-    internal DiscoveryTargetBuilder WithSystemLabels()
+    private void ApplySystemLabels()
     {
         // Store system labels
         List<DiscoverySystemLabel> systemLabels =
@@ -122,15 +142,31 @@ internal DiscoveryTargetBuilder WithSystemLabels()
         labels = systemLabels.Select(DiscoveryLabel (l) => new DiscoveryMetadataLabel(l)).ToList();
 
         WithLabels(labels);
-
-        return this;
     }
 
-    public DiscoveryTarget Build()
+    /// <summary>
+    /// Applies relabel configurations to the target
+    /// </summary>
+    private void ApplyRelabelConfigurations()
     {
-        // Add system labels
-        WithSystemLabels();
+        foreach (var relabelConfiguration in _relabelConfigurations)
+        {
+            var label = relabelConfiguration.ToDiscoveryLabel(_discoveryTarget.Labels);
+            if (label == null) continue;
+            // It is very important to add the label at each iteration,
+            // because the same label can be re-labeled multiple times,
+            // or it may be used in next replacement patterns.
+            WithLabels([label]);
+        }
 
-        return _discoveryTarget;
+        var temporaryLabelsKeys = _relabelConfigurations
+            .Where(l => l.Temporary)
+            .Select(l => l.TargetLabelName)
+            .ToList();
+
+        // Remove temporary labels (these are a byproduct of re-labelling), if any
+        _discoveryTarget.Labels.RemoveAll(
+            l => l.Name.StartsWith("_tmp", StringComparison.OrdinalIgnoreCase) && temporaryLabelsKeys.Contains(l.Name)
+        );
     }
 }

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Factories/DiscoveryTargetFactory.cs

@@ -50,6 +50,9 @@ public ICollection<DiscoveryTarget> Create(DiscoveryResult discoveryResult)
                     // Build scrape configurations based on the discovery result and the labels aggregated
                     BuildScrapeConfigurations(builder, discoveryOptions);
 
+                    // Apply re-labeling rules
+                    builder.WithRelabelConfigurations(discoveryOptions.GetRelabelConfigurations());
+
                     discoveryTargets.Add(builder.Build());
                 }
             }

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabel.cs

@@ -56,6 +56,4 @@ public override int GetHashCode()
     {
         return HashCode.Combine(Name, Value);
     }
-
-    public bool HasValue => !string.IsNullOrWhiteSpace(Value);
 }

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabelPriority.cs

@@ -6,15 +6,24 @@ namespace Apptality.CloudMapEcsPrometheusDiscovery.Discovery.Models;
 public enum DiscoveryLabelPriority
 {
     /// <summary>
-    /// System labels are of the highest priority, intended for internal use,
+    /// Relabel labels are of the highest priority and will take precedence over other labels.
+    /// This is because relabeling is done after all labels have been constructed, but right before
+    /// the target is built. This gives developer full control over the target labels, and how they
+    /// should be constructed.
+    /// </summary>
+    Relabel = 0,
+
+    /// <summary>
+    /// System labels are of the second-highest priority, intended for internal use,
     /// and will take precedence over other labels.
     /// </summary>
-    System = 0,
+    System = 1,
+
     /// <summary>
-    /// Metadata labels are of the second-highest priority and will take precedence over other labels.
+    /// Metadata labels are of the third-highest priority and will take precedence over other labels.
     /// These used by Prometheus to group targets.
     /// </summary>
-    Metadata = 1,
+    Metadata = 2,
 
     /// <summary>
     /// Labels inferred from tags resolved from the ECS Tasks are of the highest priority,

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/RelabelConfiguration.cs

@@ -0,0 +1,129 @@
+using System.Text.RegularExpressions;
+using Apptality.CloudMapEcsPrometheusDiscovery.Discovery.Options;
+using Apptality.CloudMapEcsPrometheusDiscovery.Prometheus;
+using Serilog;
+
+namespace Apptality.CloudMapEcsPrometheusDiscovery.Discovery.Models;
+
+public class RelabelConfiguration
+{
+    /// <summary>
+    /// Target label name
+    /// </summary>
+    public string TargetLabelName { get; init; } = null!;
+
+    /// <summary>
+    /// Replacement pattern
+    /// </summary>
+    public string? ReplacementPattern { get; init; }
+
+    /// <summary>
+    /// Temporary labels are not persisted in the discovery process,
+    /// and are used only for re-labeling as temporary value holders.
+    /// These must start with a _tmp prefix.
+    /// </summary>
+    /// <example>
+    /// Adding prefix to all labels is the most common use case for temporary labels:
+    /// <code>
+    /// _tmp_p=my_awesome_app;ecs_cluster={{_tmp_p}}_{{ecs_cluster}};ecs_service={{_tmp_p}}_{{ecs_service}};
+    /// </code>
+    /// will result in ecs_cluster and ecs_service labels being prefixed with "my_awesome_app_",
+    /// while the '_tmp_p' label will not be written to the target label.
+    /// </example>
+    public bool Temporary { get; set; }
+
+    /// <summary>
+    /// Based on pattern defined and labels provided, builds a new label where tokens are replaced with values
+    /// from the labels.
+    /// </summary>
+    /// <param name="labels">
+    /// Collection of labels to use for replacements in the replacement pattern
+    /// </param>
+    /// <remarks>
+    /// Label can only be constructed if all the labels required for the replacement pattern are present.
+    /// </remarks>
+    /// <returns>
+    /// New label or null if it cannot be built.
+    /// </returns>
+    public DiscoveryLabel? ToDiscoveryLabel(ICollection<DiscoveryLabel> labels)
+    {
+        // If no replacement pattern is provided, return null
+        if (string.IsNullOrWhiteSpace(ReplacementPattern)) return null;
+
+        // Extract token keys from the replacement pattern
+        var tokens = ExtractTokenKeys(ReplacementPattern);
+
+        // If no tokens are found - means that the replacement pattern is a static value.
+        // Write a warning, because static values should be used instead.
+        // I see this probably being used to not repeat the same value in multiple places,
+        // and referencing it instead. Well, I hope you know what you're doing.
+        if (tokens.Count == 0)
+        {
+            if (!TargetLabelName.ToLower().StartsWith("_tmp"))
+            {
+                Log.Warning(
+                    """
+                    Replacement pattern '{ReplacementPattern}' for target label '{TargetLabel}' is a static value.
+                    If you're not using this for further processing, use '{ExtraPrometheusLabels}' configuration option instead.
+                    """,
+                    ReplacementPattern, TargetLabelName,
+                    nameof(DiscoveryOptions.ExtraPrometheusLabels)
+                );
+                return null;
+            }
+
+            // Mark the label as temporary, because it's not used for anything else
+            Temporary = true;
+
+            return new DiscoveryLabel
+            {
+                Name = TargetLabelName.ToValidPrometheusLabelName(),
+                Value = ReplacementPattern,
+                Priority = DiscoveryLabelPriority.Relabel
+            };
+        }
+
+        // Pre-select all keys from existing labels
+        var existingLabelsLookup = labels.ToDictionary(l => l.Name.ToLower());
+
+        // Ensure all the tokens are present in the labels
+        if (!tokens.All(t => existingLabelsLookup.Keys.Contains(t)))
+        {
+            // If not all the tokens are present, return null, because the label cannot be built
+            return null;
+        }
+
+        // Replace tokens with the values from the labels
+        var newLabelValue = tokens.Aggregate(
+            ReplacementPattern,
+            (current, token) => current.Replace($$$"""{{{{{token}}}}}""", existingLabelsLookup[token].Value)
+        );
+
+        return new DiscoveryLabel
+        {
+            Name = TargetLabelName,
+            Value = newLabelValue,
+            Priority = DiscoveryLabelPriority.Relabel
+        };
+
+        // If not all the tokens are present, return null, because the label cannot be built
+    }
+
+    /// <summary>
+    /// Extracts token keys from the replacement pattern
+    /// </summary>
+    /// <param name="replacementPattern">
+    /// Replacement pattern to extract token keys from: "{{token1}} {{token2}} {{token3}}"
+    /// </param>
+    /// <returns>
+    /// Returns a collection of token keys extracted from the replacement pattern in the order they appear,
+    /// with duplicates removed, and all keys are lowercased.
+    /// </returns>
+    internal static ICollection<string> ExtractTokenKeys(string replacementPattern)
+    {
+        return Regex.Matches(replacementPattern, @"\{\{(\w+)\}\}")
+            .Select(m => m.Groups[1].Value.ToLower())
+            .Distinct()
+            .ToList();
+    }
+}

src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Options/DiscoveryOptions.cs

@@ -251,7 +251,8 @@ public sealed class DiscoveryOptions
     /// <ul>
     /// <li>1) If amy of the replacement tokens are not found in the source labels, it will not be replaced at all.</li>
     /// <li>2) You can replace existing labels ("ecs_cluster=my-{{ecs_cluster}};").</li>
-    /// <li>3) You can have multiple relabel configurations for the same label. They will be applied in order defined ("a=1;a={{a}}-2;" will result in a="1-2").</li>
+    /// <li>3) All relabel configurations are applied in order defined.</li>
+    /// <li>4) You can have multiple relabel configurations for the same label. They will be applied in order defined ("a=1;a={{a}}-2;" will result in a="1-2").</li>
     /// </ul>
     /// <br/>
     /// To have a fall-backs, use 'ExtraPrometheusLabels' option. These have the lowest priority,