diff --git a/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Factories/DiscoveryTargetBuilder.cs b/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Factories/DiscoveryTargetBuilder.cs index 520110a..10cd7e0 100644 --- a/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Factories/DiscoveryTargetBuilder.cs +++ b/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Factories/DiscoveryTargetBuilder.cs @@ -89,8 +89,41 @@ public DiscoveryTargetBuilder WithoutLabels(ICollection labels) public ICollection Labels => _discoveryTarget.Labels; + /// + /// Adds system labels to the target - these are inferred from most valuable information + /// + internal DiscoveryTargetBuilder WithSystemLabels() + { + // Store system labels + List systemLabels = + [ + new EcsClusterDiscoverySystemLabel(_discoveryTarget), + new EcsServiceDiscoverySystemLabel(_discoveryTarget), + new EcsTaskDiscoverySystemLabel(_discoveryTarget), + new EcsTaskDefinitionDiscoverySystemLabel(_discoveryTarget), + new CloudMapServiceDiscoverySystemLabel(_discoveryTarget), + new CloudMapServiceInstanceDiscoverySystemLabel(_discoveryTarget), + new CloudMapServiceTypeDiscoverySystemLabel(_discoveryTarget) + ]; + + // Cast system labels to DiscoveryLabel + var labels = systemLabels.Select(DiscoveryLabel (l) => l).ToList(); + + WithLabels(labels); + + // Cast system labels to DiscoveryMetadataLabel and provide these values as metadata to allow relabelling in Prometheus + labels = systemLabels.Select(DiscoveryLabel (l) => new DiscoveryMetadataLabel(l)).ToList(); + + WithLabels(labels); + + return this; + } + public DiscoveryTarget Build() { + // Add system labels + WithSystemLabels(); + return _discoveryTarget; } } \ No newline at end of file diff --git a/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabel.cs b/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabel.cs index 087e4a4..c157586 100644 --- a/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabel.cs +++ b/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabel.cs @@ -8,17 +8,17 @@ public class DiscoveryLabel : IEquatable /// /// Label name /// - public required string Name { get; init; } + public string Name { get; init; } = null!; /// /// Label value /// - public required string Value { get; init; } + public string Value { get; init; } = null!; /// /// Label priority /// - public required DiscoveryLabelPriority Priority { get; init; } + public DiscoveryLabelPriority Priority { get; init; } public bool Equals(DiscoveryLabel? other) { @@ -49,11 +49,13 @@ public override bool Equals(object? obj) return true; } - return obj.GetType() == GetType() && Equals((DiscoveryLabel) obj); + return obj.GetType() == GetType() && Equals((DiscoveryLabel)obj); } public override int GetHashCode() { return HashCode.Combine(Name, Value); } + + public bool HasValue => !string.IsNullOrWhiteSpace(Value); } \ No newline at end of file diff --git a/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabelPriority.cs b/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabelPriority.cs index 8545c97..2a5e695 100644 --- a/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabelPriority.cs +++ b/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoveryLabelPriority.cs @@ -5,6 +5,17 @@ namespace Apptality.CloudMapEcsPrometheusDiscovery.Discovery.Models; /// public enum DiscoveryLabelPriority { + /// + /// System labels are of the highest priority, intended for internal use, + /// and will take precedence over other labels. + /// + System = 0, + /// + /// Metadata labels are of the second-highest priority and will take precedence over other labels. + /// These used by Prometheus to group targets. + /// + Metadata = 1, + /// /// Labels inferred from tags resolved from the ECS Tasks are of the highest priority, /// and will take precedence over other tags resolved from parent service, CloudMap service, diff --git a/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoverySystemLabels.cs b/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoverySystemLabels.cs new file mode 100644 index 0000000..53d6e23 --- /dev/null +++ b/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Models/DiscoverySystemLabels.cs @@ -0,0 +1,75 @@ +namespace Apptality.CloudMapEcsPrometheusDiscovery.Discovery.Models; + +/// +/// System labels directly inferred from the AWS resource properties +/// +internal abstract class DiscoverySystemLabel : DiscoveryLabel +{ + const string LabelPrefix = "_sys_"; + public string OriginalName { get; init; } + + protected DiscoverySystemLabel(string name, string value) + { + OriginalName = name; + Name = $"{LabelPrefix}{name}"; + Value = value; + Priority = DiscoveryLabelPriority.System; + } +} + +/// +/// Label for metadata information. These then can be used for relabelling in Prometheus +/// +/// +/// Read more at: +/// +/// +internal class DiscoveryMetadataLabel : DiscoveryLabel +{ + const string LabelPrefix = "__meta_"; + + private DiscoveryMetadataLabel(string value) + { + Value = value; + Priority = DiscoveryLabelPriority.Metadata; + } + + public DiscoveryMetadataLabel(string name, string value) : this(value) + { + Name = $"{LabelPrefix}{name}"; + } + + public DiscoveryMetadataLabel(DiscoveryLabel label) : this(label.Value) + { + Name = $"{LabelPrefix}{label.Name}"; + } + + public DiscoveryMetadataLabel(DiscoverySystemLabel label) : this(label.Value) + { + Name = $"{LabelPrefix}{label.OriginalName}"; + } +} + +internal class EcsClusterDiscoverySystemLabel(DiscoveryTarget value) + : DiscoverySystemLabel("ecs_cluster", value.EcsCluster); + +internal class EcsServiceDiscoverySystemLabel(DiscoveryTarget value) + : DiscoverySystemLabel("ecs_service", value.EcsService); + +internal class EcsTaskDiscoverySystemLabel(DiscoveryTarget value) + : DiscoverySystemLabel("ecs_task", value.EcsTaskArn); + +internal class EcsTaskDefinitionDiscoverySystemLabel(DiscoveryTarget value) + : DiscoverySystemLabel("ecs_task_definition", value.EcsTaskDefinitionArn); + +internal class CloudMapServiceDiscoverySystemLabel(DiscoveryTarget value) + : DiscoverySystemLabel("cloudmap_service_name", value.CloudMapServiceName); + +internal class CloudMapServiceInstanceDiscoverySystemLabel(DiscoveryTarget value) + : DiscoverySystemLabel("cloudmap_service_instance_id", value.CloudMapServiceInstanceId); + +internal class CloudMapServiceTypeDiscoverySystemLabel(DiscoveryTarget value) + : DiscoverySystemLabel("cloudmap_service_type", value.ServiceType?.ToString() ?? ""); \ No newline at end of file diff --git a/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Options/DiscoveryOptions.cs b/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Options/DiscoveryOptions.cs index 3794c5e..107488c 100644 --- a/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Options/DiscoveryOptions.cs +++ b/src/Apptality.CloudMapEcsPrometheusDiscovery/Discovery/Options/DiscoveryOptions.cs @@ -138,8 +138,8 @@ public sealed class DiscoveryOptions /// /// /// This prefix has a special meaning and is used to identify the metrics path, port, and name - /// for the service scrape configuration (in a single ecs service you may have multiple - /// containers listening on different ports, possibly having a different path you need to scrape from). + /// for the service scrape configuration. In a single ecs service, you may have multiple + /// containers listening on different ports, possibly having a different path you need to scrape from. /// /// 'PATH', 'PORT', 'NAME' will be appended to the prefix to identify /// metrics path, port, and scrape configuration names respectively. @@ -174,7 +174,7 @@ public sealed class DiscoveryOptions /// /// If a path is not provided, default path "/metrics" will be used. ///
- /// If a name is not provided, the service name will be omitted from labels ('scrape_cfg_name' label). + /// If a name is not provided, the service name will be omitted from labels ('scrape_target_name' label). ///
/// This directly affects the Prometheus http sd scrape configuration: /// @@ -184,7 +184,7 @@ public sealed class DiscoveryOptions /// ], /// "labels": { /// "__metrics_path__": "${METRICS_PATH}", - /// "scrape_cfg_name": "${METRICS_NAME}", + /// "scrape_target_name": "${METRICS_NAME}", /// ... /// } /// }] @@ -193,4 +193,70 @@ public sealed class DiscoveryOptions [Required] [RegularExpression(@"^[a-zA-Z]{1}[\w-]+[_]{1}$")] public string MetricsPathPortTagPrefix { get; set; } = "METRICS_"; + + /// + /// Semicolon separated string of relabel configurations to apply to the scrape configuration result. + /// This allows using simple syntax to create new labels with existing values, + /// or to combine multiple pre-calculated labels into a single label. + /// + /// + /// The intended use case is to create new labels based on existing labels, so that you don't have to + /// modify your Grafana Dashboards or alerting rules when you change the way you tag your resources. + ///
+ /// Relabel configurations are applied to the scrape configuration before it is returned to the client. + /// You can only operate using labels that are already present in the target. + ///
+ /// Example: + /// Consider the following response of discovery target for configuration without relabeling: + /// + /// { + /// "targets": ["10.200.10.200:8080"], + /// "labels": { + /// "__metrics_path__": "/metrics", + /// "instance": "10.200.10.200", + /// "scrape_target_name": "app", + /// "ecs_cluster": "my-cluster", + /// "ecs_service": "my-service", + /// ... + /// } + /// }, + /// + /// By applying the following relabel configuration: + /// + /// "service_and_cluster={{ecs_cluster}}-{{ecs_service}};" + /// + /// This will create a new label 'service_and_cluster' with the value of 'ecs_cluster' and 'ecs_service' labels combined: + /// + /// { + /// "targets": ["10.200.10.200:8080"], + /// "labels": { + /// "__metrics_path__": "/metrics", + /// "instance": "10.200.10.200", + /// "scrape_target_name": "app", + /// "ecs_cluster": "my-cluster", + /// "ecs_service": "my-service", + /// "service_and_cluster": "my-cluster-my-service", + /// ... + /// } + /// }, + /// + ///
+ /// If your intent is to only 'rename' the existing label, you can use the following syntax: + /// + /// "cluster={{ecs_cluster}};" + /// + /// This will create new label 'cluster' with the value of 'ecs_cluster' label. + ///
+ /// IMPORTANT: + ///
    + ///
  • 1) If amy of the replacement tokens are not found in the source labels, it will not be replaced at all.
    • + ///
  • 2) You can replace existing labels ("ecs_cluster=my-{{ecs_cluster}};").
    • + ///
  • 3) You can have multiple relabel configurations for the same label. They will be applied in order defined ("a=1;a={{a}}-2;" will result in a="1-2").
    • + ///
+ ///
+ /// To have a fall-backs, use 'ExtraPrometheusLabels' option. These have the lowest priority, + /// and are only added to the response if the label is not already present in the target + /// (matching tag is not present in any of the original resources). + /// + public string RelabelConfigurations { get; set; } = string.Empty; } \ No newline at end of file diff --git a/src/Apptality.CloudMapEcsPrometheusDiscovery/Prometheus/PrometheusResponseFactory.cs b/src/Apptality.CloudMapEcsPrometheusDiscovery/Prometheus/PrometheusResponseFactory.cs index 960c4b1..0089d51 100644 --- a/src/Apptality.CloudMapEcsPrometheusDiscovery/Prometheus/PrometheusResponseFactory.cs +++ b/src/Apptality.CloudMapEcsPrometheusDiscovery/Prometheus/PrometheusResponseFactory.cs @@ -36,24 +36,20 @@ public static PrometheusResponse Create(ICollection discoveryTa // Add scrape configuration specific labele, if provided if (!string.IsNullOrWhiteSpace(scrapeConfiguration.Name)) { - staticConfig.Labels.Add("scrape_cfg_name", scrapeConfiguration.Name); + staticConfig.Labels.Add("scrape_target_name", scrapeConfiguration.Name); } - // Add ECS specific labels - staticConfig.Labels.AddLabelWithValue("ecs_cluster", discoveryTarget.EcsCluster, true); - staticConfig.Labels.AddLabelWithValue("ecs_service", discoveryTarget.EcsService, true); - staticConfig.Labels.AddLabelWithValue("ecs_task", discoveryTarget.EcsTaskArn); - staticConfig.Labels.AddLabelWithValue("ecs_task_definition_arn", discoveryTarget.EcsTaskDefinitionArn); - - // Add optional CloudMap labels, when not empty - staticConfig.Labels.AddLabelWithValue("cloudmap_service_name", discoveryTarget.CloudMapServiceName, true); - staticConfig.Labels.AddLabelWithValue("cloudmap_service_instance_id", discoveryTarget.CloudMapServiceInstanceId); - staticConfig.Labels.AddLabelWithValue("cloudmap_service_type", discoveryTarget.ServiceType?.ToString() ?? ""); - - // Now add all the labels from the target + // Stage labels in a dictionary to sort them before adding to the static config + var interimLabels = new Dictionary(); foreach (var discoveryLabel in discoveryTarget.Labels) { - staticConfig.Labels.AddLabelWithValue(discoveryLabel.Name, discoveryLabel.Value); + interimLabels.AddLabelWithValue(discoveryLabel.Name, discoveryLabel.Value); + } + + // Now sort the labels and add them to the static config + foreach (var entry in interimLabels.OrderBy(d=>d.Key.ToLower())) + { + staticConfig.Labels.Add(entry.Key, entry.Value); } response.Add(staticConfig); @@ -69,21 +65,16 @@ public static PrometheusResponse Create(ICollection discoveryTa internal static void AddLabelWithValue( this Dictionary labels, string labelName, - string labelValue, - bool isMetaLabel = false + string labelValue ) { if (string.IsNullOrWhiteSpace(labelValue)) return; var validLabelName = labelName.ToValidPrometheusLabelName(); - if (labels.TryAdd(validLabelName, labelValue) && isMetaLabel) + if (!labels.TryAdd(validLabelName, labelValue)) { - // Add meta-label if it was added successfully - labels.TryAdd($"__meta_{validLabelName}", labelValue); - return; + Log.Debug("Failed to add label {LabelName} with value {LabelValue}", validLabelName, labelValue); } - - Log.Debug("Failed to add label {LabelName} with value {LabelValue}", validLabelName, labelValue); } /// diff --git a/src/Apptality.CloudMapEcsPrometheusDiscovery/appsettings.json b/src/Apptality.CloudMapEcsPrometheusDiscovery/appsettings.json index b764362..edb2bb7 100644 --- a/src/Apptality.CloudMapEcsPrometheusDiscovery/appsettings.json +++ b/src/Apptality.CloudMapEcsPrometheusDiscovery/appsettings.json @@ -99,7 +99,7 @@ // * METRICS_PORT = 9001 (default path "/metrics" will be used, no configuration name will be added) // // If a path is not provided, default path "/metrics" will be used. - // If a name is not provided, the service name will be omitted from labels ('scrape_cfg_name' label). + // If a name is not provided, the service name will be omitted from labels ('scrape_target_name' label). // // This directly affects the Prometheus http sd scrape configuration: // [{ @@ -108,7 +108,7 @@ // ], // "labels": { // "__metrics_path__": "${METRICS_PATH}", - // "scrape_cfg_name": "${METRICS_NAME}", + // "scrape_target_name": "${METRICS_NAME}", // ... // } // }]