Add metrics extraction (#1)

README.md

@@ -1 +1,8 @@
 # analysis-quality-extractor
+
+Basic instructions:
+
+The main task is task.AnalysisQualityGenerator.main, which will extract all information.
+
+You need to have valid AWS credentials (in order to extract the metrics), plus export the following env variables: AWS_PROFILE and AWS_REGION.
+

build.gradle

@@ -3,18 +3,36 @@ plugins {
     id 'application'
 }
 
+sourceCompatibility = 11
+
 group 'com.sonarsource.sonarcloud.extractor'
 version '1.0-SNAPSHOT'
 mainClassName = 'task.AnalysisQualityGenerator'
 
 repositories {
-    mavenCentral()
+    maven {
+        url "https://repox.jfrog.io/repox/sonarsource"
+        // The environment variables ARTIFACTORY_USERNAME and ARTIFACTORY_PASSWORD are used on QA env
+        // On local box, please add artifactoryUsername and artifactoryPassword to ~/.gradle/gradle.properties
+        def artifactoryUsername = System.env.'ARTIFACTORY_USERNAME' ?: (project.hasProperty('artifactoryUsername') ? project.getProperty('artifactoryUsername') : '')
+        def artifactoryPassword = System.env.'ARTIFACTORY_PASSWORD' ?: (project.hasProperty('artifactoryPassword') ? project.getProperty('artifactoryPassword') : '')
+        if (artifactoryUsername && artifactoryPassword) {
+            credentials {
+                username artifactoryUsername
+                password artifactoryPassword
+            }
+        }
+    }
 }
 
 dependencies {
+    implementation platform('com.amazonaws:aws-java-sdk-bom:1.11.1000')
+    implementation 'com.amazonaws:aws-java-sdk-logs'
+    implementation 'com.amazonaws:aws-java-sdk-sts'
+
     implementation 'com.google.code.gson:gson:2.8.9'
-    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
-    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1'
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2'
+    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.2'
 }
 
 test {

src/main/java/extractor/ApiConnector.java

@@ -19,14 +19,14 @@
 import model.ComponentSearchProjects;
 import model.ComponentTree;
 import model.Issue;
-import model.measure.ComponentMeasure;
-import model.measure.Measure;
 import model.NavigationComponent;
 import model.PluginsInstalled;
 import model.ProjectBranch;
 import model.ProjectBranches;
 import model.QualityProfile;
 import model.Rule;
+import model.measure.ComponentMeasure;
+import model.measure.Measure;
 
 import static java.util.logging.Level.WARNING;
 
@@ -134,7 +134,7 @@ public Optional<NavigationComponent> getNavigationComponent(String projectKey) {
   }
 
   public List<Component> getOrganizationProjects(String organization) {
-    URI uri = createURI(baseUrl, API_COMPONENTS_SEARCH_PROJECTS, "ps=500&organization=" + organization);
+    URI uri = createURI(baseUrl, API_COMPONENTS_SEARCH_PROJECTS, "ps=500&f=analysisDate&organization=" + organization);
     return GSON.fromJson(doHttpRequest(uri), ComponentSearchProjects.class).getComponents();
   }
 

src/main/java/extractor/MetricsConnector.java

@@ -0,0 +1,84 @@
+package extractor;
+
+import com.amazonaws.services.logs.AWSLogs;
+import com.amazonaws.services.logs.model.FilterLogEventsRequest;
+import com.amazonaws.services.logs.model.FilteredLogEvent;
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.logging.Logger;
+import model.ProjectAnalysisMetrics;
+import model.logs.CloudWatchEntry;
+import model.logs.CloudWatchMetricDefinition;
+
+public class MetricsConnector {
+
+  private static final Logger LOGGER = Logger.getLogger(MetricsConnector.class.getName());
+  private static final Gson GSON = new Gson();
+
+  private static final String LOG_GROUP = "/SonarCloud/Autoscan/WorkerForJava/Fargate/Task/squad-1";
+  private static final int START_TIME_MARGIN_HOUR = -3;
+  private static final int END_TIME_MARGIN_HOUR = 3;
+
+  private static final List<String> TARGET_METRICS = Arrays.asList(
+    "JavaAnalysisFinishedCount",
+    "WorkerForJavaCloneDuration",
+    "WorkerForJavaTaskDuration",
+    "DownloadedArtifactsPercentage",
+    "ParsedArtifactsCount",
+    "ConstructDependencyGraphDuration",
+    "DownloadDependenciesDuration",
+    "ResolveDependenciesDuration",
+    "WorkerForJavaQueueLatency"
+  );
+
+  private final AWSLogs logsClient;
+  private ProjectAnalysisMetrics metrics;
+
+  public MetricsConnector(AWSLogs logsClient) {
+    this.logsClient = logsClient;
+  }
+
+  public ProjectAnalysisMetrics getMetrics(String projectKey, String branch, Date analysisDate) {
+    metrics = new ProjectAnalysisMetrics();
+    FilterLogEventsRequest filterRequest = new FilterLogEventsRequest()
+      .withStartTime(getTimeWithMargin(analysisDate, START_TIME_MARGIN_HOUR))
+      .withEndTime(getTimeWithMargin(analysisDate, END_TIME_MARGIN_HOUR))
+      .withFilterPattern("{($.log_type = \"METRICS\") && ($.project=\"" + projectKey + "\") && ($.branch=\"" + branch + "\")}")
+      .withLogGroupName(LOG_GROUP);
+
+    logsClient.filterLogEvents(filterRequest).getEvents().forEach(this::extractMetricsFromEvent);
+    return metrics;
+  }
+
+  private long getTimeWithMargin(Date current, int amount) {
+    Calendar calendar = Calendar.getInstance();
+    calendar.setTime(current);
+    calendar.add(Calendar.HOUR, amount);
+    return calendar.getTimeInMillis();
+  }
+
+  private void extractMetricsFromEvent(FilteredLogEvent event) {
+    CloudWatchEntry cw = GSON.fromJson(event.getMessage(), CloudWatchEntry.class);
+    cw.getCloudWatchMetrics().getMetrics().forEach(cwMetrics ->
+      cwMetrics.getMetrics().stream()
+        .filter(this::isTargetMetric)
+        .forEach(metricDefinition -> populateMetric(event, metricDefinition))
+    );
+  }
+
+  private boolean isTargetMetric(CloudWatchMetricDefinition metricDefinition) {
+    return TARGET_METRICS.contains(metricDefinition.getName());
+  }
+
+  private void populateMetric(FilteredLogEvent event, CloudWatchMetricDefinition metricDefinition) {
+    JsonObject json = GSON.fromJson(event.getMessage(), JsonObject.class);
+    ProjectAnalysisMetrics.PopulateMetricCommand.populateValue(
+      metrics, metricDefinition.getName(), json.get(metricDefinition.getName()).getAsString());
+    LOGGER.info(metricDefinition.getName() + ": " + json.get(metricDefinition.getName()).getAsString());
+  }
+
+}

src/main/java/extractor/ProjectAnalysis.java

@@ -20,9 +20,11 @@ public class ProjectAnalysis {
   private static final Logger LOGGER = Logger.getLogger(ProjectAnalysis.class.getName());
 
   private final ApiConnector apiConnector;
+  private final MetricsConnector metricsConnector;
 
-  public ProjectAnalysis(ApiConnector apiConnector) {
+  public ProjectAnalysis(ApiConnector apiConnector, MetricsConnector metricsConnector) {
     this.apiConnector = apiConnector;
+    this.metricsConnector = metricsConnector;
   }
 
   public ProjectAnalysisResult extractResult(String projectKey, String branch) {
@@ -138,6 +140,16 @@ public ProjectAnalysisQuality processDifferences(ProjectAnalysisQuality pq) {
     ));
   }
 
+  public ProjectAnalysisQuality extractMetrics(ProjectAnalysisQuality pq) {
+    pq.setTargetComponentAnalysisMetrics(
+      metricsConnector.getMetrics(
+        pq.getTargetComponent().getKey(),
+        pq.getTargetComponentDefaultBranch(),
+        pq.getBaseComponent().getAnalysisDate())
+    );
+    return pq;
+  }
+
   private Optional<Component> findMatchingComponent(Component base, List<Component> availableTargets) {
     return availableTargets.stream()
       .filter(component -> base.getName().equalsIgnoreCase(component.getName()))

src/main/java/model/Component.java

@@ -2,13 +2,18 @@
 
 import java.util.Objects;
 
+import java.util.Date;
+
 public class Component {
 
   private String organization;
   private String key;
   private String name;
   private String qualifier;
   private String language;
+  private Date analysisDate;
+  private String eligibilityStatus;
+  private boolean eligible;
   private String comparableKey;
 
   public String getOrganization() {
@@ -58,6 +63,30 @@ public void setLanguage(String language) {
     this.language = language;
   }
 
+  public Date getAnalysisDate() {
+    return analysisDate;
+  }
+
+  public void setAnalysisDate(Date analysisDate) {
+    this.analysisDate = analysisDate;
+  }
+
+  public String getEligibilityStatus() {
+    return eligibilityStatus;
+  }
+
+  public void setEligibilityStatus(String eligibilityStatus) {
+    this.eligibilityStatus = eligibilityStatus;
+  }
+
+  public boolean isEligible() {
+    return eligible;
+  }
+
+  public void setEligible(boolean eligible) {
+    this.eligible = eligible;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;

src/main/java/model/ProjectAnalysisMetrics.java

@@ -0,0 +1,133 @@
+package model;
+
+import java.util.Map;
+
+import static java.lang.Float.parseFloat;
+import static java.lang.Integer.parseInt;
+
+public class ProjectAnalysisMetrics {
+
+  private int JavaAnalysisFinishedCount;
+  private int WorkerForJavaCloneDuration;
+  private int WorkerForJavaTaskDuration;
+  private float DownloadedArtifactsPercentage;
+  private int ParsedArtifactsCount;
+  private int ConstructDependencyGraphDuration;
+  private int DownloadDependenciesDuration;
+  private int ResolveDependenciesDuration;
+  private int WorkerForJavaQueueLatency;
+
+  public int getJavaAnalysisFinishedCount() {
+    return JavaAnalysisFinishedCount;
+  }
+
+  public void setJavaAnalysisFinishedCount(int javaAnalysisFinishedCount) {
+    JavaAnalysisFinishedCount = javaAnalysisFinishedCount;
+  }
+
+  public int getWorkerForJavaCloneDuration() {
+    return WorkerForJavaCloneDuration;
+  }
+
+  public void setWorkerForJavaCloneDuration(int workerForJavaCloneDuration) {
+    WorkerForJavaCloneDuration = workerForJavaCloneDuration;
+  }
+
+  public int getWorkerForJavaTaskDuration() {
+    return WorkerForJavaTaskDuration;
+  }
+
+  public void setWorkerForJavaTaskDuration(int workerForJavaTaskDuration) {
+    WorkerForJavaTaskDuration = workerForJavaTaskDuration;
+  }
+
+  public float getDownloadedArtifactsPercentage() {
+    return DownloadedArtifactsPercentage;
+  }
+
+  public void setDownloadedArtifactsPercentage(float downloadedArtifactsPercentage) {
+    DownloadedArtifactsPercentage = downloadedArtifactsPercentage;
+  }
+
+  public int getParsedArtifactsCount() {
+    return ParsedArtifactsCount;
+  }
+
+  public void setParsedArtifactsCount(int parsedArtifactsCount) {
+    ParsedArtifactsCount = parsedArtifactsCount;
+  }
+
+  public int getConstructDependencyGraphDuration() {
+    return ConstructDependencyGraphDuration;
+  }
+
+  public void setConstructDependencyGraphDuration(int constructDependencyGraphDuration) {
+    ConstructDependencyGraphDuration = constructDependencyGraphDuration;
+  }
+
+  public int getDownloadDependenciesDuration() {
+    return DownloadDependenciesDuration;
+  }
+
+  public void setDownloadDependenciesDuration(int downloadDependenciesDuration) {
+    DownloadDependenciesDuration = downloadDependenciesDuration;
+  }
+
+  public int getResolveDependenciesDuration() {
+    return ResolveDependenciesDuration;
+  }
+
+  public void setResolveDependenciesDuration(int resolveDependenciesDuration) {
+    ResolveDependenciesDuration = resolveDependenciesDuration;
+  }
+
+  public int getWorkerForJavaQueueLatency() {
+    return WorkerForJavaQueueLatency;
+  }
+
+  public void setWorkerForJavaQueueLatency(int workerForJavaQueueLatency) {
+    WorkerForJavaQueueLatency = workerForJavaQueueLatency;
+  }
+
+  private interface PopulateCommand {
+    void populate(ProjectAnalysisMetrics metrics, String value);
+  }
+
+  public static class PopulateMetricCommand {
+
+    private PopulateMetricCommand() {
+      throw new IllegalStateException("Utility class");
+    }
+
+    private static Map<String, PopulateCommand> populateCommandProvider() {
+      return Map.of(
+        "JavaAnalysisFinishedCount",
+        (metrics, value) -> metrics.setJavaAnalysisFinishedCount(parseInt(value)),
+        "WorkerForJavaCloneDuration",
+        (metrics, value) -> metrics.setWorkerForJavaCloneDuration(parseInt(value)),
+        "WorkerForJavaTaskDuration",
+        (metrics, value) -> metrics.setWorkerForJavaTaskDuration(parseInt(value)),
+        "DownloadedArtifactsPercentage",
+        (metrics, value) -> metrics.setDownloadedArtifactsPercentage(parseFloat(value)),
+        "ParsedArtifactsCount",
+        (metrics, value) -> metrics.setParsedArtifactsCount(parseInt(value)),
+        "ConstructDependencyGraphDuration",
+        (metrics, value) -> metrics.setConstructDependencyGraphDuration(parseInt(value)),
+        "DownloadDependenciesDuration",
+        (metrics, value) -> metrics.setDownloadDependenciesDuration(parseInt(value)),
+        "ResolveDependenciesDuration",
+        (metrics, value) -> metrics.setResolveDependenciesDuration(parseInt(value)),
+        "WorkerForJavaQueueLatency",
+        (metrics, value) -> metrics.setWorkerForJavaQueueLatency(parseInt(value))
+      );
+    }
+
+    public static void populateValue(ProjectAnalysisMetrics metrics, String metric, String value) {
+      PopulateCommand command = populateCommandProvider().get(metric);
+      command.populate(metrics, value);
+    }
+
+  }
+
+
+}