SONAR-18872 improve total ncloc computation

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStep.java

@@ -0,0 +1,51 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.step;
+
+import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
+import org.sonar.ce.task.step.ComputationStep;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+
+public class ProjectNclocComputationStep implements ComputationStep {
+
+  private final AnalysisMetadataHolder analysisMetadataHolder;
+  private final DbClient dbClient;
+
+  public ProjectNclocComputationStep(AnalysisMetadataHolder analysisMetadataHolder, DbClient dbClient) {
+    this.analysisMetadataHolder = analysisMetadataHolder;
+    this.dbClient = dbClient;
+  }
+
+  @Override
+  public void execute(Context context) {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      String projectUuid = analysisMetadataHolder.getProject().getUuid();
+      long maxncloc = dbClient.liveMeasureDao().sumNclocOfBiggestBranchForProject(dbSession, projectUuid);
+      dbClient.projectDao().updateNcloc(dbSession, projectUuid, maxncloc);
+      dbSession.commit();
+    }
+  }
+
+  @Override
+  public String getDescription() {
+    return "Compute total Project ncloc";
+  }
+}

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java

@@ -110,6 +110,7 @@ public class ReportComputationSteps extends AbstractComputationSteps {
     PurgeDatastoresStep.class,
     IndexAnalysisStep.class,
     UpdateNeedIssueSyncStep.class,
+    ProjectNclocComputationStep.class,
     PersistPushEventsStep.class,
 
     // notifications are sent at the end, so that webapp displays up-to-date information

server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStepIT.java

@@ -0,0 +1,65 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.step;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolderRule;
+import org.sonar.ce.task.step.TestComputationStepContext;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbTester;
+import org.sonar.db.component.BranchType;
+import org.sonar.db.component.ComponentDto;
+import org.sonar.db.metric.MetricDto;
+import org.sonar.server.project.Project;
+
+import static java.util.Collections.emptyList;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.sonar.api.measures.Metric.ValueType.INT;
+
+public class ProjectNclocComputationStepIT {
+  @Rule
+  public DbTester db = DbTester.create();
+  private final DbClient dbClient = db.getDbClient();
+
+  @Rule
+  public AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule();
+
+  private final ProjectNclocComputationStep step = new ProjectNclocComputationStep(analysisMetadataHolder, dbClient);
+
+  @Test
+  public void test_computing_branch_ncloc() {
+    MetricDto ncloc = db.measures().insertMetric(m -> m.setKey("ncloc").setValueType(INT.toString()));
+    ComponentDto project = db.components().insertPublicProject();
+    ComponentDto branch1 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH));
+    db.measures().insertLiveMeasure(branch1, ncloc, m -> m.setValue(200d));
+    ComponentDto branch2 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH));
+    db.measures().insertLiveMeasure(branch2, ncloc, m -> m.setValue(10d));
+    analysisMetadataHolder.setProject(new Project(project.uuid(), project.getKey(), project.name(), project.description(), emptyList()));
+    step.execute(TestComputationStepContext.TestStatistics::new);
+
+    assertThat(dbClient.projectDao().getNclocSum(db.getSession())).isEqualTo(200L);
+  }
+
+  @Test
+  public void description_is_not_missing() {
+    assertThat(step.getDescription()).isNotBlank();
+  }
+}

server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java

@@ -94,14 +94,8 @@ public void selectTreeByQuery(DbSession dbSession, ComponentDto baseComponent, M
     mapper(dbSession).selectTreeByQuery(query, baseComponent.uuid(), query.getUuidPath(baseComponent), resultHandler);
   }
 
-  /**
-   * Example:
-   * If Main Branch = 0 LOCs (provisioned but never analyzed) and the "largest branch" is 120 LOCs, I'm expecting to consider the value 120.
-   * If Main Branch = 100 LOCs and the "largest branch" is 120 LOCs, I'm expecting to consider the value 120.
-   * If Main Branch = 100 LOCs and the "largest branch" is 80 LOCs, I'm expecting to consider the value 100.
-   */
-  public long sumNclocOfBiggestBranch(DbSession dbSession, SumNclocDbQuery dbQuery) {
-    Long ncloc = mapper(dbSession).sumNclocOfBiggestBranch(NCLOC_KEY, dbQuery.getOnlyPrivateProjects(), dbQuery.getProjectUuidToExclude());
+  public long sumNclocOfBiggestBranchForProject(DbSession dbSession, String projectUuid){
+    Long ncloc = mapper(dbSession).sumNclocOfBiggestBranchForProject(projectUuid, NCLOC_KEY);
     return ncloc == null ? 0L : ncloc;
   }
 

server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java

@@ -21,7 +21,7 @@
 
 import java.util.Collection;
 import java.util.List;
-import javax.annotation.Nullable;
+import javax.annotation.CheckForNull;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.session.ResultHandler;
 
@@ -57,10 +57,8 @@ void selectTreeByQuery(
     @Param("baseUuidPath") String baseUuidPath,
     ResultHandler<LiveMeasureDto> resultHandler);
 
-  Long sumNclocOfBiggestBranch(
-    @Param("ncloc") String nclocKey,
-    @Param("private") Boolean privateProject,
-    @Nullable @Param("projectUuidToExclude") String projectUuidToExclude);
+  @CheckForNull
+  Long sumNclocOfBiggestBranchForProject(@Param("projectUuid") String projectUuid, @Param("ncloc") String nclocKey);
 
   List<LargestBranchNclocDto> getLargestBranchNclocPerProject();
 

server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectDao.java

@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import javax.annotation.Nullable;
 import org.sonar.api.utils.System2;
 import org.sonar.db.Dao;
 import org.sonar.db.DbSession;
@@ -126,4 +127,16 @@ public Set<String> selectProjectUuidsAssociatedToDefaultQualityProfileByLanguage
     Set<String> languageFilters = Set.of(language + "=%", "%;" + language + "=%");
     return mapper(session).selectProjectUuidsAssociatedToDefaultQualityProfileByLanguage(languageFilters);
   }
+
+  public void updateNcloc(DbSession dbSession, String projectUuid, long ncloc) {
+    mapper(dbSession).updateNcloc(projectUuid, ncloc);
+  }
+
+  public long getNclocSum(DbSession dbSession) {
+    return getNclocSum(dbSession, null);
+  }
+
+  public long getNclocSum(DbSession dbSession, @Nullable String projectUuidToExclude) {
+    return Optional.ofNullable(mapper(dbSession).getNclocSum(projectUuidToExclude)).orElse(0L);
+  }
 }

server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectMapper.java

@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Set;
 import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
 import org.apache.ibatis.annotations.Param;
 
 public interface ProjectMapper {
@@ -62,4 +63,10 @@ public interface ProjectMapper {
   List<String> selectAllProjectUuids();
 
   Set<String> selectProjectUuidsAssociatedToDefaultQualityProfileByLanguage(@Param("languageFilters") Set<String> languageFilters);
+
+  void updateNcloc(@Param("projectUuid") String projectUuid, @Param("ncloc") long ncloc);
+
+  @CheckForNull
+  Long getNclocSum(@Nullable @Param("projectUuidToExclude") String projectUuidToExclude);
+
 }

server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml

@@ -61,21 +61,16 @@
     and lm.component_uuid = #{componentUuid, jdbcType=VARCHAR}
   </select>
 
-  <select id="sumNclocOfBiggestBranch" parameterType="map" resultType="long">
-    select sum(sumncloc.maxncloc) from (
+  <select id="sumNclocOfBiggestBranchForProject" parameterType="map" resultType="long">
+    select sumncloc.maxncloc from (
       select b.project_uuid as projectUuid, max(lm.value) as maxncloc
       from live_measures lm
       inner join metrics m on m.uuid = lm.metric_uuid
       inner join project_branches b on b.uuid = lm.component_uuid
       inner join projects p on p.uuid = b.project_uuid and p.qualifier = 'TRK'
       <where>
         m.name = #{ncloc, jdbcType=VARCHAR}
-        <if test="private">
-          and p.private=${_true}
-        </if>
-        <if test="projectUuidToExclude != null">
-          and b.project_uuid &lt;&gt; #{projectUuidToExclude,jdbcType=VARCHAR}
-        </if>
+        and b.project_uuid = #{projectUuid,jdbcType=VARCHAR}
       </where>
       group by b.project_uuid
     ) sumncloc

server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectMapper.xml

@@ -179,4 +179,19 @@
     uuid = #{uuid,jdbcType=VARCHAR}
   </update>
 
+
+  <update id="updateNcloc">
+    update projects set
+    ncloc = #{ncloc,jdbcType=BIGINT}
+    where
+    uuid = #{projectUuid,jdbcType=VARCHAR}
+  </update>
+
+  <select id="getNclocSum" parameterType="string" resultType="long">
+    select sum(ncloc) from projects where qualifier = 'TRK'
+    <if test="projectUuidToExclude != null">
+      and uuid &lt;&gt; #{projectUuidToExclude,jdbcType=VARCHAR}
+    </if>
+  </select>
+
 </mapper>

server/sonar-db-dao/src/schema/schema-sq.ddl

@@ -723,7 +723,8 @@ CREATE TABLE "PROJECTS"(
     "PRIVATE" BOOLEAN NOT NULL,
     "TAGS" CHARACTER VARYING(500),
     "CREATED_AT" BIGINT,
-    "UPDATED_AT" BIGINT NOT NULL
+    "UPDATED_AT" BIGINT NOT NULL,
+    "NCLOC" BIGINT
 );
 ALTER TABLE "PROJECTS" ADD CONSTRAINT "PK_NEW_PROJECTS" PRIMARY KEY("UUID");
 CREATE UNIQUE INDEX "UNIQ_PROJECTS_KEE" ON "PROJECTS"("KEE" NULLS FIRST);

server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java

@@ -365,12 +365,9 @@ public void countNcloc() {
     db.measures().insertLiveMeasure(projectWithLinesButNoLoc, lines, m -> m.setValue(365d));
     db.measures().insertLiveMeasure(projectWithLinesButNoLoc, ncloc, m -> m.setValue(0d));
 
-    SumNclocDbQuery query = SumNclocDbQuery.builder()
-      .setOnlyPrivateProjects(false)
-      .build();
-    long result = underTest.sumNclocOfBiggestBranch(db.getSession(), query);
-
-    assertThat(result).isEqualTo(10L + 200L);
+    assertThat(underTest.sumNclocOfBiggestBranchForProject(db.getSession(), simpleProject.uuid())).isEqualTo(10L);
+    assertThat(underTest.sumNclocOfBiggestBranchForProject(db.getSession(), projectWithBiggerBranch.uuid())).isEqualTo(200L);
+    assertThat(underTest.sumNclocOfBiggestBranchForProject(db.getSession(), projectWithLinesButNoLoc.uuid())).isZero();
   }
 
   @Test
@@ -410,40 +407,11 @@ public void get_loc_language_distribution() {
   public void countNcloc_empty() {
     db.measures().insertMetric(m -> m.setKey("ncloc").setValueType(INT.toString()));
     db.measures().insertMetric(m -> m.setKey("lines").setValueType(INT.toString()));
-    SumNclocDbQuery query = SumNclocDbQuery.builder()
-      .setOnlyPrivateProjects(false)
-      .build();
-    long result = underTest.sumNclocOfBiggestBranch(db.getSession(), query);
+    long result = underTest.sumNclocOfBiggestBranchForProject(db.getSession(), "non-existing-project-uuid");
 
     assertThat(result).isZero();
   }
 
-  @Test
-  public void countNcloc_and_exclude_project() {
-    MetricDto ncloc = db.measures().insertMetric(m -> m.setKey("ncloc").setValueType(INT.toString()));
-
-    ComponentDto simpleProject = db.components().insertPublicProject();
-    db.measures().insertLiveMeasure(simpleProject, ncloc, m -> m.setValue(10d));
-
-    ComponentDto projectWithBiggerBranch = db.components().insertPublicProject();
-    ComponentDto bigBranch = db.components().insertProjectBranch(projectWithBiggerBranch, b -> b.setBranchType(BranchType.BRANCH));
-    db.measures().insertLiveMeasure(projectWithBiggerBranch, ncloc, m -> m.setValue(100d));
-    db.measures().insertLiveMeasure(bigBranch, ncloc, m -> m.setValue(200d));
-
-    ComponentDto projectToExclude = db.components().insertPublicProject();
-    ComponentDto projectToExcludeBranch = db.components().insertProjectBranch(projectToExclude, b -> b.setBranchType(BranchType.BRANCH));
-    db.measures().insertLiveMeasure(projectToExclude, ncloc, m -> m.setValue(300d));
-    db.measures().insertLiveMeasure(projectToExcludeBranch, ncloc, m -> m.setValue(400d));
-
-    SumNclocDbQuery query = SumNclocDbQuery.builder()
-      .setProjectUuidToExclude(projectToExclude.uuid())
-      .setOnlyPrivateProjects(false)
-      .build();
-    long result = underTest.sumNclocOfBiggestBranch(db.getSession(), query);
-
-    assertThat(result).isEqualTo(10L + 200L);
-  }
-
   @Test
   public void insert_data() {
     byte[] data = "text_value".getBytes(StandardCharsets.UTF_8);

server/sonar-db-dao/src/test/java/org/sonar/db/project/ProjectDaoTest.java

@@ -31,6 +31,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import javax.annotation.Nullable;
+import org.assertj.core.api.Assertions;
 import org.assertj.core.groups.Tuple;
 import org.junit.Rule;
 import org.junit.Test;
@@ -279,6 +280,31 @@ public void select_project_uuids_associated_to_default_quality_profile_for_speci
       .containsExactlyInAnyOrderElementsOf(extractComponentUuids(projects));
   }
 
+  @Test
+  public void update_ncloc_should_update_project() {
+    ComponentDto project = db.components().insertPublicProject();
+
+    projectDao.updateNcloc(db.getSession(), project.uuid(), 10L);
+
+    Assertions.assertThat(projectDao.getNclocSum(db.getSession())).isEqualTo(10L);
+  }
+
+  @Test
+  public void getNcloc_sum_compute_correctly_sum_of_projects() {
+    projectDao.updateNcloc(db.getSession(), db.components().insertPublicProject().uuid(), 1L);
+    projectDao.updateNcloc(db.getSession(), db.components().insertPublicProject().uuid(), 20L);
+    projectDao.updateNcloc(db.getSession(), db.components().insertPublicProject().uuid(), 100L);
+    Assertions.assertThat(projectDao.getNclocSum(db.getSession())).isEqualTo(121L);
+  }
+
+  @Test
+  public void getNcloc_sum_compute_correctly_sum_of_projects_while_excluding_project() {
+    projectDao.updateNcloc(db.getSession(), db.components().insertPublicProject().uuid(), 1L);
+    projectDao.updateNcloc(db.getSession(), db.components().insertPublicProject().uuid(), 20L);
+    ComponentDto project3 = db.components().insertPublicProject();
+    projectDao.updateNcloc(db.getSession(), project3.uuid(), 100L);
+    Assertions.assertThat(projectDao.getNclocSum(db.getSession(), project3.uuid())).isEqualTo(21L);
+  }
   @Test
   public void selectAllProjectUuids_shouldOnlyReturnProjectWithTRKQualifier() {
     ComponentDto application = db.components().insertPrivateApplication();