Merge branch 'Naksha_maintenance' into MCPODS-6260_dev_deployment

.github/workflows/test.yaml

@@ -0,0 +1,68 @@
+name: Run Naksha tests
+
+on:
+  pull_request:
+    types:
+      - opened       # run whenever PR is opened
+      - synchronize  # run whenever PR is updated
+      - reopened     # run whenever PR is reopoened
+  workflow_dispatch: # manual run
+
+
+permissions:
+  checks: write         # for junit reporting
+  pull-requests: write  # for jacoco PR comments
+
+env:
+  MIN_COVERAGE_OVERALL: 25
+  MIN_COVERAGE_CHANGED_FILES: 65
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgis/postgis # Postgres with PostGIS extension
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+          POSTGRES_DB: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+        with:
+          gradle-version: 8.2
+      - name: Execute tests & verify overall coverage
+        run: gradle test jacocoTestReport jacocoTestCoverageVerification
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: success() || failure() # always run even if the previous step fails
+        with:
+          report_paths: '**/build/test-results/test/TEST-*.xml'
+      - name: Publish code coverage report as PR comment
+        id: jacoco
+        uses: madrapps/jacoco-report@v1.6.1
+        with:
+          paths: '**/build/reports/jacoco/test/jacocoTestReport.xml'
+          token: ${{ secrets.GITHUB_TOKEN }}
+          min-coverage-overall: $MIN_COVERAGE_OVERALL
+          min-coverage-changed-files: $MIN_COVERAGE_CHANGED_FILES
+          title: Code Coverage
+      - name: Fail when coverage of changed files is too low
+        run: |
+          CHANGED_FILES_FAILED=$(echo '${{ steps.jacoco.outputs.coverage-changed-files }} < ${{ env.MIN_COVERAGE_CHANGED_FILES }}' | bc)
+          [[ $CHANGED_FILES_FAILED -ne 0 ]] && echo 'Changed files coverage ${{ steps.jacoco.outputs.coverage-changed-files }}% is smaller than required ${{ env.MIN_COVERAGE_CHANGED_FILES }}%'
+          [[ $CHANGED_FILES_FAILED -ne 0 ]] && exit 1 || exit 0

README.md

@@ -190,6 +190,25 @@ The service will respond with the inserted geo features:
 }
 ```
 
+# Testing locally
+
+To run tests locally run Gradle `test` task:
+```bash
+./gradlew test
+```
+
+Code coverage report is generated with use of [jacoco](https://www.jacoco.org/)
+To generate coverage use Gradle task `jacocoTestReport`:
+```bash
+./gradlew test jacocoTestReport
+```
+Outputs for each subproject will be stored in `/[module]/build/reports/jacoco/test/html/index.html`
+
+To validate test coverage, run `jacocoTestCoverageVerification` Gradle task:
+```bash
+./gradlew test jacocoTestReport jacocoTestCoverageVerification
+```
+
 # Acknowledgements
 
 XYZ Hub uses:

build.gradle.kts

@@ -1,8 +1,10 @@
 @file:Suppress("PropertyName")
 
+import org.gradle.api.tasks.testing.logging.TestExceptionFormat
 import java.net.URI
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
+import org.gradle.kotlin.dsl.KotlinClosure2
 
 repositories {
     maven {
@@ -21,22 +23,25 @@ plugins {
     id("com.github.johnrengelman.shadow") version "8.1.1"
     // Don't apply for all projects, we individually only apply where Kotlin is used.
     kotlin("jvm") version "1.8.21" apply false
+    // overall code coverage
+    jacoco
 }
 
 group = "com.here.naksha"
 version = rootProject.properties["version"] as String
 
 val jetbrains_annotations = "org.jetbrains:annotations:24.0.1"
 
-val vertx_core = "io.vertx:vertx-core:4.4.4"
-val vertx_config = "io.vertx:vertx-config:4.4.4"
-val vertx_auth_jwt = "io.vertx:vertx-auth-jwt:4.4.4"
-val vertx_redis_client = "io.vertx:vertx-redis-client:4.4.4"
-val vertx_jdbc_client = "io.vertx:vertx-jdbc-client:4.4.4"
-val vertx_web = "io.vertx:vertx-web:4.4.4"
-val vertx_web_openapi = "io.vertx:vertx-web-openapi:4.4.4"
-val vertx_web_client = "io.vertx:vertx-web-client:4.4.4"
-val vertx_web_templ = "io.vertx:vertx-web-templ-handlebars:4.4.4"
+val vertx_version = "4.5.0"
+val vertx_core = "io.vertx:vertx-core:$vertx_version"
+val vertx_config = "io.vertx:vertx-config:$vertx_version"
+val vertx_auth_jwt = "io.vertx:vertx-auth-jwt:$vertx_version"
+val vertx_redis_client = "io.vertx:vertx-redis-client:$vertx_version"
+val vertx_jdbc_client = "io.vertx:vertx-jdbc-client:$vertx_version"
+val vertx_web = "io.vertx:vertx-web:$vertx_version"
+val vertx_web_openapi = "io.vertx:vertx-web-openapi:$vertx_version"
+val vertx_web_client = "io.vertx:vertx-web-client:$vertx_version"
+val vertx_web_templ = "io.vertx:vertx-web-templ-handlebars:$vertx_version"
 
 val netty_transport_native_kqueue = "io.netty:netty-transport-native-kqueue:4.1.90.Final"
 val netty_transport_native_epoll = "io.netty:netty-transport-native-epoll:4.1.90.Final"
@@ -102,13 +107,21 @@ val mockito = "org.mockito:mockito-core:3.12.4"
 
 val flipkart_zjsonpatch = "com.flipkart.zjsonpatch:zjsonpatch:0.4.13"
 val json_assert = "org.skyscreamer:jsonassert:1.5.1"
+val resillience4j_retry = "io.github.resilience4j:resilience4j-retry:2.0.0"
 
 val otel = "io.opentelemetry:opentelemetry-api:1.28.0"
 
 val mavenUrl = rootProject.properties["mavenUrl"] as String
 val mavenUser = rootProject.properties["mavenUser"] as String
 val mavenPassword = rootProject.properties["mavenPassword"] as String
 
+/*
+    Overall coverage of subproject - it might be different for different subprojects
+    Configurable per project - see `setOverallCoverage`
+ */
+val minOverallCoverageKey: String = "minOverallCoverage"
+val defaultOverallMinCoverage: Double = 0.8 // Don't decrease me!
+
 /*
 
     IMPORTANT: api vs implementation
@@ -130,6 +143,7 @@ subprojects {
     apply(plugin = "java")
     apply(plugin = "com.diffplug.spotless")
     apply(plugin = "java-library")
+    apply(plugin = "jacoco")
 
     repositories {
         maven(uri("https://repo.osgeo.org/repository/release/"))
@@ -176,7 +190,17 @@ subprojects {
         test {
             maxHeapSize = "4g"
             useJUnitPlatform()
-            testLogging.showStandardStreams = true
+            testLogging {
+                showStandardStreams = true
+                exceptionFormat = TestExceptionFormat.FULL
+                events("standardOut", "started", "passed", "skipped", "failed")
+            }
+            afterTest(KotlinClosure2(
+                    { descriptor: TestDescriptor, result: TestResult ->
+                        val totalTime = result.endTime - result.startTime
+                        println("Total time of $descriptor.name was $totalTime")
+                    }
+            ))
         }
 
         compileJava {
@@ -191,6 +215,24 @@ subprojects {
                 addStringOption("Xmaxwarns", "1")
             }
         }
+
+        jacocoTestReport {
+            dependsOn(test)
+            reports {
+                xml.required = true
+            }
+        }
+
+        jacocoTestCoverageVerification {
+            dependsOn(jacocoTestReport)
+            violationRules {
+                rule {
+                    limit {
+                        minimum = getOverallCoverage().toBigDecimal()
+                    }
+                }
+            }
+        }
     }
 
     java {
@@ -253,6 +295,7 @@ project(":here-naksha-lib-core") {
         implementation(vividsolutions_jts_core)
         implementation(google_flatbuffers)
     }
+    setOverallCoverage(0.3) // only increasing allowed!
 }
 
 project(":here-naksha-lib-heapcache") {
@@ -266,6 +309,7 @@ project(":here-naksha-lib-heapcache") {
         testImplementation(mockito)
         implementation(vividsolutions_jts_core)
     }
+    setOverallCoverage(0.5) // only increasing allowed!
 }
 
 project(":here-naksha-lib-psql") {
@@ -286,6 +330,7 @@ project(":here-naksha-lib-psql") {
         testImplementation(mockito)
         testImplementation(spatial4j)
     }
+    setOverallCoverage(0.0) // only increasing allowed!
 }
 
 /*
@@ -294,6 +339,7 @@ project(":here-naksha-lib-extension") {
     dependencies {
         api(project(":here-naksha-lib-core"))
     }
+    setOverallCoverage(0.4) // only increasing allowed!
 }
 */
 
@@ -307,6 +353,7 @@ project(":here-naksha-handler-activitylog") {
         implementation(flipkart_zjsonpatch)
         testImplementation(jayway_jsonpath)
     }
+    setOverallCoverage(0.4) // only increasing allowed!
 }
 */
 
@@ -392,6 +439,7 @@ project(":here-naksha-lib-handlers") {
             testImplementation(json_assert)
             testImplementation(mockito)
         }
+        setOverallCoverage(0.3) // only increasing allowed!
     }
 //} catch (ignore: UnknownProjectException) {
 //}
@@ -420,7 +468,9 @@ project(":here-naksha-lib-handlers") {
             implementation(vertx_web_openapi)
 
             testImplementation(json_assert)
+            testImplementation(resillience4j_retry)
         }
+        setOverallCoverage(0.25) // only increasing allowed!
     }
 //} catch (ignore: UnknownProjectException) {
 //}
@@ -478,3 +528,16 @@ rootProject.tasks.shadowJar {
 rootProject.tasks.register("printAppVersion") {
     println(rootProject.version)
 }
+
+fun Project.setOverallCoverage(minOverallCoverage: Double) {
+    ext.set(minOverallCoverageKey, minOverallCoverage)
+}
+
+fun Project.getOverallCoverage(): Double {
+    return if (ext.has(minOverallCoverageKey)) {
+        ext.get(minOverallCoverageKey) as? Double
+                ?: throw IllegalStateException("Property '$minOverallCoverageKey' is expected to be Double")
+    } else {
+        defaultOverallMinCoverage
+    }
+}

here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/NakshaHttpVerticle.java

@@ -42,7 +42,13 @@
 
 import com.here.naksha.app.service.AbstractNakshaHubVerticle;
 import com.here.naksha.app.service.NakshaApp;
-import com.here.naksha.app.service.http.apis.*;
+import com.here.naksha.app.service.http.apis.Api;
+import com.here.naksha.app.service.http.apis.EventHandlerApi;
+import com.here.naksha.app.service.http.apis.HealthApi;
+import com.here.naksha.app.service.http.apis.ReadFeatureApi;
+import com.here.naksha.app.service.http.apis.SpaceApi;
+import com.here.naksha.app.service.http.apis.StorageApi;
+import com.here.naksha.app.service.http.apis.WriteFeatureApi;
 import com.here.naksha.app.service.http.auth.NakshaJwtAuthHandler;
 import com.here.naksha.app.service.util.logging.AccessLog;
 import com.here.naksha.app.service.util.logging.AccessLogUtil;
@@ -63,9 +69,11 @@
 import com.here.naksha.lib.core.util.StreamInfo;
 import com.here.naksha.lib.hub.NakshaHubConfig;
 import io.netty.handler.codec.http.HttpResponseStatus;
+import io.vertx.core.Handler;
 import io.vertx.core.MultiMap;
 import io.vertx.core.Promise;
 import io.vertx.core.buffer.Buffer;
+import io.vertx.core.http.HttpConnection;
 import io.vertx.core.http.HttpMethod;
 import io.vertx.core.http.HttpServerOptions;
 import io.vertx.core.http.HttpServerResponse;
@@ -224,21 +232,24 @@ public void start(final @NotNull Promise<Void> startPromise) {
         // round-robin strategy.
         //
         // https://vertx.io/docs/vertx-core/java/#_server_sharing
-        vertx.createHttpServer(SERVER_OPTIONS).requestHandler(router).listen(hubConfig.httpPort, result -> {
-          if (result.succeeded()) {
-            log.atInfo()
-                .setMessage("HTTP Server started on port {}")
-                .addArgument(hubConfig.httpPort)
-                .log();
-            startPromise.complete();
-          } else {
-            log.atError()
-                .setMessage("An error occurred, during the initialization of the server.")
-                .setCause(result.cause())
-                .log();
-            startPromise.fail(result.cause());
-          }
-        });
+        vertx.createHttpServer(SERVER_OPTIONS)
+            .requestHandler(router)
+            .connectionHandler(loggingConnectionHandler())
+            .listen(hubConfig.httpPort, result -> {
+              if (result.succeeded()) {
+                log.atInfo()
+                    .setMessage("HTTP Server started on port {}")
+                    .addArgument(hubConfig.httpPort)
+                    .log();
+                startPromise.complete();
+              } else {
+                log.atError()
+                    .setMessage("An error occurred, during the initialization of the server.")
+                    .setCause(result.cause())
+                    .log();
+                startPromise.fail(result.cause());
+              }
+            });
       } catch (Throwable t) {
         log.atError()
             .setMessage(
@@ -406,6 +417,15 @@ private void onNewRequest(final @NotNull RoutingContext routingContext) {
     routingContext.next();
   }
 
+  /**
+   * @return simple http connection handler that logs closing and failing connections
+   */
+  private Handler<HttpConnection> loggingConnectionHandler() {
+    return httpConnection -> httpConnection
+        .closeHandler(v -> log.info("Closing connection"))
+        .exceptionHandler(t -> log.info("Connection exception", t));
+  }
+
   private static final Pattern FATAL_ERROR_MSG_PATTERN = Pattern.compile("^[0-9a-zA-Z.-_\\-]+$");
 
   /**
@@ -563,7 +583,7 @@ public void sendErrorResponse(@NotNull RoutingContext routingContext, @NotNull T
   /**
    * Prepare XyzFeatureCollection response by extracting feature results from ModifyFeaturesResp object
    *
-   * @param modifyResponse       The object to extract feature results from
+   * @param modifyResponse The object to extract feature results from
    * @return The XyzFeatureCollection response containing list of inserted/updated/delete features
    */
   public XyzFeatureCollection transformModifyResponse(@NotNull ModifyFeaturesResp modifyResponse) {