Download certs from security plugin instead of storing local copy

.github/workflows/opensearch-observability-test-and-build-workflow.yml

@@ -21,13 +21,12 @@ jobs:
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
       # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
-
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-      - uses: actions/checkout@v1
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+      - uses: actions/checkout@v4
 
       - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v1
@@ -46,7 +45,7 @@ jobs:
           su `id -un 1000` -c "./gradlew build"
 
       - name: Upload coverage
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v4
         with:
           flags: opensearch-observability
           directory: opensearch-observability/
@@ -58,7 +57,7 @@ jobs:
           cp -r ./build/distributions/*.zip opensearch-observability-builds/
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: opensearch-observability-ubuntu-latest
           path: opensearch-observability-builds
@@ -73,7 +72,7 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
 
       - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v1
@@ -90,7 +89,7 @@ jobs:
           cp -r ./build/distributions/*.zip opensearch-observability-builds/
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: opensearch-observability-${{ matrix.os }}
           path: opensearch-observability-builds

README.md

@@ -56,8 +56,6 @@ Observability is collection of plugins and applications that let you visualize d
 [dco-badge-link]: https://github.com/opensearch-project/observability/actions/workflows/dco.yml
 [link-check-badge]: https://github.com/opensearch-project/observability/actions/workflows/link-checker.yml/badge.svg
 [link-check-link]: https://github.com/opensearch-project/observability/actions/workflows/link-checker.yml
-[dashboard-build-badge]: https://github.com/opensearch-project/observability/actions/workflows/dashboards-observability-test-and-build-workflow.yml/badge.svg
-[dashboard-build-link]: https://github.com/opensearch-project/observability/actions/workflows/dashboards-observability-test-and-build-workflow.yml
 [opensearch-build-badge]: https://github.com/opensearch-project/observability/actions/workflows/opensearch-observability-test-and-build-workflow.yml/badge.svg
 [opensearch-build-link]: https://github.com/opensearch-project/observability/actions/workflows/opensearch-observability-test-and-build-workflow.yml
 [dashboard-codecov-badge]: https://codecov.io/gh/opensearch-project/observability/branch/main/graphs/badge.svg?flag=dashboards-observability

build.gradle

@@ -21,7 +21,7 @@ buildscript {
             opensearch_build += "-SNAPSHOT"
         }
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
-        kotlin_version = System.getProperty("kotlin.version", "1.8.21")
+        kotlin_version = System.getProperty("kotlin.version", "2.0.21")
     }
 
     repositories {
@@ -35,12 +35,13 @@ buildscript {
         classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}"
         classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}"
-        classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.23.4"
+        classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.23.7"
         classpath "org.jacoco:org.jacoco.agent:0.8.11"
     }
 }
 
 plugins {
+    id "de.undercouch.download" version "5.3.0"
     id "com.netflix.nebula.ospackage-base" version "11.6.0"
     id "com.dorongold.task-tree" version "1.5"
     id 'java-library'
@@ -124,6 +125,7 @@ configurations.all {
     resolutionStrategy {
         force "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
         force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlin_version}"
+        force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${kotlin_version}"
         force "org.jetbrains.kotlin:kotlin-reflect:${kotlin_version}"
         force "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
         force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
@@ -146,30 +148,44 @@ ext {
     projectSubstitutions = [:]
     licenseFile = rootProject.file('LICENSE.txt')
     noticeFile = rootProject.file('NOTICE.txt')
+
+    ['sample.pem', 'test-kirk.jks'].forEach { file ->
+        File local = getLayout().getBuildDirectory().file(file).get().getAsFile()
+        download.run {
+            src "https://raw.githubusercontent.com/opensearch-project/security/refs/heads/main/bwc-test/src/test/resources/security/" + file
+            dest local
+            overwrite false
+        }
+    }
+
+    processTestResources {
+        from(getLayout().getBuildDirectory().file('sample.pem').get().getAsFile())
+        from(getLayout().getBuildDirectory().file('test-kirk.jks').get().getAsFile())
+    }
 }
 
 plugins.withId('java') {
-    sourceCompatibility = targetCompatibility = "11"
+    sourceCompatibility = targetCompatibility = "21"
 }
 
 plugins.withId('org.jetbrains.kotlin.jvm') {
-    compileKotlin.kotlinOptions.jvmTarget = compileTestKotlin.kotlinOptions.jvmTarget = "11"
+    compileKotlin.kotlinOptions.jvmTarget = compileTestKotlin.kotlinOptions.jvmTarget = "21"
 }
 
 allprojects {
     group = "org.opensearch"
 
     version = "${opensearch_build}"
     plugins.withId('java') {
-        sourceCompatibility = targetCompatibility = "11"
+        sourceCompatibility = targetCompatibility = "21"
     }
 }
 
 dependencies {
     implementation "org.opensearch:opensearch:${opensearch_version}"
     implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
     implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
-    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9"
+    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.9.0"
     implementation "${group}:common-utils:${common_utils_version}"
     implementation 'org.json:json:20231013'
     implementation group: 'com.github.wnameless.json', name: 'json-flattener', version: '0.15.1'
@@ -308,7 +324,7 @@ testClusters.integTest {
     setting 'path.repo', repo.absolutePath
 }
 
-String baseVersion = "2.16.0"
+String baseVersion = "2.19.0"
 String bwcVersion = baseVersion + ".0"
 String baseName = "obsBwcCluster"
 String bwcFilePath = "src/test/resources/bwc/"
@@ -549,10 +565,5 @@ task updateVersion {
          // String tokenization to support -SNAPSHOT
         ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
         ant.replaceregexp(file:'../.github/workflows/dashboards-observability-test-and-build-workflow.yml', match:'OPENSEARCH_PLUGIN_VERSION: \\d+.\\d+.\\d+.\\d+', replace:'OPENSEARCH_PLUGIN_VERSION: ' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true)
-        // Match key version in JSON files.
-        ant.replaceregexp(file:'../dashboards-observability/opensearch_dashboards.json', match:'"version": "\\d+.\\d+.\\d+.\\d+', replace:'"version": ' + '"' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true)
-        ant.replaceregexp(file:'../dashboards-observability/package.json', match:'"version": "\\d+.\\d+.\\d+.\\d+', replace:'"version": ' + '"' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true)
-        // Match key opensearchDashboardsVersion in JSON files.
-        ant.replaceregexp(file:'../dashboards-observability/opensearch_dashboards.json', match:'"opensearchDashboardsVersion": "\\d+.\\d+.\\d+', replace:'"opensearchDashboardsVersion": ' + '"' + newVersion.tokenize('-')[0], flags:'g', byline:true)
     }
 }

docs/ObservabilityPlayground-LandingPage.md

@@ -24,7 +24,7 @@ Your experience of exploring data might differ, but if you’re new to exploring
 ![](https://opensearch.org/docs/latest/images/notebooks.png)
 
 ---
-7. OpenSearch OpenTelemetry [Demo Store Application](https://opentelemetry.io/ecosystem/demo/) hosted in our [Observability Demo playground](https://playground.opensearch.org/)
+7. OpenSearch OpenTelemetry [Demo Store Application](https://opentelemetry.io/ecosystem/demo/) hosted in our [Observability Demo playground](https://playground.opensearch.org/app/login)
 
 - **_Demo Shop_**
 ![](https://opentelemetry.io/docs/demo/screenshots/frontend-1.png)

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# SPDX-License-Identifier: Apache-2.0
+#
 
 ##############################################################################
 #
@@ -55,7 +57,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -84,7 +86,8 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum

gradlew.bat

@@ -13,6 +13,8 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 @rem
+@rem SPDX-License-Identifier: Apache-2.0
+@rem
 
 @if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
@@ -43,11 +45,11 @@ set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
 if %ERRORLEVEL% equ 0 goto execute
 
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
 
 goto fail
 
@@ -57,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
 if exist "%JAVA_EXE%" goto execute
 
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
 
 goto fail
 

src/main/kotlin/org/opensearch/observability/action/PluginBaseAction.kt

@@ -25,6 +25,7 @@ import org.opensearch.index.engine.VersionConflictEngineException
 import org.opensearch.indices.InvalidIndexNameException
 import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX
 import org.opensearch.observability.metrics.Metrics
+import org.opensearch.observability.security.SecurityAccess
 import org.opensearch.observability.util.logger
 import org.opensearch.tasks.Task
 import org.opensearch.transport.TransportService
@@ -53,50 +54,52 @@ abstract class PluginBaseAction<Request : ActionRequest, Response : ActionRespon
     ) {
         val userStr: String? = client.threadPool().threadContext.getTransient<String>(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)
         val user: User? = User.parse(userStr)
-        scope.launch {
-            try {
-                listener.onResponse(executeRequest(request, user))
-            } catch (exception: OpenSearchStatusException) {
-                Metrics.OBSERVABILITY_EXCEPTIONS_OPENSEARCH_STATUS_EXCEPTION.counter.increment()
-                log.warn("$LOG_PREFIX:OpenSearchStatusException: message:${exception.message}")
-                listener.onFailure(exception)
-            } catch (exception: OpenSearchSecurityException) {
-                Metrics.OBSERVABILITY_EXCEPTIONS_OPENSEARCH_SECURITY_EXCEPTION.counter.increment()
-                log.warn("$LOG_PREFIX:OpenSearchSecurityException:", exception)
-                listener.onFailure(
-                    OpenSearchStatusException(
-                        "Permissions denied: ${exception.message} - Contact administrator",
-                        RestStatus.FORBIDDEN
+        SecurityAccess.doPrivileged {
+            scope.launch {
+                try {
+                    listener.onResponse(executeRequest(request, user))
+                } catch (exception: OpenSearchStatusException) {
+                    Metrics.OBSERVABILITY_EXCEPTIONS_OPENSEARCH_STATUS_EXCEPTION.counter.increment()
+                    log.warn("$LOG_PREFIX:OpenSearchStatusException: message:${exception.message}")
+                    listener.onFailure(exception)
+                } catch (exception: OpenSearchSecurityException) {
+                    Metrics.OBSERVABILITY_EXCEPTIONS_OPENSEARCH_SECURITY_EXCEPTION.counter.increment()
+                    log.warn("$LOG_PREFIX:OpenSearchSecurityException:", exception)
+                    listener.onFailure(
+                        OpenSearchStatusException(
+                            "Permissions denied: ${exception.message} - Contact administrator",
+                            RestStatus.FORBIDDEN
+                        )
                     )
-                )
-            } catch (exception: VersionConflictEngineException) {
-                Metrics.OBSERVABILITY_EXCEPTIONS_VERSION_CONFLICT_ENGINE_EXCEPTION.counter.increment()
-                log.warn("$LOG_PREFIX:VersionConflictEngineException:", exception)
-                listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.CONFLICT))
-            } catch (exception: IndexNotFoundException) {
-                Metrics.OBSERVABILITY_EXCEPTIONS_INDEX_NOT_FOUND_EXCEPTION.counter.increment()
-                log.warn("$LOG_PREFIX:IndexNotFoundException:", exception)
-                listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.NOT_FOUND))
-            } catch (exception: InvalidIndexNameException) {
-                Metrics.OBSERVABILITY_EXCEPTIONS_INVALID_INDEX_NAME_EXCEPTION.counter.increment()
-                log.warn("$LOG_PREFIX:InvalidIndexNameException:", exception)
-                listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.BAD_REQUEST))
-            } catch (exception: IllegalArgumentException) {
-                Metrics.OBSERVABILITY_EXCEPTIONS_ILLEGAL_ARGUMENT_EXCEPTION.counter.increment()
-                log.warn("$LOG_PREFIX:IllegalArgumentException:", exception)
-                listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.BAD_REQUEST))
-            } catch (exception: IllegalStateException) {
-                Metrics.OBSERVABILITY_EXCEPTIONS_ILLEGAL_STATE_EXCEPTION.counter.increment()
-                log.warn("$LOG_PREFIX:IllegalStateException:", exception)
-                listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.SERVICE_UNAVAILABLE))
-            } catch (exception: IOException) {
-                Metrics.OBSERVABILITY_EXCEPTIONS_IO_EXCEPTION.counter.increment()
-                log.error("$LOG_PREFIX:Uncaught IOException:", exception)
-                listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.FAILED_DEPENDENCY))
-            } catch (exception: Exception) {
-                Metrics.OBSERVABILITY_EXCEPTIONS_INTERNAL_SERVER_ERROR.counter.increment()
-                log.error("$LOG_PREFIX:Uncaught Exception:", exception)
-                listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.INTERNAL_SERVER_ERROR))
+                } catch (exception: VersionConflictEngineException) {
+                    Metrics.OBSERVABILITY_EXCEPTIONS_VERSION_CONFLICT_ENGINE_EXCEPTION.counter.increment()
+                    log.warn("$LOG_PREFIX:VersionConflictEngineException:", exception)
+                    listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.CONFLICT))
+                } catch (exception: IndexNotFoundException) {
+                    Metrics.OBSERVABILITY_EXCEPTIONS_INDEX_NOT_FOUND_EXCEPTION.counter.increment()
+                    log.warn("$LOG_PREFIX:IndexNotFoundException:", exception)
+                    listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.NOT_FOUND))
+                } catch (exception: InvalidIndexNameException) {
+                    Metrics.OBSERVABILITY_EXCEPTIONS_INVALID_INDEX_NAME_EXCEPTION.counter.increment()
+                    log.warn("$LOG_PREFIX:InvalidIndexNameException:", exception)
+                    listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.BAD_REQUEST))
+                } catch (exception: IllegalArgumentException) {
+                    Metrics.OBSERVABILITY_EXCEPTIONS_ILLEGAL_ARGUMENT_EXCEPTION.counter.increment()
+                    log.warn("$LOG_PREFIX:IllegalArgumentException:", exception)
+                    listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.BAD_REQUEST))
+                } catch (exception: IllegalStateException) {
+                    Metrics.OBSERVABILITY_EXCEPTIONS_ILLEGAL_STATE_EXCEPTION.counter.increment()
+                    log.warn("$LOG_PREFIX:IllegalStateException:", exception)
+                    listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.SERVICE_UNAVAILABLE))
+                } catch (exception: IOException) {
+                    Metrics.OBSERVABILITY_EXCEPTIONS_IO_EXCEPTION.counter.increment()
+                    log.error("$LOG_PREFIX:Uncaught IOException:", exception)
+                    listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.FAILED_DEPENDENCY))
+                } catch (exception: Exception) {
+                    Metrics.OBSERVABILITY_EXCEPTIONS_INTERNAL_SERVER_ERROR.counter.increment()
+                    log.error("$LOG_PREFIX:Uncaught Exception:", exception)
+                    listener.onFailure(OpenSearchStatusException(exception.message, RestStatus.INTERNAL_SERVER_ERROR))
+                }
             }
         }
     }

src/main/plugin-metadata/plugin-security.policy

@@ -6,6 +6,7 @@
 grant {
     // needed to find the classloader to load allowlisted classes.
     permission java.lang.RuntimePermission "createClassLoader";
+    permission java.lang.RuntimePermission "setContextClassLoader";
     permission java.lang.RuntimePermission "getClassLoader";
 
     permission java.net.SocketPermission "*", "connect,resolve";

src/test/kotlin/org/opensearch/observability/PluginRestTestCase.kt

@@ -104,7 +104,7 @@ abstract class PluginRestTestCase : OpenSearchRestTestCase() {
             return when (keystore != null) {
                 true -> {
                     // create adminDN (super-admin) client
-                    val uri = javaClass.classLoader.getResource("security/sample.pem").toURI()
+                    val uri = javaClass.classLoader.getResource("sample.pem").toURI()
                     val configPath = PathUtils.get(uri).parent.toAbsolutePath()
                     SecureRestClientBuilder(settings, configPath).setSocketTimeout(60000).build()
                 }