From 2be2da6cd796e762dd36a6cc0487056b7308c5c4 Mon Sep 17 00:00:00 2001 From: abhishekbafna Date: Thu, 16 Jan 2025 17:30:01 +0530 Subject: [PATCH 1/2] Support for groovy static analysis for groovy scripts. --- .../pinot/common/metrics/ServerMeter.java | 1 + .../api/resources/PinotClusterConfigs.java | 74 +++++++++ .../org/apache/pinot/core/auth/Actions.java | 2 + .../function/GroovyFunctionEvaluatorTest.java | 110 ++++++++++++- .../GroovyStaticAnalyzerConfigTest.java | 140 ++++++++++++++++ .../function/GroovyFunctionEvaluator.java | 119 +++++++++++++- .../function/GroovyStaticAnalyzerConfig.java | 153 ++++++++++++++++++ .../starter/helix/BaseServerStarter.java | 25 +++ .../pinot/spi/utils/CommonConstants.java | 1 + .../pinot/tools/RealtimeQuickStart.java | 1 + 10 files changed, 618 insertions(+), 8 deletions(-) create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyStaticAnalyzerConfigTest.java create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/function/GroovyStaticAnalyzerConfig.java diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java index 6a53ddc57130..df5aece8716b 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java @@ -27,6 +27,7 @@ public enum ServerMeter implements AbstractMetrics.Meter { QUERIES("queries", true), UNCAUGHT_EXCEPTIONS("exceptions", true), + GROOVY_SECURITY_VIOLATIONS("exceptions", true), REQUEST_DESERIALIZATION_EXCEPTIONS("exceptions", true), RESPONSE_SERIALIZATION_EXCEPTIONS("exceptions", true), SCHEDULING_TIMEOUT_EXCEPTIONS("exceptions", true), diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java index f54751db16bf..97be9d7f69fa 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java @@ -55,6 +55,9 @@ import org.apache.pinot.core.auth.Actions; import org.apache.pinot.core.auth.Authorize; import org.apache.pinot.core.auth.TargetType; +import org.apache.pinot.segment.local.function.GroovyFunctionEvaluator; +import org.apache.pinot.segment.local.function.GroovyStaticAnalyzerConfig; +import org.apache.pinot.spi.utils.CommonConstants; import org.apache.pinot.spi.utils.JsonUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -168,4 +171,75 @@ public SuccessResponse deleteClusterConfig( throw new ControllerApplicationException(LOGGER, errStr, Response.Status.INTERNAL_SERVER_ERROR, e); } } + + @GET + @Path("/cluster/configs/groovy/staticAnalyzerConfig") + @Authorize(targetType = TargetType.CLUSTER, action = Actions.Cluster.GET_GROOVY_STATIC_ANALYZER_CONFIG) + @Produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Get the configuration for Groovy Static analysis", + notes = "Get the configuration for Groovy static analysis") + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Success"), + @ApiResponse(code = 500, message = "Internal server error") + }) + public GroovyStaticAnalyzerConfig getGroovyStaticAnalysisConfig() throws Exception { + HelixAdmin helixAdmin = _pinotHelixResourceManager.getHelixAdmin(); + HelixConfigScope configScope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER) + .forCluster(_pinotHelixResourceManager.getHelixClusterName()).build(); + Map configs = helixAdmin.getConfig(configScope, + List.of(CommonConstants.Server.GROOVY_STATIC_ANALYZER_CONFIG)); + String json = configs.get(CommonConstants.Server.GROOVY_STATIC_ANALYZER_CONFIG); + if (json != null) { + return GroovyStaticAnalyzerConfig.fromJson(json); + } else { + return null; + } + } + + + @POST + @Path("/cluster/configs/groovy/staticAnalyzerConfig") + @Authorize(targetType = TargetType.CLUSTER, action = Actions.Cluster.UPDATE_GROOVY_STATIC_ANALYZER_CONFIG) + @Authenticate(AccessType.UPDATE) + @ApiOperation(value = "Update Groovy static analysis configuration") + @Produces(MediaType.APPLICATION_JSON) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Success"), + @ApiResponse(code = 500, message = "Server error updating configuration") + }) + public SuccessResponse setGroovyStaticAnalysisConfig(String body) throws Exception { + try { + HelixAdmin admin = _pinotHelixResourceManager.getHelixAdmin(); + HelixConfigScope configScope = + new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER).forCluster( + _pinotHelixResourceManager.getHelixClusterName()).build(); + Map properties = new TreeMap<>(); + GroovyStaticAnalyzerConfig groovyConfig = GroovyStaticAnalyzerConfig.fromJson(body); + properties.put(CommonConstants.Server.GROOVY_STATIC_ANALYZER_CONFIG, + groovyConfig == null ? null : groovyConfig.toJson()); + admin.setConfig(configScope, properties); + GroovyFunctionEvaluator.setConfig(groovyConfig); + return new SuccessResponse("Updated Groovy Static Analyzer config."); + } catch (IOException e) { + throw new ControllerApplicationException(LOGGER, "Error converting request to cluster config", + Response.Status.BAD_REQUEST, e); + } catch (Exception e) { + throw new ControllerApplicationException(LOGGER, "Failed to update Groovy Static Analyzer config", + Response.Status.INTERNAL_SERVER_ERROR, e); + } + } + + @GET + @Path("/cluster/configs/groovy/staticAnalyzerConfig/default") + @Authorize(targetType = TargetType.CLUSTER, action = Actions.Cluster.GET_GROOVY_STATIC_ANALYZER_CONFIG) + @Produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Get the default configuration for Groovy Static analysis", + notes = "Get the default configuration for Groovy static analysis") + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Success"), + @ApiResponse(code = 500, message = "Internal server error") + }) + public GroovyStaticAnalyzerConfig getDefaultGroovyStaticAnalysisConfig() { + return GroovyStaticAnalyzerConfig.createDefault(); + } } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java b/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java index 96e4f27790a7..2fa066e991fb 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java @@ -99,6 +99,8 @@ public static class Cluster { public static final String UPDATE_INSTANCE_PARTITIONS = "UpdateInstancePartitions"; public static final String GET_RESPONSE_STORE = "GetResponseStore"; public static final String DELETE_RESPONSE_STORE = "DeleteResponseStore"; + public static final String GET_GROOVY_STATIC_ANALYZER_CONFIG = "GetGroovyStaticAnalyzerConfig"; + public static final String UPDATE_GROOVY_STATIC_ANALYZER_CONFIG = "UpdateGroovyStaticAnalyzerConfig"; } // Action names for table diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java index 29e28f475ec2..6e3222850cba 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java @@ -18,22 +18,114 @@ */ package org.apache.pinot.core.data.function; +import com.fasterxml.jackson.core.JsonProcessingException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.pinot.segment.local.function.GroovyFunctionEvaluator; +import org.apache.pinot.segment.local.function.GroovyStaticAnalyzerConfig; import org.apache.pinot.spi.data.readers.GenericRow; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import org.testng.collections.Lists; +import static org.apache.pinot.segment.local.function.GroovyStaticAnalyzerConfig.getDefaultAllowedImports; +import static org.apache.pinot.segment.local.function.GroovyStaticAnalyzerConfig.getDefaultAllowedReceivers; + /** * Tests Groovy functions for transforming schema columns */ public class GroovyFunctionEvaluatorTest { + @Test + public void testLegalGroovyScripts() + throws JsonProcessingException { + // TODO: Add separate tests for these rules: receivers, imports, static imports, and method names. + List scripts = List.of( + "Groovy({2})", + "Groovy({![\"pinot_minion_totalOutputSegmentSize_Value\"].contains(\"\");2})", + "Groovy({airtime == null ? (arrdelay == null ? 0 : arrdelay.value) : airtime.value; 2}, airtime, arrdelay)" + ); + + GroovyStaticAnalyzerConfig config = new GroovyStaticAnalyzerConfig( + getDefaultAllowedReceivers(), + getDefaultAllowedImports(), + getDefaultAllowedImports(), + List.of("invoke", "execute"), + false); + GroovyFunctionEvaluator.setConfig(config); + + for (String script : scripts) { + GroovyFunctionEvaluator groovyFunctionEvaluator = new GroovyFunctionEvaluator(script); + GenericRow row = new GenericRow(); + Object result = groovyFunctionEvaluator.evaluate(row); + Assert.assertEquals(2, result); + } + } + + @Test + public void testIllegalGroovyScripts() + throws JsonProcessingException { + // TODO: Add separate tests for these rules: receivers, imports, static imports, and method names. + List scripts = List.of( + "Groovy({\"ls\".execute()})", + "Groovy({[\"ls\"].execute()})", + "Groovy({System.exit(5)})", + "Groovy({System.metaClass.methods.each { method -> if (method.name.md5() == " + + "\"f24f62eeb789199b9b2e467df3b1876b\") {method.invoke(System, 10)} }})", + "Groovy({System.metaClass.methods.each { method -> if (method.name.reverse() == (\"ti\" + \"xe\")) " + + "{method.invoke(System, 10)} }})", + "groovy({def args = [\"QuickStart\", \"-type\", \"REALTIME\"] as String[]; " + + "org.apache.pinot.tools.admin.PinotAdministrator.main(args); 2})", + "Groovy({return [\"bash\", \"-c\", \"env\"].execute().text})" + ); + + GroovyStaticAnalyzerConfig config = new GroovyStaticAnalyzerConfig( + getDefaultAllowedReceivers(), + getDefaultAllowedImports(), + getDefaultAllowedImports(), + List.of("invoke", "execute"), + false); + GroovyFunctionEvaluator.setConfig(config); + + for (String script : scripts) { + try { + GroovyFunctionEvaluator groovyFunctionEvaluator = new GroovyFunctionEvaluator(script); + GenericRow row = new GenericRow(); + groovyFunctionEvaluator.evaluate(row); + Assert.fail("Groovy analyzer failed to catch malicious script"); + } catch (Exception ignored) { + } + } + } + + @Test + public void testUpdatingConfiguration() + throws JsonProcessingException { + // TODO: Figure out how to test this with the singleton initializer + // These tests would pass by default but the configuration will be updated so that they fail + List scripts = List.of( + "Groovy({2})", + "Groovy({![\"pinot_minion_totalOutputSegmentSize_Value\"].contains(\"\");2})", + "Groovy({airtime == null ? (arrdelay == null ? 0 : arrdelay.value) : airtime.value; 2}, airtime, arrdelay)" + ); + + GroovyStaticAnalyzerConfig config = + new GroovyStaticAnalyzerConfig(List.of(), List.of(), List.of(), List.of(), false); + GroovyFunctionEvaluator.setConfig(config); + + for (String script : scripts) { + try { + GroovyFunctionEvaluator groovyFunctionEvaluator = new GroovyFunctionEvaluator(script); + GenericRow row = new GenericRow(); + groovyFunctionEvaluator.evaluate(row); + Assert.fail(String.format("Groovy analyzer failed to catch malicious script: %s", script)); + } catch (Exception ignored) { + } + } + } @Test(dataProvider = "groovyFunctionEvaluationDataProvider") public void testGroovyFunctionEvaluation(String transformFunction, List arguments, GenericRow genericRow, @@ -108,20 +200,26 @@ public Object[][] groovyFunctionEvaluationDataProvider() { GenericRow genericRow9 = new GenericRow(); genericRow9.putValue("ArrTime", 101); genericRow9.putValue("ArrTimeV2", null); - entries.add(new Object[]{"Groovy({ArrTimeV2 != null ? ArrTimeV2: ArrTime }, ArrTime, ArrTimeV2)", - Lists.newArrayList("ArrTime", "ArrTimeV2"), genericRow9, 101}); + entries.add(new Object[]{ + "Groovy({ArrTimeV2 != null ? ArrTimeV2: ArrTime }, ArrTime, ArrTimeV2)", + Lists.newArrayList("ArrTime", "ArrTimeV2"), genericRow9, 101 + }); GenericRow genericRow10 = new GenericRow(); String jello = "Jello"; genericRow10.putValue("jello", jello); - entries.add(new Object[]{"Groovy({jello != null ? jello.length() : \"Jello\" }, jello)", - Lists.newArrayList("jello"), genericRow10, 5}); + entries.add(new Object[]{ + "Groovy({jello != null ? jello.length() : \"Jello\" }, jello)", + Lists.newArrayList("jello"), genericRow10, 5 + }); //Invalid groovy script GenericRow genericRow11 = new GenericRow(); genericRow11.putValue("nullValue", null); - entries.add(new Object[]{"Groovy({nullValue == null ? nullValue.length() : \"Jello\" }, nullValue)", - Lists.newArrayList("nullValue"), genericRow11, null}); + entries.add(new Object[]{ + "Groovy({nullValue == null ? nullValue.length() : \"Jello\" }, nullValue)", + Lists.newArrayList("nullValue"), genericRow11, null + }); return entries.toArray(new Object[entries.size()][]); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyStaticAnalyzerConfigTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyStaticAnalyzerConfigTest.java new file mode 100644 index 000000000000..f051de395520 --- /dev/null +++ b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyStaticAnalyzerConfigTest.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.core.data.function; + +import java.util.Iterator; +import java.util.List; +import org.apache.helix.zookeeper.datamodel.ZNRecord; +import org.apache.pinot.segment.local.function.GroovyStaticAnalyzerConfig; +import org.apache.pinot.spi.utils.JsonUtils; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + + +/** + * Test serialization and deserialization. + */ +public class GroovyStaticAnalyzerConfigTest { + @Test + public void testEmptyConfig() + throws Exception { + GroovyStaticAnalyzerConfig config = new GroovyStaticAnalyzerConfig(null, null, null, null, false); + String encodedConfig = JsonUtils.objectToString(config); + GroovyStaticAnalyzerConfig decodedConfig = + JsonUtils.stringToObject(encodedConfig, GroovyStaticAnalyzerConfig.class); + + Assert.assertNull(decodedConfig.getAllowedReceivers()); + Assert.assertNull(decodedConfig.getAllowedImports()); + Assert.assertNull(decodedConfig.getAllowedStaticImports()); + Assert.assertNull(decodedConfig.getDisallowedMethodNames()); + } + + @Test + public void testAllowedReceivers() + throws Exception { + GroovyStaticAnalyzerConfig config = new GroovyStaticAnalyzerConfig( + GroovyStaticAnalyzerConfig.getDefaultAllowedReceivers(), null, null, null, false); + String encodedConfig = JsonUtils.objectToString(config); + GroovyStaticAnalyzerConfig decodedConfig = + JsonUtils.stringToObject(encodedConfig, GroovyStaticAnalyzerConfig.class); + + Assert.assertEquals(GroovyStaticAnalyzerConfig.getDefaultAllowedReceivers(), decodedConfig.getAllowedReceivers()); + Assert.assertNull(decodedConfig.getAllowedImports()); + Assert.assertNull(decodedConfig.getAllowedStaticImports()); + Assert.assertNull(decodedConfig.getDisallowedMethodNames()); + } + + @Test + public void testAllowedImports() + throws Exception { + GroovyStaticAnalyzerConfig config = + new GroovyStaticAnalyzerConfig(null, GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), null, null, false); + String encodedConfig = JsonUtils.objectToString(config); + GroovyStaticAnalyzerConfig decodedConfig = + JsonUtils.stringToObject(encodedConfig, GroovyStaticAnalyzerConfig.class); + + Assert.assertNull(decodedConfig.getAllowedReceivers()); + Assert.assertEquals(GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), decodedConfig.getAllowedImports()); + Assert.assertNull(decodedConfig.getAllowedStaticImports()); + Assert.assertNull(decodedConfig.getDisallowedMethodNames()); + } + + @Test + public void testAllowedStaticImports() + throws Exception { + GroovyStaticAnalyzerConfig config = + new GroovyStaticAnalyzerConfig(null, null, GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), null, false); + String encodedConfig = JsonUtils.objectToString(config); + GroovyStaticAnalyzerConfig decodedConfig = + JsonUtils.stringToObject(encodedConfig, GroovyStaticAnalyzerConfig.class); + + Assert.assertNull(decodedConfig.getAllowedReceivers()); + Assert.assertNull(decodedConfig.getAllowedImports()); + Assert.assertEquals(GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), decodedConfig.getAllowedStaticImports()); + Assert.assertNull(decodedConfig.getDisallowedMethodNames()); + } + + @Test + public void testDisallowedMethodNames() + throws Exception { + GroovyStaticAnalyzerConfig config = + new GroovyStaticAnalyzerConfig(null, null, null, List.of("method1", "method2"), false); + String encodedConfig = JsonUtils.objectToString(config); + GroovyStaticAnalyzerConfig decodedConfig = + JsonUtils.stringToObject(encodedConfig, GroovyStaticAnalyzerConfig.class); + + Assert.assertNull(decodedConfig.getAllowedReceivers()); + Assert.assertNull(decodedConfig.getAllowedImports()); + Assert.assertNull(decodedConfig.getAllowedStaticImports()); + Assert.assertEquals(List.of("method1", "method2"), decodedConfig.getDisallowedMethodNames()); + } + + @Test(dataProvider = "config_provider") + public void testToZnRecord(GroovyStaticAnalyzerConfig config) + throws Exception { + ZNRecord zr = config.toZNRecord(); + GroovyStaticAnalyzerConfig znConfig = GroovyStaticAnalyzerConfig.fromZNRecord(zr); + Assert.assertTrue(equals(znConfig, config)); + } + + @DataProvider(name = "config_provider") + Iterator configProvider() { + return List.of( + new GroovyStaticAnalyzerConfig(null, null, null, List.of("method1", "method2"), false), + new GroovyStaticAnalyzerConfig( + GroovyStaticAnalyzerConfig.getDefaultAllowedReceivers(), null, null, null, false), + new GroovyStaticAnalyzerConfig(null, GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), null, null, false), + new GroovyStaticAnalyzerConfig(null, null, GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), null, false), + new GroovyStaticAnalyzerConfig(null, null, null, List.of("method1", "method2"), false) + ).iterator(); + } + + private boolean equals(GroovyStaticAnalyzerConfig a, GroovyStaticAnalyzerConfig b) { + return a != null && b != null + && (a.getAllowedStaticImports() == b.getAllowedStaticImports() + || a.getAllowedStaticImports().equals(b.getAllowedStaticImports())) + && (a.getAllowedImports() == null && b.getAllowedImports() == null + || a.getAllowedImports().equals(b.getAllowedImports())) + && (a.getAllowedReceivers() == null && b.getAllowedReceivers() == null + || a.getAllowedReceivers().equals(b.getAllowedReceivers())) + && (a.getDisallowedMethodNames() == null && b.getDisallowedMethodNames() == null + || a.getDisallowedMethodNames().equals(b.getDisallowedMethodNames())); + } +} diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/function/GroovyFunctionEvaluator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/function/GroovyFunctionEvaluator.java index 52f36465bb16..df510063d02b 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/function/GroovyFunctionEvaluator.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/function/GroovyFunctionEvaluator.java @@ -18,6 +18,7 @@ */ package org.apache.pinot.segment.local.function; +import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import groovy.lang.Binding; @@ -27,7 +28,15 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.pinot.common.metrics.ServerMeter; +import org.apache.pinot.common.metrics.ServerMetrics; import org.apache.pinot.spi.data.readers.GenericRow; +import org.codehaus.groovy.ast.expr.MethodCallExpression; +import org.codehaus.groovy.control.CompilerConfiguration; +import org.codehaus.groovy.control.customizers.ImportCustomizer; +import org.codehaus.groovy.control.customizers.SecureASTCustomizer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -45,6 +54,7 @@ * ] */ public class GroovyFunctionEvaluator implements FunctionEvaluator { + private static final Logger LOGGER = LoggerFactory.getLogger(GroovyFunctionEvaluator.class); private static final String GROOVY_EXPRESSION_PREFIX = "Groovy"; private static final String GROOVY_FUNCTION_REGEX = "Groovy\\(\\{(?