diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java index ee034ec952ca..d82e058daf61 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerMeter.java @@ -68,7 +68,9 @@ public enum ControllerMeter implements AbstractMetrics.Meter { NUMBER_ADHOC_TASKS_SUBMITTED("adhocTasks", false), IDEAL_STATE_UPDATE_FAILURE("IdealStateUpdateFailure", false), IDEAL_STATE_UPDATE_RETRY("IdealStateUpdateRetry", false), - IDEAL_STATE_UPDATE_SUCCESS("IdealStateUpdateSuccess", false); + IDEAL_STATE_UPDATE_SUCCESS("IdealStateUpdateSuccess", false), + GROOVY_SECURITY_VIOLATIONS("exceptions", true); + private final String _brokerMeterName; diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionMeter.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionMeter.java index 220fd7cb7d2b..96f2c87a6074 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionMeter.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/MinionMeter.java @@ -38,7 +38,8 @@ public enum MinionMeter implements AbstractMetrics.Meter { SEGMENT_BYTES_UPLOADED("bytes", false), RECORDS_PROCESSED_COUNT("rows", false), RECORDS_PURGED_COUNT("rows", false), - COMPACTED_RECORDS_COUNT("rows", false); + COMPACTED_RECORDS_COUNT("rows", false), + GROOVY_SECURITY_VIOLATIONS("exceptions", true); private final String _meterName; private final String _unit; diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java index 6a53ddc57130..df5aece8716b 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java @@ -27,6 +27,7 @@ public enum ServerMeter implements AbstractMetrics.Meter { QUERIES("queries", true), UNCAUGHT_EXCEPTIONS("exceptions", true), + GROOVY_SECURITY_VIOLATIONS("exceptions", true), REQUEST_DESERIALIZATION_EXCEPTIONS("exceptions", true), RESPONSE_SERIALIZATION_EXCEPTIONS("exceptions", true), SCHEDULING_TIMEOUT_EXCEPTIONS("exceptions", true), diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java index a0ce503d41a1..27f19eb93c38 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java @@ -122,6 +122,7 @@ import org.apache.pinot.core.segment.processing.lifecycle.PinotSegmentLifecycleEventListenerManager; import org.apache.pinot.core.transport.ListenerConfig; import org.apache.pinot.core.util.ListenerConfigUtil; +import org.apache.pinot.segment.local.function.GroovyFunctionEvaluator; import org.apache.pinot.segment.local.utils.TableConfigUtils; import org.apache.pinot.spi.config.table.TableConfig; import org.apache.pinot.spi.crypt.PinotCrypterFactory; @@ -382,7 +383,8 @@ public ControllerConf getConfig() { } @Override - public void start() { + public void start() + throws Exception { LOGGER.info("Starting Pinot controller in mode: {}. (Version: {})", _controllerMode.name(), PinotVersion.VERSION); LOGGER.info("Controller configs: {}", new PinotAppConfigs(getConfig()).toJSONString()); long startTimeMs = System.currentTimeMillis(); @@ -407,6 +409,12 @@ public void start() { break; } + // Initializing Groovy execution security + GroovyFunctionEvaluator.configureGroovySecurity( + _config.getProperty(CommonConstants.GROOVY_STATIC_ANALYZER_CONFIG)); + GroovyFunctionEvaluator.setMetrics(_controllerMetrics, ControllerMeter.GROOVY_SECURITY_VIOLATIONS); + + ServiceStatus.setServiceStatusCallback(_helixParticipantInstanceId, new ServiceStatus.MultipleCallbackServiceStatusCallback(_serviceStatusCallbackList)); _controllerMetrics.addTimedValue(ControllerTimer.STARTUP_SUCCESS_DURATION_MS, diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java index f54751db16bf..07c4394f76ef 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java @@ -55,6 +55,9 @@ import org.apache.pinot.core.auth.Actions; import org.apache.pinot.core.auth.Authorize; import org.apache.pinot.core.auth.TargetType; +import org.apache.pinot.segment.local.function.GroovyFunctionEvaluator; +import org.apache.pinot.segment.local.function.GroovyStaticAnalyzerConfig; +import org.apache.pinot.spi.utils.CommonConstants; import org.apache.pinot.spi.utils.JsonUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -168,4 +171,75 @@ public SuccessResponse deleteClusterConfig( throw new ControllerApplicationException(LOGGER, errStr, Response.Status.INTERNAL_SERVER_ERROR, e); } } + + @GET + @Path("/cluster/configs/groovy/staticAnalyzerConfig") + @Authorize(targetType = TargetType.CLUSTER, action = Actions.Cluster.GET_GROOVY_STATIC_ANALYZER_CONFIG) + @Produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Get the configuration for Groovy Static analysis", + notes = "Get the configuration for Groovy static analysis") + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Success"), + @ApiResponse(code = 500, message = "Internal server error") + }) + public GroovyStaticAnalyzerConfig getGroovyStaticAnalysisConfig() throws Exception { + HelixAdmin helixAdmin = _pinotHelixResourceManager.getHelixAdmin(); + HelixConfigScope configScope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER) + .forCluster(_pinotHelixResourceManager.getHelixClusterName()).build(); + Map configs = helixAdmin.getConfig(configScope, + List.of(CommonConstants.GROOVY_STATIC_ANALYZER_CONFIG)); + String json = configs.get(CommonConstants.GROOVY_STATIC_ANALYZER_CONFIG); + if (json != null) { + return GroovyStaticAnalyzerConfig.fromJson(json); + } else { + return null; + } + } + + + @POST + @Path("/cluster/configs/groovy/staticAnalyzerConfig") + @Authorize(targetType = TargetType.CLUSTER, action = Actions.Cluster.UPDATE_GROOVY_STATIC_ANALYZER_CONFIG) + @Authenticate(AccessType.UPDATE) + @ApiOperation(value = "Update Groovy static analysis configuration") + @Produces(MediaType.APPLICATION_JSON) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Success"), + @ApiResponse(code = 500, message = "Server error updating configuration") + }) + public SuccessResponse setGroovyStaticAnalysisConfig(String body) throws Exception { + try { + HelixAdmin admin = _pinotHelixResourceManager.getHelixAdmin(); + HelixConfigScope configScope = + new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER).forCluster( + _pinotHelixResourceManager.getHelixClusterName()).build(); + Map properties = new TreeMap<>(); + GroovyStaticAnalyzerConfig groovyConfig = GroovyStaticAnalyzerConfig.fromJson(body); + properties.put(CommonConstants.GROOVY_STATIC_ANALYZER_CONFIG, + groovyConfig == null ? null : groovyConfig.toJson()); + admin.setConfig(configScope, properties); + GroovyFunctionEvaluator.setConfig(groovyConfig); + return new SuccessResponse("Updated Groovy Static Analyzer config."); + } catch (IOException e) { + throw new ControllerApplicationException(LOGGER, "Error converting request to cluster config", + Response.Status.BAD_REQUEST, e); + } catch (Exception e) { + throw new ControllerApplicationException(LOGGER, "Failed to update Groovy Static Analyzer config", + Response.Status.INTERNAL_SERVER_ERROR, e); + } + } + + @GET + @Path("/cluster/configs/groovy/staticAnalyzerConfig/default") + @Authorize(targetType = TargetType.CLUSTER, action = Actions.Cluster.GET_GROOVY_STATIC_ANALYZER_CONFIG) + @Produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Get the default configuration for Groovy Static analysis", + notes = "Get the default configuration for Groovy static analysis") + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Success"), + @ApiResponse(code = 500, message = "Internal server error") + }) + public GroovyStaticAnalyzerConfig getDefaultGroovyStaticAnalysisConfig() { + return GroovyStaticAnalyzerConfig.createDefault(); + } } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java b/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java index 96e4f27790a7..2fa066e991fb 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java @@ -99,6 +99,8 @@ public static class Cluster { public static final String UPDATE_INSTANCE_PARTITIONS = "UpdateInstancePartitions"; public static final String GET_RESPONSE_STORE = "GetResponseStore"; public static final String DELETE_RESPONSE_STORE = "DeleteResponseStore"; + public static final String GET_GROOVY_STATIC_ANALYZER_CONFIG = "GetGroovyStaticAnalyzerConfig"; + public static final String UPDATE_GROOVY_STATIC_ANALYZER_CONFIG = "UpdateGroovyStaticAnalyzerConfig"; } // Action names for table diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java index 29e28f475ec2..1f57dd193b99 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java @@ -18,32 +18,123 @@ */ package org.apache.pinot.core.data.function; +import com.fasterxml.jackson.core.JsonProcessingException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.pinot.segment.local.function.GroovyFunctionEvaluator; +import org.apache.pinot.segment.local.function.GroovyStaticAnalyzerConfig; import org.apache.pinot.spi.data.readers.GenericRow; -import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import org.testng.collections.Lists; +import static org.apache.pinot.segment.local.function.GroovyStaticAnalyzerConfig.getDefaultAllowedImports; +import static org.apache.pinot.segment.local.function.GroovyStaticAnalyzerConfig.getDefaultAllowedReceivers; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + /** * Tests Groovy functions for transforming schema columns */ public class GroovyFunctionEvaluatorTest { + @Test + public void testLegalGroovyScripts() + throws JsonProcessingException { + // TODO: Add separate tests for these rules: receivers, imports, static imports, and method names. + List scripts = List.of( + "Groovy({2})", + "Groovy({![\"pinot_minion_totalOutputSegmentSize_Value\"].contains(\"\");2})", + "Groovy({airtime == null ? (arrdelay == null ? 0 : arrdelay.value) : airtime.value; 2}, airtime, arrdelay)" + ); + + GroovyStaticAnalyzerConfig config = new GroovyStaticAnalyzerConfig( + getDefaultAllowedReceivers(), + getDefaultAllowedImports(), + getDefaultAllowedImports(), + List.of("invoke", "execute"), + false); + GroovyFunctionEvaluator.setConfig(config); + + for (String script : scripts) { + GroovyFunctionEvaluator groovyFunctionEvaluator = new GroovyFunctionEvaluator(script); + GenericRow row = new GenericRow(); + Object result = groovyFunctionEvaluator.evaluate(row); + assertEquals(2, result); + } + } + + @Test + public void testIllegalGroovyScripts() + throws JsonProcessingException { + // TODO: Add separate tests for these rules: receivers, imports, static imports, and method names. + List scripts = List.of( + "Groovy({\"ls\".execute()})", + "Groovy({[\"ls\"].execute()})", + "Groovy({System.exit(5)})", + "Groovy({System.metaClass.methods.each { method -> if (method.name.md5() == " + + "\"f24f62eeb789199b9b2e467df3b1876b\") {method.invoke(System, 10)} }})", + "Groovy({System.metaClass.methods.each { method -> if (method.name.reverse() == (\"ti\" + \"xe\")) " + + "{method.invoke(System, 10)} }})", + "groovy({def args = [\"QuickStart\", \"-type\", \"REALTIME\"] as String[]; " + + "org.apache.pinot.tools.admin.PinotAdministrator.main(args); 2})", + "Groovy({return [\"bash\", \"-c\", \"env\"].execute().text})" + ); + + GroovyStaticAnalyzerConfig config = new GroovyStaticAnalyzerConfig( + getDefaultAllowedReceivers(), + getDefaultAllowedImports(), + getDefaultAllowedImports(), + List.of("invoke", "execute"), + false); + GroovyFunctionEvaluator.setConfig(config); + + for (String script : scripts) { + try { + new GroovyFunctionEvaluator(script); + fail("Groovy analyzer failed to catch malicious script"); + } catch (Exception ignored) { + } + } + } + + @Test + public void testUpdatingConfiguration() + throws JsonProcessingException { + // TODO: Figure out how to test this with the singleton initializer + // These tests would pass by default but the configuration will be updated so that they fail + List scripts = List.of( + "Groovy({2})", + "Groovy({![\"pinot_minion_totalOutputSegmentSize_Value\"].contains(\"\");2})", + "Groovy({airtime == null ? (arrdelay == null ? 0 : arrdelay.value) : airtime.value; 2}, airtime, arrdelay)" + ); + + GroovyStaticAnalyzerConfig config = + new GroovyStaticAnalyzerConfig(List.of(), List.of(), List.of(), List.of(), false); + GroovyFunctionEvaluator.setConfig(config); + + for (String script : scripts) { + try { + GroovyFunctionEvaluator groovyFunctionEvaluator = new GroovyFunctionEvaluator(script); + GenericRow row = new GenericRow(); + groovyFunctionEvaluator.evaluate(row); + fail(String.format("Groovy analyzer failed to catch malicious script: %s", script)); + } catch (Exception ignored) { + } + } + } @Test(dataProvider = "groovyFunctionEvaluationDataProvider") public void testGroovyFunctionEvaluation(String transformFunction, List arguments, GenericRow genericRow, Object expectedResult) { GroovyFunctionEvaluator groovyExpressionEvaluator = new GroovyFunctionEvaluator(transformFunction); - Assert.assertEquals(groovyExpressionEvaluator.getArguments(), arguments); + assertEquals(groovyExpressionEvaluator.getArguments(), arguments); Object result = groovyExpressionEvaluator.evaluate(genericRow); - Assert.assertEquals(result, expectedResult); + assertEquals(result, expectedResult); } @DataProvider(name = "groovyFunctionEvaluationDataProvider") @@ -108,20 +199,26 @@ public Object[][] groovyFunctionEvaluationDataProvider() { GenericRow genericRow9 = new GenericRow(); genericRow9.putValue("ArrTime", 101); genericRow9.putValue("ArrTimeV2", null); - entries.add(new Object[]{"Groovy({ArrTimeV2 != null ? ArrTimeV2: ArrTime }, ArrTime, ArrTimeV2)", - Lists.newArrayList("ArrTime", "ArrTimeV2"), genericRow9, 101}); + entries.add(new Object[]{ + "Groovy({ArrTimeV2 != null ? ArrTimeV2: ArrTime }, ArrTime, ArrTimeV2)", + Lists.newArrayList("ArrTime", "ArrTimeV2"), genericRow9, 101 + }); GenericRow genericRow10 = new GenericRow(); String jello = "Jello"; genericRow10.putValue("jello", jello); - entries.add(new Object[]{"Groovy({jello != null ? jello.length() : \"Jello\" }, jello)", - Lists.newArrayList("jello"), genericRow10, 5}); + entries.add(new Object[]{ + "Groovy({jello != null ? jello.length() : \"Jello\" }, jello)", + Lists.newArrayList("jello"), genericRow10, 5 + }); //Invalid groovy script GenericRow genericRow11 = new GenericRow(); genericRow11.putValue("nullValue", null); - entries.add(new Object[]{"Groovy({nullValue == null ? nullValue.length() : \"Jello\" }, nullValue)", - Lists.newArrayList("nullValue"), genericRow11, null}); + entries.add(new Object[]{ + "Groovy({nullValue == null ? nullValue.length() : \"Jello\" }, nullValue)", + Lists.newArrayList("nullValue"), genericRow11, null + }); return entries.toArray(new Object[entries.size()][]); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyStaticAnalyzerConfigTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyStaticAnalyzerConfigTest.java new file mode 100644 index 000000000000..d0a0c1c04934 --- /dev/null +++ b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyStaticAnalyzerConfigTest.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.core.data.function; + +import java.util.Iterator; +import java.util.List; +import org.apache.helix.zookeeper.datamodel.ZNRecord; +import org.apache.pinot.segment.local.function.GroovyStaticAnalyzerConfig; +import org.apache.pinot.spi.utils.JsonUtils; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + + +/** + * Test serialization and deserialization. + */ +public class GroovyStaticAnalyzerConfigTest { + @Test + public void testEmptyConfig() + throws Exception { + GroovyStaticAnalyzerConfig config = new GroovyStaticAnalyzerConfig(null, null, null, null, false); + String encodedConfig = JsonUtils.objectToString(config); + GroovyStaticAnalyzerConfig decodedConfig = + JsonUtils.stringToObject(encodedConfig, GroovyStaticAnalyzerConfig.class); + + Assert.assertNull(decodedConfig.getAllowedReceivers()); + Assert.assertNull(decodedConfig.getAllowedImports()); + Assert.assertNull(decodedConfig.getAllowedStaticImports()); + Assert.assertNull(decodedConfig.getDisallowedMethodNames()); + } + + @Test + public void testAllowedReceivers() + throws Exception { + GroovyStaticAnalyzerConfig config = new GroovyStaticAnalyzerConfig( + GroovyStaticAnalyzerConfig.getDefaultAllowedReceivers(), null, null, null, false); + String encodedConfig = JsonUtils.objectToString(config); + GroovyStaticAnalyzerConfig decodedConfig = + JsonUtils.stringToObject(encodedConfig, GroovyStaticAnalyzerConfig.class); + + Assert.assertEquals(GroovyStaticAnalyzerConfig.getDefaultAllowedReceivers(), decodedConfig.getAllowedReceivers()); + Assert.assertNull(decodedConfig.getAllowedImports()); + Assert.assertNull(decodedConfig.getAllowedStaticImports()); + Assert.assertNull(decodedConfig.getDisallowedMethodNames()); + } + + @Test + public void testAllowedImports() + throws Exception { + GroovyStaticAnalyzerConfig config = + new GroovyStaticAnalyzerConfig(null, GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), null, null, false); + String encodedConfig = JsonUtils.objectToString(config); + GroovyStaticAnalyzerConfig decodedConfig = + JsonUtils.stringToObject(encodedConfig, GroovyStaticAnalyzerConfig.class); + + Assert.assertNull(decodedConfig.getAllowedReceivers()); + Assert.assertEquals(GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), decodedConfig.getAllowedImports()); + Assert.assertNull(decodedConfig.getAllowedStaticImports()); + Assert.assertNull(decodedConfig.getDisallowedMethodNames()); + } + + @Test + public void testAllowedStaticImports() + throws Exception { + GroovyStaticAnalyzerConfig config = + new GroovyStaticAnalyzerConfig(null, null, GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), null, false); + String encodedConfig = JsonUtils.objectToString(config); + GroovyStaticAnalyzerConfig decodedConfig = + JsonUtils.stringToObject(encodedConfig, GroovyStaticAnalyzerConfig.class); + + Assert.assertNull(decodedConfig.getAllowedReceivers()); + Assert.assertNull(decodedConfig.getAllowedImports()); + Assert.assertEquals(GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), decodedConfig.getAllowedStaticImports()); + Assert.assertNull(decodedConfig.getDisallowedMethodNames()); + Assert.assertFalse(decodedConfig.isMethodDefinitionAllowed()); + } + + @Test + public void testDisallowedMethodNames() + throws Exception { + GroovyStaticAnalyzerConfig config = + new GroovyStaticAnalyzerConfig(null, null, null, List.of("method1", "method2"), false); + String encodedConfig = JsonUtils.objectToString(config); + GroovyStaticAnalyzerConfig decodedConfig = + JsonUtils.stringToObject(encodedConfig, GroovyStaticAnalyzerConfig.class); + + Assert.assertNull(decodedConfig.getAllowedReceivers()); + Assert.assertNull(decodedConfig.getAllowedImports()); + Assert.assertNull(decodedConfig.getAllowedStaticImports()); + Assert.assertEquals(List.of("method1", "method2"), decodedConfig.getDisallowedMethodNames()); + } + + @Test(dataProvider = "config_provider") + public void testToZnRecord(GroovyStaticAnalyzerConfig config) + throws Exception { + ZNRecord zr = config.toZNRecord(); + GroovyStaticAnalyzerConfig znConfig = GroovyStaticAnalyzerConfig.fromZNRecord(zr); + Assert.assertTrue(equals(znConfig, config)); + } + + @DataProvider(name = "config_provider") + Iterator configProvider() { + return List.of( + new GroovyStaticAnalyzerConfig(null, null, null, List.of("method1", "method2"), false), + new GroovyStaticAnalyzerConfig( + GroovyStaticAnalyzerConfig.getDefaultAllowedReceivers(), null, null, null, false), + new GroovyStaticAnalyzerConfig(null, GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), null, null, false), + new GroovyStaticAnalyzerConfig(null, null, GroovyStaticAnalyzerConfig.getDefaultAllowedImports(), null, false), + new GroovyStaticAnalyzerConfig(null, null, null, List.of("method1", "method2"), false) + ).iterator(); + } + + private boolean equals(GroovyStaticAnalyzerConfig a, GroovyStaticAnalyzerConfig b) { + return a != null && b != null + && (a.getAllowedStaticImports() == b.getAllowedStaticImports() + || a.getAllowedStaticImports().equals(b.getAllowedStaticImports())) + && (a.getAllowedImports() == null && b.getAllowedImports() == null + || a.getAllowedImports().equals(b.getAllowedImports())) + && (a.getAllowedReceivers() == null && b.getAllowedReceivers() == null + || a.getAllowedReceivers().equals(b.getAllowedReceivers())) + && (a.getDisallowedMethodNames() == null && b.getDisallowedMethodNames() == null + || a.getDisallowedMethodNames().equals(b.getDisallowedMethodNames())); + } +} diff --git a/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java b/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java index 8dab73386fed..b88eda178204 100644 --- a/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java +++ b/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java @@ -61,6 +61,7 @@ import org.apache.pinot.minion.executor.PinotTaskExecutorFactory; import org.apache.pinot.minion.executor.TaskExecutorFactoryRegistry; import org.apache.pinot.minion.taskfactory.TaskFactoryRegistry; +import org.apache.pinot.segment.local.function.GroovyFunctionEvaluator; import org.apache.pinot.spi.crypt.PinotCrypterFactory; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.filesystem.PinotFSFactory; @@ -272,6 +273,10 @@ public void start() minionContext.setSSLContext(sslContext); } + // Initializing Groovy execution security + GroovyFunctionEvaluator.configureGroovySecurity(_config.getProperty(CommonConstants.GROOVY_STATIC_ANALYZER_CONFIG)); + GroovyFunctionEvaluator.setMetrics(minionMetrics, MinionMeter.GROOVY_SECURITY_VIOLATIONS); + // Join the Helix cluster LOGGER.info("Joining the Helix cluster"); _helixManager.getStateMachineEngine().registerStateModelFactory("Task", new TaskStateModelFactory(_helixManager, diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/function/GroovyFunctionEvaluator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/function/GroovyFunctionEvaluator.java index 52f36465bb16..d324b8949bc9 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/function/GroovyFunctionEvaluator.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/function/GroovyFunctionEvaluator.java @@ -18,6 +18,7 @@ */ package org.apache.pinot.segment.local.function; +import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import groovy.lang.Binding; @@ -27,7 +28,14 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.pinot.common.metrics.AbstractMetrics; import org.apache.pinot.spi.data.readers.GenericRow; +import org.codehaus.groovy.ast.expr.MethodCallExpression; +import org.codehaus.groovy.control.CompilerConfiguration; +import org.codehaus.groovy.control.customizers.ImportCustomizer; +import org.codehaus.groovy.control.customizers.SecureASTCustomizer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -45,6 +53,7 @@ * ] */ public class GroovyFunctionEvaluator implements FunctionEvaluator { + private static final Logger LOGGER = LoggerFactory.getLogger(GroovyFunctionEvaluator.class); private static final String GROOVY_EXPRESSION_PREFIX = "Groovy"; private static final String GROOVY_FUNCTION_REGEX = "Groovy\\(\\{(?