Improve resiliency when executor plugins are loaded (#14747)

pinot-spi/src/main/java/org/apache/pinot/spi/executor/ExecutorServiceUtils.java

@@ -19,11 +19,14 @@
 package org.apache.pinot.spi.executor;
 
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.ServiceConfigurationError;
 import java.util.ServiceLoader;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 import org.apache.pinot.spi.env.PinotConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,14 +54,38 @@ public class ExecutorServiceUtils {
 
   static {
     PROVIDERS = new HashMap<>();
-    for (ExecutorServicePlugin plugin : ServiceLoader.load(ExecutorServicePlugin.class)) {
+    forEachExecutorThatLoads(plugin -> {
       ExecutorServiceProvider provider = plugin.provider();
       ExecutorServiceProvider old = PROVIDERS.put(plugin.id(), provider);
       if (old != null) {
         LOGGER.warn("Duplicate executor provider for id '{}': {} and {}", plugin.id(), old, provider);
       } else {
         LOGGER.info("Registered executor provider for id '{}': {}", plugin.id(), provider);
       }
+    });
+  }
+
+  private static void forEachExecutorThatLoads(Consumer<ExecutorServicePlugin> consumer) {
+    Iterator<ExecutorServicePlugin> iterator = ServiceLoader.load(ExecutorServicePlugin.class).iterator();
+    while (hasNextOrSkip(iterator)) {
+      ExecutorServicePlugin next;
+      try {
+        next = iterator.next();
+      } catch (ServiceConfigurationError e) {
+        LOGGER.warn("Skipping executor service plugin that doesn't load", e);
+        continue;
+      }
+      consumer.accept(next);
+    }
+  }
+
+  private static boolean hasNextOrSkip(Iterator<ExecutorServicePlugin> loader) {
+    while (true) {
+      try {
+        return loader.hasNext();
+      } catch (ServiceConfigurationError e) {
+        LOGGER.warn("Skipping executor service plugin", e);
+      }
     }
   }