From d5b3211db2bc030753b20538f350d74a84385b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Wed, 16 Aug 2023 12:39:27 +0200 Subject: [PATCH 01/52] Added a simple tool to delete spam projects. --- .../controller/tools/GitlabCleaner.java | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 platform-controller/src/main/java/org/hobbit/controller/tools/GitlabCleaner.java diff --git a/platform-controller/src/main/java/org/hobbit/controller/tools/GitlabCleaner.java b/platform-controller/src/main/java/org/hobbit/controller/tools/GitlabCleaner.java new file mode 100644 index 00000000..ee436be2 --- /dev/null +++ b/platform-controller/src/main/java/org/hobbit/controller/tools/GitlabCleaner.java @@ -0,0 +1,93 @@ +package org.hobbit.controller.tools; + +import java.util.List; +import java.util.Scanner; +import java.util.stream.Collectors; + +import org.gitlab.api.GitlabAPI; +import org.gitlab.api.models.GitlabBranch; +import org.gitlab.api.models.GitlabProject; +import org.gitlab.api.models.GitlabRepositoryFile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GitlabCleaner { + + private static final Logger LOGGER = LoggerFactory.getLogger(GitlabCleaner.class); + + public GitlabCleaner() { + super(); + } + + public void run(String url, String token) { + final GitlabAPI api = GitlabAPI.connect(url, token); + + // https://docs.gitlab.com/ee/api/projects.html#list-all-projects + // Get a list of all visible projects across GitLab for the authenticated user. + // When accessed without authentication, only public projects with simple fields + // are returned. + LOGGER.info("Requesting projects..."); + List gitProjects = api.getProjects(); + int projectsCount = gitProjects.size(); + + List spamProjects = gitProjects.stream() + // Filter all projects that couldn't be converted + .filter(p -> isSpamProject(p, api)) + // Put remaining projects in a map + .collect(Collectors.toList()); + + for (GitlabProject project : spamProjects) { + LOGGER.info("Deleting {}...", project.getNameWithNamespace()); + // api.deleteProject(project.getId()); + } + LOGGER.info("Deleted {}/{} projects", spamProjects.size(), projectsCount); + } + + protected boolean isSpamProject(GitlabProject project, GitlabAPI api) { + return hasNoFile(project, api) && hasNoImage(project, api); + } + + protected boolean hasNoFile(GitlabProject project, GitlabAPI api) { + String defaultBranch = project.getDefaultBranch(); + // If there is no default branch + if ((defaultBranch == null) || ("".equals(defaultBranch))) { + return true; + } + + GitlabBranch branch; + try { + branch = api.getBranch(project, project.getDefaultBranch()); + } catch (Exception e) { + // there is no default graph -> the project is empty + return true; + } + return !hasFile("system.ttl", project, branch, api) && !hasFile("benchmark.ttl", project, branch, api); + } + + protected boolean hasFile(String fileName, GitlabProject project, GitlabBranch branch, GitlabAPI api) { + GitlabRepositoryFile file = null; + try { + file = api.getRepositoryFile(project, fileName, branch.getName()); + } catch (Exception e) { + // nothing to do + } + return ((file != null) && (file.getFileName().contains(fileName))); + } + + protected boolean hasNoImage(GitlabProject project, GitlabAPI api) { + return !project.isContainerRegistryEnabled(); + } + + public static void main(String[] args) { + if (args.length < 2) { + System.err.println( + "Error: wrong usage. The following parameters are necessary:\n "); + return; + } + String url = args[0]; + String token = args[1]; + + GitlabCleaner cleaner = new GitlabCleaner(); + cleaner.run(url, token); + } +} From 4b857f9e891e0c576ec4df0d81cc0988ad5b3189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Mon, 11 Sep 2023 18:52:11 +0200 Subject: [PATCH 02/52] Increased the version of the core library and updated the code accordingly. --- parent-pom/pom.xml | 7 +- platform-controller/pom.xml | 2 +- .../hobbit/controller/ExperimentManager.java | 88 +++++++-------- .../hobbit/controller/PlatformController.java | 2 +- .../main/java/org/hobbit/controller/Temp.java | 68 ++++++++++++ .../data/NodeHardwareInformation.java | 42 +++---- .../data/SetupHardwareInformation.java | 9 +- .../controller/docker/ContainerManager.java | 9 +- .../docker/ContainerManagerImpl.java | 103 ++++++++++-------- .../controller/front/FrontEndApiHandler.java | 2 +- .../test/ControllerStatusRequest.java | 6 +- .../test/RequestBenchmarkDetails.java | 6 +- .../controller/test/RequestBenchmarks.java | 8 +- .../test/RequestSystemResources.java | 3 +- .../test/StartBenchmarkRequest.java | 6 +- .../test/TriggerAllCorrelationAnalysis.java | 8 +- .../docker/MetaDataFactoryTest.java | 7 +- .../mocks/DummyContainerManager.java | 15 ++- 18 files changed, 239 insertions(+), 152 deletions(-) create mode 100644 platform-controller/src/main/java/org/hobbit/controller/Temp.java diff --git a/parent-pom/pom.xml b/parent-pom/pom.xml index 1fc568b1..a0d73137 100644 --- a/parent-pom/pom.xml +++ b/parent-pom/pom.xml @@ -3,12 +3,11 @@ 4.0.0 org.hobbit parent - ${hobbitplatform.version} + 2.0.17-SNAPSHOT pom - 2.0.16 1.8 1.7.10 4.4.0 @@ -68,13 +67,13 @@ org.hobbit core - 1.0.19 + 1.0.22-SNAPSHOT com.rabbitmq amqp-client - 4.8.0 + 5.7.3 diff --git a/platform-controller/pom.xml b/platform-controller/pom.xml index dd3ca19b..bf303218 100644 --- a/platform-controller/pom.xml +++ b/platform-controller/pom.xml @@ -22,7 +22,7 @@ org.hobbit parent - ${hobbitplatform.version} + 2.0.17-SNAPSHOT ../parent-pom platform-controller diff --git a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java index 678c824e..64638c67 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java @@ -19,7 +19,10 @@ import java.io.Closeable; import java.io.IOException; import java.io.StringWriter; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.Timer; @@ -71,12 +74,13 @@ public class ExperimentManager implements Closeable { private static final int DEFAULT_MAX_EXECUTION_TIME = 20 * 60 * 1000; /** - * Key of the environmental variable used to define - * which docker image to use as RabbitMQ. + * Key of the environmental variable used to define which docker image to use as + * RabbitMQ. */ private static final String RABBIT_IMAGE_ENV_KEY = "HOBBIT_RABBIT_IMAGE"; /** - * Environmental variable key for the RabbitMQ broker host name used for experiments. + * Environmental variable key for the RabbitMQ broker host name used for + * experiments. */ private static final String RABBIT_MQ_EXPERIMENTS_HOST_NAME_KEY = "HOBBIT_RABBIT_EXPERIMENTS_HOST"; /** @@ -177,15 +181,14 @@ public void createNextExperiment() { } LOGGER.info("Creating next experiment " + config.id + " with benchmark " + config.benchmarkUri + " and system " + config.systemUri + " to the queue."); - experimentStatus = new ExperimentStatus(config, - HobbitExperiments.getExperimentURI(config.id)); + experimentStatus = new ExperimentStatus(config, HobbitExperiments.getExperimentURI(config.id)); createRabbitMQ(config); BenchmarkMetaData benchmark = controller.imageManager().getBenchmark(config.benchmarkUri); if ((benchmark == null) || (benchmark.mainImage == null)) { - experimentStatus = new ExperimentStatus(config, - HobbitExperiments.getExperimentURI(config.id), this, defaultMaxExecutionTime); + experimentStatus = new ExperimentStatus(config, HobbitExperiments.getExperimentURI(config.id), + this, defaultMaxExecutionTime); experimentStatus.addError(HobbitErrors.BenchmarkImageMissing); throw new Exception("Couldn't find image name for benchmark " + config.benchmarkUri); } @@ -239,12 +242,13 @@ public void createNextExperiment() { LOGGER.info("Creating benchmark controller " + benchmark.mainImage); String containerId = controller.containerManager.startContainer(benchmark.mainImage, Constants.CONTAINER_TYPE_BENCHMARK, experimentStatus.getRootContainer(), - new String[] { Constants.RABBIT_MQ_HOST_NAME_KEY + "=" + experimentStatus.getRabbitMQContainer(), + new String[] { + Constants.RABBIT_MQ_HOST_NAME_KEY + "=" + experimentStatus.getRabbitMQContainer(), Constants.HOBBIT_SESSION_ID_KEY + "=" + config.id, Constants.HOBBIT_EXPERIMENT_URI_KEY + "=" + experimentStatus.experimentUri, Constants.BENCHMARK_PARAMETERS_MODEL_KEY + "=" + config.serializedBenchParams, Constants.SYSTEM_URI_KEY + "=" + config.systemUri }, - null, null, config.id); + null, null, config.id, Collections.emptyMap()); if (containerId == null) { experimentStatus.addError(HobbitErrors.BenchmarkCreationError); throw new Exception("Couldn't create benchmark controller " + config.benchmarkUri); @@ -259,10 +263,11 @@ public void createNextExperiment() { String serializedSystemParams = getSerializedSystemParams(config, benchmark, system); containerId = controller.containerManager.startContainer(system.mainImage, Constants.CONTAINER_TYPE_SYSTEM, experimentStatus.getRootContainer(), - new String[] { Constants.RABBIT_MQ_HOST_NAME_KEY + "=" + experimentStatus.getRabbitMQContainer(), + new String[] { + Constants.RABBIT_MQ_HOST_NAME_KEY + "=" + experimentStatus.getRabbitMQContainer(), Constants.HOBBIT_SESSION_ID_KEY + "=" + config.id, Constants.SYSTEM_PARAMETERS_MODEL_KEY + "=" + serializedSystemParams }, - null, null, config.id); + null, null, config.id, benchmark.getSystemHardwareConstraints()); if (containerId == null) { LOGGER.error("Couldn't start the system. Trying to cancel the benchmark."); forceBenchmarkTerminate_unsecured(HobbitErrors.SystemCreationError); @@ -284,13 +289,12 @@ public void createNextExperiment() { } private void createRabbitMQ(ExperimentConfiguration config) throws Exception { - String rabbitMQAddress = EnvVariables.getString(RABBIT_MQ_EXPERIMENTS_HOST_NAME_KEY, (String)null); + String rabbitMQAddress = EnvVariables.getString(RABBIT_MQ_EXPERIMENTS_HOST_NAME_KEY, (String) null); if (rabbitMQAddress == null) { LOGGER.info("Starting new RabbitMQ for the experiment..."); rabbitMQAddress = controller.containerManager.startContainer(EnvVariables.getString(RABBIT_IMAGE_ENV_KEY), - Constants.CONTAINER_TYPE_BENCHMARK, null, - new String[] { }, - null, null, config.id); + Constants.CONTAINER_TYPE_BENCHMARK, null, new String[] {}, null, null, config.id, + Collections.emptyMap()); if (rabbitMQAddress == null) { experimentStatus.addError(HobbitErrors.UnexpectedError); // FIXME throw new Exception("Couldn't start new RabbitMQ for the experiment"); @@ -303,7 +307,8 @@ private void createRabbitMQ(ExperimentConfiguration config) throws Exception { } experimentStatus.setRabbitMQContainer(rabbitMQAddress); - RabbitMQConnector rabbitMQConnector = new RabbitMQConnector(controller, experimentStatus.getRabbitMQContainer()); + RabbitMQConnector rabbitMQConnector = new RabbitMQConnector(controller, + experimentStatus.getRabbitMQContainer()); controller.setExpRabbitMQConnector(rabbitMQConnector); rabbitMQConnector.init(); } @@ -355,13 +360,10 @@ protected void prefetchImages(BenchmarkMetaData benchmark, SystemMetaData system * {@link #experimentStatus} object and therefore blocking all other operations * on that object. * - * @param sessionId - * the experiment ID to which the result model belongs to - * @param data - * binary data containing a serialized RDF model - * @param function - * a deserialization function transforming the binary data into an - * RDF model + * @param sessionId the experiment ID to which the result model belongs to + * @param data binary data containing a serialized RDF model + * @param function a deserialization function transforming the binary data into + * an RDF model */ public void setResultModel(String sessionId, byte[] data, Function function) { synchronized (experimentMutex) { @@ -377,8 +379,7 @@ public void setResultModel(String sessionId, byte[] data, Functionnull it is added to the result model of the * experiment. * - * @param error - * error that is added to the result model of the experiment + * @param error error that is added to the result model of the experiment */ private void forceBenchmarkTerminate_unsecured(Resource error) { if (experimentStatus != null) { @@ -525,10 +524,8 @@ private void forceBenchmarkTerminate_unsecured(Resource error) { * Handles the termination of the container with the given container Id and the * given exit code. * - * @param containerId - * Id of the terminated container - * @param exitCode - * exit code of the termination + * @param containerId Id of the terminated container + * @param exitCode exit code of the termination */ public void notifyTermination(String containerId, long exitCode) { boolean consumed = false; @@ -590,9 +587,9 @@ public void notifyTermination(String containerId, long exitCode) { * Handles the messages that either the system or the benchmark controller are * ready. * - * @param systemReportedReady - * true if the message was sent by the system, - * false if the benchmark controller is ready + * @param systemReportedReady true if the message was sent by the + * system, false if the benchmark + * controller is ready */ public void systemOrBenchmarkReady(boolean systemReportedReady, String sessionId) { synchronized (experimentMutex) { @@ -626,9 +623,9 @@ public void systemOrBenchmarkReady(boolean systemReportedReady, String sessionId /** * Sends the start message to the benchmark controller. * - * @throws IOException - * if there is a communication problem or if the name of the system - * container can not be retrieved from the docker daemon + * @throws IOException if there is a communication problem or if the name of the + * system container can not be retrieved from the docker + * daemon */ private void startBenchmark_unsecured() throws IOException { String containerName = controller.containerManager.getContainerName(experimentStatus.getSystemContainer()); @@ -652,8 +649,7 @@ private void startBenchmark_unsecured() throws IOException { /** * Adds the status of the current experiment to the given status object. * - * @param status - * the status object to which the data should be added + * @param status the status object to which the data should be added */ public void addStatusInfo(ControllerStatus status, String userName) { // copy the pointer to the experiment status to make sure that we can @@ -702,10 +698,9 @@ public void taskGenFinished(String sessionId) { * Called by the {@link ExperimentAbortTimerTask} if the maximum runtime of an * experiment has been reached. * - * @param expiredState - * the experiment status the timer was working on which is used to - * make sure that the timer was started for the currently running - * experiment. + * @param expiredState the experiment status the timer was working on which is + * used to make sure that the timer was started for the + * currently running experiment. */ public void notifyExpRuntimeExpired(ExperimentStatus expiredState) { Objects.requireNonNull(expiredState); @@ -728,8 +723,7 @@ public void notifyExpRuntimeExpired(ExperimentStatus expiredState) { /** * Stops the currently running experiment if it has the given experiment id. * - * @param experimentId - * the id of the experiment that should be stopped + * @param experimentId the id of the experiment that should be stopped */ public void stopExperimentIfRunning(String experimentId) { synchronized (experimentMutex) { diff --git a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java index f31a7883..21f64def 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java +++ b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java @@ -481,7 +481,7 @@ private String createContainer(StartCommandData data) { } String containerId = containerManager.startContainer(data.image, data.type, parentId, data.environmentVariables, - data.networkAliases, null, pullImage); + data.networkAliases, null, pullImage, null); if (containerId == null) { return null; } else { diff --git a/platform-controller/src/main/java/org/hobbit/controller/Temp.java b/platform-controller/src/main/java/org/hobbit/controller/Temp.java new file mode 100644 index 00000000..d736945d --- /dev/null +++ b/platform-controller/src/main/java/org/hobbit/controller/Temp.java @@ -0,0 +1,68 @@ +package org.hobbit.controller; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.List; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.vocabulary.RDF; +import org.hobbit.utils.rdf.RdfHelper; +import org.hobbit.vocab.HOBBIT; +import org.hobbit.vocab.HobbitExperiments; + +public class Temp { + + public static void main(String[] args) { + Model model = ModelFactory.createDefaultModel(); + try (InputStream in = new FileInputStream("a2kb-results.ttl")) { + model.read(in, "", "TTL"); + } catch (Exception e) { + e.printStackTrace(); + return; + } + Property dataset = model.getProperty("http://w3id.org/gerbil/hobbit/vocab#hasDataset"); + Property micPre = model.getProperty("http://w3id.org/gerbil/vocab#microPrecision"); + Property micRec = model.getProperty("http://w3id.org/gerbil/vocab#microRecall"); + Property micF1 = model.getProperty("http://w3id.org/gerbil/vocab#microF1"); + Property macPre = model.getProperty("http://w3id.org/gerbil/vocab#macroPrecision"); + Property macRec = model.getProperty("http://w3id.org/gerbil/vocab#macroRecall"); + Property macF1 = model.getProperty("http://w3id.org/gerbil/vocab#macroF1"); + Property errors = model.getProperty("http://w3id.org/gerbil/vocab#errorCount"); + Property avgMillis = model.getProperty("http://w3id.org/gerbil/vocab#avgMillisPerDoc"); + System.out.println("\"ID\",\"system\",\"dataset\",\"mic. P\",\"mic. R\",\"mic. F1\",\"mac. P\", \"mac. R\", \"mac. F1\",\"errors\",\"avg millis/doc\""); + List experiments = RdfHelper.getSubjectResources(model, RDF.type, HOBBIT.Experiment); + for (Resource experiment : experiments) { + System.out.print('"'); + System.out.print(HobbitExperiments.getExperimentId(experiment)); + System.out.print("\",\""); + System.out.print(RdfHelper.getLabel(model, RdfHelper.getObjectResource(model, experiment, HOBBIT.involvesSystemInstance))); + System.out.print("\",\""); + System.out.print(RdfHelper.getObjectResource(model, experiment, dataset).getLocalName()); + System.out.print("\",\""); + if (model.contains(experiment, HOBBIT.terminatedWithError, (RDFNode) null)) { + System.out.println("\""); + } else { + System.out.print(RdfHelper.getStringValue(model, experiment, micPre)); + System.out.print("\",\""); + System.out.print(RdfHelper.getStringValue(model, experiment, micRec)); + System.out.print("\",\""); + System.out.print(RdfHelper.getStringValue(model, experiment, micF1)); + System.out.print("\",\""); + System.out.print(RdfHelper.getStringValue(model, experiment, macPre)); + System.out.print("\",\""); + System.out.print(RdfHelper.getStringValue(model, experiment, macRec)); + System.out.print("\",\""); + System.out.print(RdfHelper.getStringValue(model, experiment, macF1)); + System.out.print("\",\""); + System.out.print(RdfHelper.getStringValue(model, experiment, errors)); + System.out.print("\",\""); + System.out.print(RdfHelper.getStringValue(model, experiment, avgMillis)); + System.out.println("\""); + } + } + } +} diff --git a/platform-controller/src/main/java/org/hobbit/controller/data/NodeHardwareInformation.java b/platform-controller/src/main/java/org/hobbit/controller/data/NodeHardwareInformation.java index 4620d9c8..83ac5834 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/data/NodeHardwareInformation.java +++ b/platform-controller/src/main/java/org/hobbit/controller/data/NodeHardwareInformation.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; + import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.Resource; @@ -28,12 +29,13 @@ import org.apache.jena.rdf.model.impl.StmtIteratorImpl; import org.apache.jena.sparql.vocabulary.DOAP; import org.apache.jena.vocabulary.RDFS; -import org.hobbit.utils.rdf.RdfHelper; +import org.hobbit.utils.rdf.TripleHashCalculator; import org.hobbit.vocab.HobbitHardware; import org.hobbit.vocab.MEXCORE; /** - * This class is used to store information about hardware the experiment runs on. + * This class is used to store information about hardware the experiment runs + * on. * * @author Denis Kuchelev * @@ -51,8 +53,7 @@ public class NodeHardwareInformation { /** * Formats a frequency value. * - * @param frequency - * the frequency value in Hz + * @param frequency the frequency value in Hz * @return formatted value with a unit of measure */ private String formatFrequencyValue(Long frequency) { @@ -62,8 +63,7 @@ private String formatFrequencyValue(Long frequency) { /** * Formats a memory amount value. * - * @param memory - * the memory value in B + * @param memory the memory value in B * @return formatted value with a unit of measure */ private String formatMemoryValue(Long memory) { @@ -86,8 +86,7 @@ public void setCpu(Long cores, List frequencies) { } if (frequencies.size() != 0) { - builder - .append("(") + builder.append("(") .append(frequencies.stream().map(this::formatFrequencyValue).collect(Collectors.joining(", "))) .append(")"); } @@ -123,8 +122,8 @@ public void setOs(String os) { private String hash() { Model dummyModel = ModelFactory.createDefaultModel(); - Resource dummyRes = dummyModel.createResource(RdfHelper.HASH_SELF_URI); - return RdfHelper.hashProperties(distinguishingProperties(dummyModel, dummyRes)); + Resource dummyRes = dummyModel.createResource(TripleHashCalculator.HASH_SELF_URI); + return TripleHashCalculator.calculateHash(distinguishingProperties(dummyModel, dummyRes)); } public String getURI() { @@ -138,27 +137,18 @@ public Resource addToModel(Model model) { } private StmtIterator distinguishingProperties(Model model, Resource self) { - return new StmtIteratorImpl(Stream.of( - (Statement) new StatementImpl( - self, RDFS.label, model.createLiteral(instance)), - (Statement) new StatementImpl( - self, MEXCORE.cpu, model.createLiteral(cpu)), - (Statement) new StatementImpl( - self, MEXCORE.memory, model.createLiteral(memory)), - (Statement) new StatementImpl( - self, DOAP.os, model.createLiteral(os)) - ).iterator()); + return new StmtIteratorImpl( + Stream.of((Statement) new StatementImpl(self, RDFS.label, model.createLiteral(instance)), + (Statement) new StatementImpl(self, MEXCORE.cpu, model.createLiteral(cpu)), + (Statement) new StatementImpl(self, MEXCORE.memory, model.createLiteral(memory)), + (Statement) new StatementImpl(self, DOAP.os, model.createLiteral(os))).iterator()); } @Override public String toString() { StringBuilder builder = new StringBuilder(); - builder - .append("NodeHardwareInformation [") - .append("instance=").append(instance).append(", ") - .append("cpu=").append(cpu).append(", ") - .append("memory=").append(memory).append(", ") - .append("os=").append(os) + builder.append("NodeHardwareInformation [").append("instance=").append(instance).append(", ").append("cpu=") + .append(cpu).append(", ").append("memory=").append(memory).append(", ").append("os=").append(os) .append("]"); return builder.toString(); } diff --git a/platform-controller/src/main/java/org/hobbit/controller/data/SetupHardwareInformation.java b/platform-controller/src/main/java/org/hobbit/controller/data/SetupHardwareInformation.java index f4a67fd7..91297dd7 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/data/SetupHardwareInformation.java +++ b/platform-controller/src/main/java/org/hobbit/controller/data/SetupHardwareInformation.java @@ -18,14 +18,15 @@ import java.util.ArrayList; import java.util.List; + import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.Statement; import org.apache.jena.rdf.model.StmtIterator; -import org.apache.jena.rdf.model.impl.StmtIteratorImpl; import org.apache.jena.rdf.model.impl.StatementImpl; -import org.hobbit.utils.rdf.RdfHelper; +import org.apache.jena.rdf.model.impl.StmtIteratorImpl; +import org.hobbit.utils.rdf.TripleHashCalculator; import org.hobbit.vocab.HOBBIT; import org.hobbit.vocab.HobbitHardware; @@ -48,8 +49,8 @@ public void addNode(NodeHardwareInformation nodeInfo) { private String hash() { Model dummyModel = ModelFactory.createDefaultModel(); - Resource dummyRes = dummyModel.createResource(RdfHelper.HASH_SELF_URI); - return RdfHelper.hashProperties(distinguishingProperties(dummyModel, dummyRes)); + Resource dummyRes = dummyModel.createResource(TripleHashCalculator.HASH_SELF_URI); + return TripleHashCalculator.calculateHash(distinguishingProperties(dummyModel, dummyRes)); } public String getURI() { diff --git a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java index 0275e60e..c623e295 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java @@ -17,6 +17,7 @@ package org.hobbit.controller.docker; import java.util.List; +import java.util.Map; import com.spotify.docker.client.exceptions.DockerException; import com.spotify.docker.client.messages.ContainerStats; @@ -187,11 +188,13 @@ public String startContainer(String imageName, String containerType, String pare * commands that should be executed * @param pullImage * whether the image needs to be prefetched + * @param constraints + * Additional constraints for the container * * @return container Id or null if an error occurred. */ public String startContainer(String imageName, String containerType, String parentId, String[] env, - String[] netAliases, String[] command, boolean pullImage); + String[] netAliases, String[] command, boolean pullImage, Map constraints); /** * Starts the container with the given image name. @@ -208,11 +211,13 @@ public String startContainer(String imageName, String containerType, String pare * commands that should be executed * @param experimentId * experimentId to add to GELF tag + * @param constraints + * Additional constraints for the container * * @return container Id or null if an error occurred. */ public String startContainer(String imageName, String containerType, String parentId, String[] env, - String[] netAliases, String[] command, String experimentId); + String[] netAliases, String[] command, String experimentId, Map constraints); /** * Stops the container with the given container Id. diff --git a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java index e4e7c90f..12ee49be 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java +++ b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java @@ -50,6 +50,8 @@ import com.spotify.docker.client.messages.swarm.Driver; import com.spotify.docker.client.messages.swarm.NetworkAttachmentConfig; import com.spotify.docker.client.messages.swarm.Placement; +import com.spotify.docker.client.messages.swarm.ResourceRequirements; +import com.spotify.docker.client.messages.swarm.Resources; import com.spotify.docker.client.messages.swarm.RestartPolicy; import com.spotify.docker.client.messages.swarm.Service; import com.spotify.docker.client.messages.swarm.ServiceMode; @@ -74,6 +76,8 @@ public class ContainerManagerImpl implements ContainerManager { public static final String USER_EMAIL_KEY = "GITLAB_EMAIL"; public static final String USER_PASSWORD_KEY = GitlabControllerImpl.GITLAB_TOKEN_KEY; public static final String REGISTRY_URL_KEY = "REGISTRY_URL"; + public static final String MEMORY_LIMIT_CONSTRAINT = "memory-limit"; + public static final String NANO_CPU_LIMIT_CONSTRAINT = "nanoCPU-limit"; private static final int DOCKER_MAX_NAME_LENGTH = 63; @@ -188,8 +192,7 @@ public ContainerManagerImpl() throws Exception { /** * Generates new unique instance name based on image name * - * @param imageName - * base image name + * @param imageName base image name * * @return instance name */ @@ -200,10 +203,8 @@ private String getInstanceName(String imageName) { /** * Generates new unique instance name based on image name * - * @param imageName - * base image name - * @param prefix - * additional prefix + * @param imageName base image name + * @param prefix additional prefix * * @return instance name */ @@ -240,7 +241,8 @@ private String getInstanceName(String imageName, String prefix) { private ServiceCreateResponse createService(ServiceSpec serviceSpec) throws DockerException, InterruptedException { // If we have authentication credentials and the image name contains // the server address of these credentials, we should use them - if ((gitlabAuth != null) && (serviceSpec.taskTemplate().containerSpec().image().startsWith(gitlabAuth.serverAddress()))) { + if ((gitlabAuth != null) + && (serviceSpec.taskTemplate().containerSpec().image().startsWith(gitlabAuth.serverAddress()))) { return dockerClient.createService(serviceSpec, gitlabAuth); } else { return dockerClient.createService(serviceSpec, nullAuth); @@ -250,8 +252,7 @@ private ServiceCreateResponse createService(ServiceSpec serviceSpec) throws Dock /** * Pulls the image with the given name. * - * @param imageName - * the name of the image that should be pulled + * @param imageName the name of the image that should be pulled */ public void pullImage(String imageName) { // do not pull if env var is set to false @@ -320,7 +321,8 @@ public void pullImage(String imageName) { if (state.equals(TaskStatus.TASK_STATE_REJECTED)) { LOGGER.error("Couldn't pull image {} on node {}. {}", imageName, pullingTask.nodeId(), pullingTask.status().err()); - throw new Exception("Couldn't pull image on node " + pullingTask.nodeId() + ": " + pullingTask.status().err()); + throw new Exception("Couldn't pull image on node " + pullingTask.nodeId() + ": " + + pullingTask.status().err()); } finshedTaskIds.add(pullingTask.id()); } @@ -348,21 +350,17 @@ public void pullImage(String imageName) { /** * Creates new container using given image and assigns given type and parent * - * @param imageName - * image to use as base for container - * @param containerType - * container type - * @param parentId - * parent id - * @param env - * (optional) environment variables - * @param command - * (optional) command to be executed with image + * @param imageName image to use as base for container + * @param containerType container type + * @param parentId parent id + * @param env (optional) environment variables + * @param command (optional) command to be executed with image + * @param constraints additional constraints * * @return String the container Id or null if an error occurs */ private String createContainer(String imageName, String containerType, String parentId, String[] env, - String[] netAliases, String[] command) { + String[] netAliases, String[] command, Map constraints) { ServiceSpec.Builder serviceCfgBuilder = ServiceSpec.builder(); TaskSpec.Builder taskCfgBuilder = TaskSpec.builder(); @@ -478,6 +476,24 @@ private String createContainer(String imageName, String containerType, String pa logOptions.put("tag", tag); taskCfgBuilder.logDriver(Driver.builder().name(LOGGING_DRIVER_GELF).options(logOptions).build()); } + // If there are resource limitations + if (constraints != null && (constraints.containsKey(MEMORY_LIMIT_CONSTRAINT) + || constraints.containsKey(NANO_CPU_LIMIT_CONSTRAINT))) { + Resources.Builder rBuilder = Resources.builder(); + // if there is a memory limitation + if (constraints.containsKey(MEMORY_LIMIT_CONSTRAINT)) { + long memory = (Long) constraints.get(MEMORY_LIMIT_CONSTRAINT); + rBuilder.memoryBytes(memory); + } + // if there is a CPU limitation + if (constraints.containsKey(NANO_CPU_LIMIT_CONSTRAINT)) { + // CPU quota in units of 10^-9 CPUs. + long nanoCPUs = (Long) constraints.get(NANO_CPU_LIMIT_CONSTRAINT); + rBuilder.nanoCpus(nanoCPUs); + } + // Add the limitations to the task config + taskCfgBuilder.resources(ResourceRequirements.builder().limits(rBuilder.build()).build()); + } // if command is present - execute it if ((command != null) && (command.length > 0)) { @@ -490,9 +506,8 @@ private String createContainer(String imageName, String containerType, String pa serviceCfgBuilder.taskTemplate(taskCfgBuilder.build()); // connect to hobbit network only - serviceCfgBuilder.networks( - NetworkAttachmentConfig.builder().target(HOBBIT_DOCKER_NETWORK).aliases(netAliases).build() - ); + serviceCfgBuilder + .networks(NetworkAttachmentConfig.builder().target(HOBBIT_DOCKER_NETWORK).aliases(netAliases).build()); serviceCfgBuilder.name(serviceName); ServiceSpec serviceCfg = serviceCfgBuilder.build(); @@ -505,7 +520,8 @@ private String createContainer(String imageName, String containerType, String pa List serviceTasks = new ArrayList(); Waiting.waitFor(() -> { serviceTasks.clear(); - serviceTasks.addAll(dockerClient.listTasks(Task.Criteria.builder().serviceName(serviceIdForLambda).build())); + serviceTasks.addAll( + dockerClient.listTasks(Task.Criteria.builder().serviceName(serviceIdForLambda).build())); if (!serviceTasks.isEmpty()) { TaskStatus status = serviceTasks.get(0).status(); @@ -564,24 +580,25 @@ public String startContainer(String imageName, String containerType, String pare @Override public String startContainer(String imageName, String containerType, String parentId, String[] env, - String[] netAliases, String[] command) { - return startContainer(imageName, containerType, parentId, env, netAliases, command, true); + String[] netAliases, String[] command) { + return startContainer(imageName, containerType, parentId, env, netAliases, command, true, + Collections.emptyMap()); } @Override public String startContainer(String imageName, String containerType, String parentId, String[] env, String[] command, boolean pullImage) { - return startContainer(imageName, containerType, parentId, env, null, command, true); + return startContainer(imageName, containerType, parentId, env, null, command, true, Collections.emptyMap()); } @Override public String startContainer(String imageName, String containerType, String parentId, String[] env, - String[] netAliases, String[] command, boolean pullImage) { + String[] netAliases, String[] command, boolean pullImage, Map constraints) { if (pullImage) { pullImage(imageName); } - String containerId = createContainer(imageName, containerType, parentId, env, netAliases, command); + String containerId = createContainer(imageName, containerType, parentId, env, netAliases, command, constraints); // if the creation was successful if (containerId != null) { @@ -595,9 +612,9 @@ public String startContainer(String imageName, String containerType, String pare @Override public String startContainer(String imageName, String containerType, String parentId, String[] env, - String[] netAliases, String[] command, String experimentId) { + String[] netAliases, String[] command, String experimentId, Map constraints) { this.experimentId = experimentId; - return startContainer(imageName, containerType, parentId, env, netAliases, command); + return startContainer(imageName, containerType, parentId, env, netAliases, command, true, constraints); } @Override @@ -605,16 +622,11 @@ public void removeContainer(String serviceName) { try { Long exitCode = getContainerExitCode(serviceName); if (DEPLOY_ENV.equals(DEPLOY_ENV_DEVELOP)) { - LOGGER.info( - "Will not remove container {}. " - + "Development mode is enabled.", - serviceName); + LOGGER.info("Will not remove container {}. " + "Development mode is enabled.", serviceName); } else if (DEPLOY_ENV.equals(DEPLOY_ENV_TESTING) && (exitCode != null && exitCode != 0)) { // In testing - do not remove containers if they returned non-zero exit code // null exit code usually means that the container is running at the moment - LOGGER.info( - "Will not remove container {}. " - + "ExitCode: {} != 0 and testing mode is enabled.", + LOGGER.info("Will not remove container {}. " + "ExitCode: {} != 0 and testing mode is enabled.", serviceName, exitCode); } else { LOGGER.info("Removing container {}. ", serviceName); @@ -658,7 +670,8 @@ public void removeParentAndChildren(String parent) { // find children try { - List services = dockerClient.listServices(Service.Criteria.builder().labels(ImmutableMap.of(LABEL_PARENT, parent)).build()); + List services = dockerClient + .listServices(Service.Criteria.builder().labels(ImmutableMap.of(LABEL_PARENT, parent)).build()); for (Service c : services) { if (c != null) { @@ -696,14 +709,17 @@ public List getContainers(Service.Criteria criteria) { @Override public Long getContainerExitCode(String serviceName) throws DockerException, InterruptedException { if (getContainerInfo(serviceName) == null) { - LOGGER.warn("Couldn't get the exit code for container {}. Service doesn't exist. Assuming it was stopped by the platform.", serviceName); + LOGGER.warn( + "Couldn't get the exit code for container {}. Service doesn't exist. Assuming it was stopped by the platform.", + serviceName); return DOCKER_EXITCODE_SIGKILL; } // Service exists, but no tasks are observed. List tasks = dockerClient.listTasks(Task.Criteria.builder().serviceName(serviceName).build()); if (tasks.size() == 0) { - LOGGER.warn("Couldn't get the exit code for container {}. Service has no tasks. Returning null.", serviceName); + LOGGER.warn("Couldn't get the exit code for container {}. Service has no tasks. Returning null.", + serviceName); return null; } @@ -712,7 +728,8 @@ public Long getContainerExitCode(String serviceName) throws DockerException, Int // Task is finished. Long exitCode = task.status().containerStatus().exitCode(); if (exitCode == null) { - LOGGER.warn("Couldn't get the exit code for container {}. Task is finished. Returning 0.", serviceName); + LOGGER.warn("Couldn't get the exit code for container {}. Task is finished. Returning 0.", + serviceName); return 0l; } return exitCode; diff --git a/platform-controller/src/main/java/org/hobbit/controller/front/FrontEndApiHandler.java b/platform-controller/src/main/java/org/hobbit/controller/front/FrontEndApiHandler.java index 2aea84d4..734fda42 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/front/FrontEndApiHandler.java +++ b/platform-controller/src/main/java/org/hobbit/controller/front/FrontEndApiHandler.java @@ -28,7 +28,7 @@ import com.rabbitmq.client.AMQP.BasicProperties; import com.rabbitmq.client.DefaultConsumer; -import com.rabbitmq.client.QueueingConsumer.Delivery; +import com.rabbitmq.client.Delivery; /** * This class implements a RabbitMQ {@link DefaultConsumer} and handles request diff --git a/platform-controller/src/main/java/org/hobbit/controller/test/ControllerStatusRequest.java b/platform-controller/src/main/java/org/hobbit/controller/test/ControllerStatusRequest.java index 365382e1..93c3d124 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/test/ControllerStatusRequest.java +++ b/platform-controller/src/main/java/org/hobbit/controller/test/ControllerStatusRequest.java @@ -17,12 +17,14 @@ package org.hobbit.controller.test; import java.io.IOException; +import java.util.concurrent.TimeUnit; import org.hobbit.core.Constants; import org.hobbit.core.FrontEndApiCommands; import org.hobbit.core.components.AbstractCommandReceivingComponent; import org.hobbit.core.data.status.ControllerStatus; import org.hobbit.core.data.status.QueuedExperiment; +import org.hobbit.core.rabbit.QueueingConsumer; import org.hobbit.core.rabbit.RabbitMQUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +32,7 @@ import com.google.gson.Gson; import com.rabbitmq.client.AMQP.BasicProperties; import com.rabbitmq.client.Channel; -import com.rabbitmq.client.QueueingConsumer; +import com.rabbitmq.client.Delivery; public class ControllerStatusRequest extends AbstractCommandReceivingComponent { @@ -69,7 +71,7 @@ public void run() throws Exception { frontEnd2Controller.basicPublish("", Constants.FRONT_END_2_CONTROLLER_QUEUE_NAME, props, new byte[] { FrontEndApiCommands.LIST_CURRENT_STATUS }); LOGGER.info("Waiting for response..."); - QueueingConsumer.Delivery delivery = consumer.nextDelivery(REQUEST_TIMEOUT); + Delivery delivery = consumer.getDeliveryQueue().poll(REQUEST_TIMEOUT, TimeUnit.MILLISECONDS); if (delivery == null) { throw new IOException("Didn't got a response after \"" + REQUEST_TIMEOUT + "\" ms."); } diff --git a/platform-controller/src/main/java/org/hobbit/controller/test/RequestBenchmarkDetails.java b/platform-controller/src/main/java/org/hobbit/controller/test/RequestBenchmarkDetails.java index 17d14d40..9d6b6d00 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/test/RequestBenchmarkDetails.java +++ b/platform-controller/src/main/java/org/hobbit/controller/test/RequestBenchmarkDetails.java @@ -21,12 +21,14 @@ import java.nio.ByteBuffer; import java.util.Collection; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.apache.jena.rdf.model.Model; import org.hobbit.core.Constants; import org.hobbit.core.FrontEndApiCommands; import org.hobbit.core.components.AbstractCommandReceivingComponent; import org.hobbit.core.data.SystemMetaData; +import org.hobbit.core.rabbit.QueueingConsumer; import org.hobbit.core.rabbit.RabbitMQUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +37,7 @@ import com.google.gson.reflect.TypeToken; import com.rabbitmq.client.AMQP.BasicProperties; import com.rabbitmq.client.Channel; -import com.rabbitmq.client.QueueingConsumer; +import com.rabbitmq.client.Delivery; public class RequestBenchmarkDetails extends AbstractCommandReceivingComponent { @@ -85,7 +87,7 @@ public void run() throws Exception { RabbitMQUtils.writeByteArrays(new byte[] { FrontEndApiCommands.GET_BENCHMARK_DETAILS }, new byte[][] { RabbitMQUtils.writeString(benchmarkUri), RabbitMQUtils.writeString(userName) }, null)); LOGGER.info("Waiting for response..."); - QueueingConsumer.Delivery delivery = consumer.nextDelivery(REQUEST_TIMEOUT); + Delivery delivery = consumer.getDeliveryQueue().poll(REQUEST_TIMEOUT, TimeUnit.MILLISECONDS); if (delivery == null) { throw new IOException("Didn't got a response after \"" + REQUEST_TIMEOUT + "\" ms."); } diff --git a/platform-controller/src/main/java/org/hobbit/controller/test/RequestBenchmarks.java b/platform-controller/src/main/java/org/hobbit/controller/test/RequestBenchmarks.java index c4b925b7..20200c73 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/test/RequestBenchmarks.java +++ b/platform-controller/src/main/java/org/hobbit/controller/test/RequestBenchmarks.java @@ -18,20 +18,22 @@ import java.io.IOException; import java.util.Collection; +import java.util.concurrent.TimeUnit; import org.hobbit.core.Constants; import org.hobbit.core.FrontEndApiCommands; import org.hobbit.core.components.AbstractCommandReceivingComponent; import org.hobbit.core.data.BenchmarkMetaData; +import org.hobbit.core.rabbit.QueueingConsumer; import org.hobbit.core.rabbit.RabbitMQUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.rabbitmq.client.AMQP.BasicProperties; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import com.rabbitmq.client.AMQP.BasicProperties; import com.rabbitmq.client.Channel; -import com.rabbitmq.client.QueueingConsumer; +import com.rabbitmq.client.Delivery; public class RequestBenchmarks extends AbstractCommandReceivingComponent { @@ -70,7 +72,7 @@ public void run() throws Exception { frontEnd2Controller.basicPublish("", Constants.FRONT_END_2_CONTROLLER_QUEUE_NAME, props, new byte[] { FrontEndApiCommands.LIST_AVAILABLE_BENCHMARKS }); LOGGER.info("Waiting for response..."); - QueueingConsumer.Delivery delivery = consumer.nextDelivery(REQUEST_TIMEOUT); + Delivery delivery = consumer.getDeliveryQueue().poll(REQUEST_TIMEOUT, TimeUnit.MILLISECONDS); if (delivery == null) { throw new IOException("Didn't got a response after \"" + REQUEST_TIMEOUT + "\" ms."); } diff --git a/platform-controller/src/main/java/org/hobbit/controller/test/RequestSystemResources.java b/platform-controller/src/main/java/org/hobbit/controller/test/RequestSystemResources.java index 063b87a6..032a055d 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/test/RequestSystemResources.java +++ b/platform-controller/src/main/java/org/hobbit/controller/test/RequestSystemResources.java @@ -23,12 +23,11 @@ import org.hobbit.core.components.AbstractPlatformConnectorComponent; import org.hobbit.core.components.utils.SystemResourceUsageRequester; import org.hobbit.core.data.usage.ResourceUsageInformation; +import org.hobbit.core.rabbit.QueueingConsumer; import org.hobbit.core.run.ComponentStarter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.rabbitmq.client.QueueingConsumer; - public class RequestSystemResources extends AbstractPlatformConnectorComponent { private static final Logger LOGGER = LoggerFactory.getLogger(RequestSystemResources.class); diff --git a/platform-controller/src/main/java/org/hobbit/controller/test/StartBenchmarkRequest.java b/platform-controller/src/main/java/org/hobbit/controller/test/StartBenchmarkRequest.java index 7de5e476..46f7ebc8 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/test/StartBenchmarkRequest.java +++ b/platform-controller/src/main/java/org/hobbit/controller/test/StartBenchmarkRequest.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.apache.commons.io.IOUtils; import org.apache.jena.rdf.model.Model; @@ -27,13 +28,14 @@ import org.hobbit.core.Constants; import org.hobbit.core.FrontEndApiCommands; import org.hobbit.core.components.AbstractCommandReceivingComponent; +import org.hobbit.core.rabbit.QueueingConsumer; import org.hobbit.core.rabbit.RabbitMQUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.rabbitmq.client.AMQP.BasicProperties; import com.rabbitmq.client.Channel; -import com.rabbitmq.client.QueueingConsumer; +import com.rabbitmq.client.Delivery; public class StartBenchmarkRequest extends AbstractCommandReceivingComponent { @@ -95,7 +97,7 @@ public void run() throws Exception { .replyTo(Constants.CONTROLLER_2_FRONT_END_QUEUE_NAME).build(); frontEnd2Controller.basicPublish("", Constants.FRONT_END_2_CONTROLLER_QUEUE_NAME, props, data); LOGGER.info("Waiting for response..."); - QueueingConsumer.Delivery delivery = consumer.nextDelivery(REQUEST_TIMEOUT); + Delivery delivery = consumer.getDeliveryQueue().poll(REQUEST_TIMEOUT, TimeUnit.MILLISECONDS); if (delivery == null) { throw new IOException( "Didn't got a response after \"" + REQUEST_TIMEOUT + "\" ms."); diff --git a/platform-controller/src/main/java/org/hobbit/controller/test/TriggerAllCorrelationAnalysis.java b/platform-controller/src/main/java/org/hobbit/controller/test/TriggerAllCorrelationAnalysis.java index 342dac86..0de75231 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/test/TriggerAllCorrelationAnalysis.java +++ b/platform-controller/src/main/java/org/hobbit/controller/test/TriggerAllCorrelationAnalysis.java @@ -16,22 +16,20 @@ */ package org.hobbit.controller.test; -import org.hobbit.vocab.HOBBIT; import org.apache.jena.query.QuerySolution; import org.apache.jena.query.ResultSet; -import org.hobbit.storage.client.StorageServiceClient; - import org.hobbit.core.Constants; import org.hobbit.core.components.AbstractPlatformConnectorComponent; import org.hobbit.core.rabbit.DataSender; import org.hobbit.core.rabbit.DataSenderImpl; +import org.hobbit.core.rabbit.QueueingConsumer; import org.hobbit.core.rabbit.RabbitMQUtils; import org.hobbit.core.run.ComponentStarter; +import org.hobbit.storage.client.StorageServiceClient; +import org.hobbit.vocab.HOBBIT; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.rabbitmq.client.QueueingConsumer; - public class TriggerAllCorrelationAnalysis extends AbstractPlatformConnectorComponent { private static final Logger LOGGER = LoggerFactory.getLogger(TriggerAllCorrelationAnalysis.class); diff --git a/platform-controller/src/test/java/org/hobbit/controller/docker/MetaDataFactoryTest.java b/platform-controller/src/test/java/org/hobbit/controller/docker/MetaDataFactoryTest.java index f0e97cb6..17152455 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/docker/MetaDataFactoryTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/docker/MetaDataFactoryTest.java @@ -4,8 +4,10 @@ import java.io.InputStream; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import org.apache.commons.compress.utils.IOUtils; import org.apache.log4j.lf5.util.StreamUtils; @@ -32,6 +34,8 @@ public void testModelToBenchmarkMetaData() throws IOException { expectedMetaData.usedImages.add("hobbit/gerbil_taskgen"); expectedMetaData.date = new Date(10); expectedMetaData.source = "test"; + Map constraints = new HashMap<>(); + expectedMetaData.setSystemHardwareConstraints(constraints); InputStream input = ParameterForwardingTest.class.getClassLoader() .getResourceAsStream("org/hobbit/controller/benchmark.ttl"); @@ -78,7 +82,8 @@ public void testModelToSystemMetaData() throws IOException { expectedMetaData.source, expectedMetaData.date); Assert.assertEquals(1, results.size()); compareMetaData(expectedMetaData, results.get(0)); - String[] expectedApis = expectedMetaData.implementedApis.toArray(new String[expectedMetaData.implementedApis.size()]); + String[] expectedApis = expectedMetaData.implementedApis + .toArray(new String[expectedMetaData.implementedApis.size()]); Arrays.sort(expectedApis); String[] actualApis = results.get(0).implementedApis.toArray(new String[0]); Arrays.sort(actualApis); diff --git a/platform-controller/src/test/java/org/hobbit/controller/mocks/DummyContainerManager.java b/platform-controller/src/test/java/org/hobbit/controller/mocks/DummyContainerManager.java index 6a71ca94..cfcdd7ac 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/mocks/DummyContainerManager.java +++ b/platform-controller/src/test/java/org/hobbit/controller/mocks/DummyContainerManager.java @@ -1,15 +1,18 @@ package org.hobbit.controller.mocks; -import com.spotify.docker.client.messages.ContainerStats; -import com.spotify.docker.client.messages.swarm.Service; -import com.spotify.docker.client.messages.swarm.Service.Criteria; import java.util.ArrayList; -import java.util.concurrent.Semaphore; import java.util.List; +import java.util.Map; +import java.util.concurrent.Semaphore; + import org.hobbit.controller.docker.ContainerManager; import org.hobbit.controller.docker.ContainerStateObserver; import org.hobbit.controller.docker.ContainerTerminationCallback; +import com.spotify.docker.client.messages.ContainerStats; +import com.spotify.docker.client.messages.swarm.Service; +import com.spotify.docker.client.messages.swarm.Service.Criteria; + public class DummyContainerManager implements ContainerManager { private Semaphore benchmarkControllerTerminated; @@ -65,13 +68,13 @@ public String startContainer(String imageName, String containerType, String pare @Override public String startContainer(String imageName, String containerType, String parentId, String[] env, - String[] netAliases, String[] command, boolean pullImage) { + String[] netAliases, String[] command, boolean pullImage, Map constraints) { return containerName(imageName); } @Override public String startContainer(String imageName, String containerType, String parentId, String[] env, - String[] netAliases, String[] command, String experimentId) { + String[] netAliases, String[] command, String experimentId, Map constraints) { return containerName(imageName); } From feced4c23dc03243febb957ec9cb2384fea204c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Mon, 11 Sep 2023 19:06:54 +0200 Subject: [PATCH 03/52] Increased commons-io version. --- parent-pom/pom.xml | 2 +- .../hobbit/controller/ExperimentManager.java | 21 ++-- .../hobbit/controller/PlatformController.java | 111 ++++++++---------- .../controller/docker/MetaDataFactory.java | 17 ++- 4 files changed, 68 insertions(+), 83 deletions(-) diff --git a/parent-pom/pom.xml b/parent-pom/pom.xml index a0d73137..35902a5c 100644 --- a/parent-pom/pom.xml +++ b/parent-pom/pom.xml @@ -111,7 +111,7 @@ commons-io commons-io - 2.7 + 2.13.0 diff --git a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java index 64638c67..9a290f4f 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java @@ -20,9 +20,7 @@ import java.io.IOException; import java.io.StringWriter; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.Timer; @@ -52,7 +50,7 @@ import org.hobbit.core.data.status.ControllerStatus; import org.hobbit.core.data.status.RunningExperiment; import org.hobbit.core.rabbit.RabbitMQUtils; -import org.hobbit.utils.EnvVariables; +import org.hobbit.utils.config.HobbitConfiguration; import org.hobbit.utils.rdf.RdfHelper; import org.hobbit.vocab.HOBBIT; import org.hobbit.vocab.HobbitErrors; @@ -117,14 +115,19 @@ public class ExperimentManager implements Closeable { * Timer used to trigger the creation of the next benchmark. */ protected Timer expStartTimer; + /** + * The configuration of this platform. + */ + protected HobbitConfiguration hobbitConfig = null; - public ExperimentManager(PlatformController controller) { - this(controller, CHECK_FOR_FIRST_EXPERIMENT, CHECK_FOR_NEW_EXPERIMENT); + public ExperimentManager(PlatformController controller, HobbitConfiguration hobbitConfig) { + this(controller, hobbitConfig, CHECK_FOR_FIRST_EXPERIMENT, CHECK_FOR_NEW_EXPERIMENT); } - protected ExperimentManager(PlatformController controller, long checkForFirstExperiment, - long checkForNewExperiment) { + protected ExperimentManager(PlatformController controller, HobbitConfiguration hobbitConfig, + long checkForFirstExperiment, long checkForNewExperiment) { this.controller = controller; + this.hobbitConfig = hobbitConfig; try { // TODO environment variable should have been used there @@ -289,10 +292,10 @@ public void createNextExperiment() { } private void createRabbitMQ(ExperimentConfiguration config) throws Exception { - String rabbitMQAddress = EnvVariables.getString(RABBIT_MQ_EXPERIMENTS_HOST_NAME_KEY, (String) null); + String rabbitMQAddress = hobbitConfig.getString(RABBIT_MQ_EXPERIMENTS_HOST_NAME_KEY, (String) null); if (rabbitMQAddress == null) { LOGGER.info("Starting new RabbitMQ for the experiment..."); - rabbitMQAddress = controller.containerManager.startContainer(EnvVariables.getString(RABBIT_IMAGE_ENV_KEY), + rabbitMQAddress = controller.containerManager.startContainer(hobbitConfig.getString(RABBIT_IMAGE_ENV_KEY), Constants.CONTAINER_TYPE_BENCHMARK, null, new String[] {}, null, null, config.id, Collections.emptyMap()); if (rabbitMQAddress == null) { diff --git a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java index 21f64def..c137945e 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java +++ b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java @@ -32,6 +32,7 @@ import java.util.TimerTask; import java.util.concurrent.Semaphore; +import org.apache.commons.configuration2.EnvironmentConfiguration; import org.apache.commons.io.Charsets; import org.apache.commons.io.IOUtils; import org.apache.jena.query.Dataset; @@ -80,7 +81,7 @@ import org.hobbit.core.rabbit.RabbitMQUtils; import org.hobbit.storage.client.StorageServiceClient; import org.hobbit.storage.queries.SparqlQueries; -import org.hobbit.utils.EnvVariables; +import org.hobbit.utils.config.HobbitConfiguration; import org.hobbit.utils.rdf.RdfHelper; import org.hobbit.vocab.HOBBIT; import org.hobbit.vocab.HobbitExperiments; @@ -100,8 +101,7 @@ * @author Michael Röder (roeder@informatik.uni-leipzig.de) * */ -public class PlatformController extends AbstractComponent - implements ContainerTerminationCallback, ExperimentAnalyzer { +public class PlatformController extends AbstractComponent implements ContainerTerminationCallback, ExperimentAnalyzer { private static final Logger LOGGER = LoggerFactory.getLogger(PlatformController.class); @@ -127,8 +127,8 @@ public class PlatformController extends AbstractComponent */ private static final String CONTAINER_PARENT_CHECK_ENV_KEY = "CONTAINER_PARENT_CHECK"; /** - * Flag indicating whether a parent check for - * newly created containers is necessary or not. + * Flag indicating whether a parent check for newly created containers is + * necessary or not. */ private static final boolean CONTAINER_PARENT_CHECK = System.getenv().containsKey(CONTAINER_PARENT_CHECK_ENV_KEY) ? System.getenv().get(CONTAINER_PARENT_CHECK_ENV_KEY) == "1" @@ -206,9 +206,11 @@ public class PlatformController extends AbstractComponent protected ClusterManager clusterManager; /** - * Timer used to trigger publishing of challenges and checking for repeatable challenges. + * Timer used to trigger publishing of challenges and checking for repeatable + * challenges. */ protected Timer challengeCheckTimer; + protected HobbitConfiguration hobbitConfig; /** * Default constructor. @@ -232,6 +234,9 @@ public void init() throws Exception { super.init(); LOGGER.debug("Platform controller initialization started."); + hobbitConfig = new HobbitConfiguration(); + hobbitConfig.addConfiguration(new EnvironmentConfiguration()); + // Set task history limit for swarm cluster to 0 (will remove all terminated // containers) // Only for prod mode @@ -258,7 +263,7 @@ public void init() throws Exception { LOGGER.debug("Container observer initialized."); List managers = new ArrayList(); - if(System.getenv().containsKey(LOCAL_METADATA_DIR_KEY)) { + if (System.getenv().containsKey(LOCAL_METADATA_DIR_KEY)) { String metadataDirectory = System.getenv().get(LOCAL_METADATA_DIR_KEY); LOGGER.info("Local metadata directory: {}", metadataDirectory); managers.add(new FileBasedImageManager(metadataDirectory)); @@ -284,7 +289,7 @@ public void init() throws Exception { // the experiment manager should be the last module to create since it // directly starts to use the other modules if (expManager == null) { - expManager = new ExperimentManager(this); + expManager = new ExperimentManager(this, hobbitConfig); } // schedule challenges re-publishing @@ -332,13 +337,11 @@ public void closeExpRabbitMQConnector() { *
  • {@link Commands#DOCKER_CONTAINER_STOP}
    • * * - * @param command - * command to be executed - * @param data - * byte-encoded supplementary json for the command + * @param command command to be executed + * @param data byte-encoded supplementary json for the command * - * 0 - start container 1 - stop container Data format for each - * command: Start container: + * 0 - start container 1 - stop container Data format for each + * command: Start container: */ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.BasicProperties props) { String replyTo = null; @@ -365,7 +368,8 @@ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.Bas containerName = createContainer(startParams); } else { LOGGER.error( - "Got a request to start a container for experiment \"{}\" which is either not running or was already stopped. Returning null.", sessionId); + "Got a request to start a container for experiment \"{}\" which is either not running or was already stopped. Returning null.", + sessionId); } if (replyTo != null) { @@ -374,8 +378,7 @@ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.Bas propsBuilder.deliveryMode(2); propsBuilder.correlationId(props.getCorrelationId()); AMQP.BasicProperties replyProps = propsBuilder.build(); - publishToCmdChannel("", replyTo, replyProps, - RabbitMQUtils.writeString(containerName)); + publishToCmdChannel("", replyTo, replyProps, RabbitMQUtils.writeString(containerName)); } catch (IOException e) { StringBuilder errMsgBuilder = new StringBuilder(); errMsgBuilder.append("Error, couldn't sent response after creation of container ("); @@ -463,8 +466,7 @@ private StartCommandData deserializeStartCommandData(byte[] data) { * Creates and starts a container based on the given {@link StartCommandData} * instance. * - * @param data - * the data needed to start the container + * @param data the data needed to start the container * @return the name of the created container */ private String createContainer(StartCommandData data) { @@ -492,8 +494,7 @@ private String createContainer(StartCommandData data) { /** * Stops the container with the given container name. * - * @param containerName - * name of the container that should be stopped + * @param containerName name of the container that should be stopped */ public void stopContainer(String containerName) { String containerId = containerManager.getContainerId(containerName); @@ -587,14 +588,10 @@ public void analyzeExperiment(String uri) throws IOException { * Sends the given command to the command queue with the given data appended and * using the given properties. * - * @param address - * address for the message - * @param command - * the command that should be sent - * @param data - * data that should be appended to the command - * @param props - * properties that should be used for the message + * @param address address for the message + * @param command the command that should be sent + * @param data data that should be appended to the command + * @param props properties that should be used for the message * @throws IOException */ protected void sendToCmdQueue(String address, byte command, byte data[], BasicProperties props) throws IOException { @@ -619,7 +616,8 @@ protected void sendToCmdQueue(String address, byte command, byte data[], BasicPr /** * A wrapper around basicPublish. */ - private void publishToCmdChannel(String exchange, String routingKey, BasicProperties props, byte[] body) throws IOException { + private void publishToCmdChannel(String exchange, String routingKey, BasicProperties props, byte[] body) + throws IOException { if (rabbitMQConnector != null) { rabbitMQConnector.basicPublish(exchange, routingKey, props, body); } else { @@ -774,11 +772,9 @@ public void handleFrontEndCmd(byte bytes[], String replyTo, BasicProperties repl * Retrieves model for the given challenge from the given graph (or without * selecting a certain graph if the graphUri is {@code null}). * - * @param challengeUri - * the URI for which the model should be retrieved - * @param graphUri - * the URI from which the data should be retrieved or {@code null} if - * all graphs should be taken into account. + * @param challengeUri the URI for which the model should be retrieved + * @param graphUri the URI from which the data should be retrieved or + * {@code null} if all graphs should be taken into account. * @return the RDF model of the challenge */ protected Model getChallengeFromUri(String challengeUri, String graphUri) { @@ -833,8 +829,7 @@ private List getChallengeTasksFromUri(String challengeU /** * Inserts the configured experiments of a challenge into the queue. * - * @param challengeUri - * the URI of the challenge + * @param challengeUri the URI of the challenge */ private void executeChallengeExperiments(String challengeUri) { // get experiments from the challenge @@ -855,12 +850,9 @@ private void executeChallengeExperiments(String challengeUri) { * Schedules the date of next execution for a repeatable challenge, or closes * it. * - * @param storage - * storage - * @param challengeUri - * challenge URI - * @param now - * time to use as current when scheduling + * @param storage storage + * @param challengeUri challenge URI + * @param now time to use as current when scheduling */ protected static synchronized void scheduleDateOfNextExecution(StorageServiceClient storage, String challengeUri, Calendar now) { @@ -933,10 +925,8 @@ protected static synchronized void scheduleDateOfNextExecution(StorageServiceCli /** * Copies the challenge from challenge definition graph to public graph. * - * @param storage - * storage - * @param challengeUri - * challenge URI + * @param storage storage + * @param challengeUri challenge URI */ protected static synchronized boolean copyChallengeToPublicResultGraph(StorageServiceClient storage, String challengeUri) { @@ -951,8 +941,7 @@ protected static synchronized boolean copyChallengeToPublicResultGraph(StorageSe * Closes the challenge with the given URI by adding the "closed" triple to its * graph and inserting the configured experiments into the queue. * - * @param challengeUri - * the URI of the challenge that should be closed + * @param challengeUri the URI of the challenge that should be closed */ private void closeChallenge(String challengeUri) { LOGGER.info("Closing challenge {}...", challengeUri); @@ -1153,18 +1142,15 @@ private Model createExpModelForChallengeTask(Model model, String challengeTaskUr * Adds a new experiment with the given benchmark, system and benchmark * parameter to the queue. * - * @param benchmarkUri - * the URI of the benchmark - * @param systemUri - * the URI of the system - * @param userName - * the name of the user who requested the creation of the experiment - * @param serializedBenchParams - * the serialized benchmark parameters - * @param executionDate - * the date at which this experiment should be executed as part of a - * challenge. Should be set to null if it is not part of - * a challenge. + * @param benchmarkUri the URI of the benchmark + * @param systemUri the URI of the system + * @param userName the name of the user who requested the creation + * of the experiment + * @param serializedBenchParams the serialized benchmark parameters + * @param executionDate the date at which this experiment should be + * executed as part of a challenge. Should be set + * to null if it is not part of a + * challenge. * @return the Id of the created experiment */ protected String addExperimentToQueue(String benchmarkUri, String systemUri, String userName, @@ -1245,8 +1231,7 @@ private synchronized String generateExperimentId() { * Generates an experiment URI using the given id and the experiment URI * namespace defined by {@link Constants#EXPERIMENT_URI_NS}. * - * @param id - * the id of the experiment + * @param id the id of the experiment * @return the experiment URI */ @Deprecated diff --git a/platform-controller/src/main/java/org/hobbit/controller/docker/MetaDataFactory.java b/platform-controller/src/main/java/org/hobbit/controller/docker/MetaDataFactory.java index 84625b0e..5731c4b1 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/docker/MetaDataFactory.java +++ b/platform-controller/src/main/java/org/hobbit/controller/docker/MetaDataFactory.java @@ -73,6 +73,7 @@ protected static void buildMetaData(Resource imageResource, ImageMetaData metada metadata.date = date; metadata.source = source; metadata.rdfModel = model; + // TODO Add system constraints if they have been defined! } public static Model ttlStringToModel(String modelString, String lang) { @@ -93,11 +94,9 @@ public static Model byteArrayToModel(byte data[], String lang) { * definitions of other hobbit:Benchmark elements than the benchmark with the * given URI. * - * @param model - * the model from which all triples will be copied - * @param benchmarkUri - * the URI of the only benchmark which is not removed from the copied - * model + * @param model the model from which all triples will be copied + * @param benchmarkUri the URI of the only benchmark which is not removed from + * the copied model * @return the copied model */ public static Model getModelWithUniqueBenchmark(Model model, String benchmarkUri) { @@ -112,11 +111,9 @@ public static Model getModelWithUniqueBenchmark(Model model, String benchmarkUri * for which a triple {@code s rdf:type hobbit:SystemInstance} can be found in * the given model. * - * @param model - * the model from which all triples will be copied - * @param systemUri - * the URI of the only system which is not removed from the copied - * model + * @param model the model from which all triples will be copied + * @param systemUri the URI of the only system which is not removed from the + * copied model * @return the copied model */ public static Model getModelWithUniqueSystem(Model model, String systemUri) { From 74ff58bb01ab4fd2bf89d19cf92931db39291947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Thu, 28 Sep 2023 15:22:53 +0200 Subject: [PATCH 04/52] Added a flag to disable the gitlab-based image manager. --- .../hobbit/controller/PlatformController.java | 119 +++++++++--------- 1 file changed, 56 insertions(+), 63 deletions(-) diff --git a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java index f31a7883..e1997278 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java +++ b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java @@ -80,7 +80,6 @@ import org.hobbit.core.rabbit.RabbitMQUtils; import org.hobbit.storage.client.StorageServiceClient; import org.hobbit.storage.queries.SparqlQueries; -import org.hobbit.utils.EnvVariables; import org.hobbit.utils.rdf.RdfHelper; import org.hobbit.vocab.HOBBIT; import org.hobbit.vocab.HobbitExperiments; @@ -100,8 +99,7 @@ * @author Michael Röder (roeder@informatik.uni-leipzig.de) * */ -public class PlatformController extends AbstractComponent - implements ContainerTerminationCallback, ExperimentAnalyzer { +public class PlatformController extends AbstractComponent implements ContainerTerminationCallback, ExperimentAnalyzer { private static final Logger LOGGER = LoggerFactory.getLogger(PlatformController.class); @@ -127,8 +125,8 @@ public class PlatformController extends AbstractComponent */ private static final String CONTAINER_PARENT_CHECK_ENV_KEY = "CONTAINER_PARENT_CHECK"; /** - * Flag indicating whether a parent check for - * newly created containers is necessary or not. + * Flag indicating whether a parent check for newly created containers is + * necessary or not. */ private static final boolean CONTAINER_PARENT_CHECK = System.getenv().containsKey(CONTAINER_PARENT_CHECK_ENV_KEY) ? System.getenv().get(CONTAINER_PARENT_CHECK_ENV_KEY) == "1" @@ -137,6 +135,10 @@ public class PlatformController extends AbstractComponent * Environmental variable key for the local metadata directory. */ private static final String LOCAL_METADATA_DIR_KEY = "LOCAL_METADATA_DIRECTORY"; + /** + * Environmental variable key for the Gitlab usage flag. + */ + private static final String USE_GITLAB_KEY = "USE_GITLAB"; /** * Time interval after which challenges are checked for being published. @@ -206,7 +208,8 @@ public class PlatformController extends AbstractComponent protected ClusterManager clusterManager; /** - * Timer used to trigger publishing of challenges and checking for repeatable challenges. + * Timer used to trigger publishing of challenges and checking for repeatable + * challenges. */ protected Timer challengeCheckTimer; @@ -258,7 +261,7 @@ public void init() throws Exception { LOGGER.debug("Container observer initialized."); List managers = new ArrayList(); - if(System.getenv().containsKey(LOCAL_METADATA_DIR_KEY)) { + if (System.getenv().containsKey(LOCAL_METADATA_DIR_KEY)) { String metadataDirectory = System.getenv().get(LOCAL_METADATA_DIR_KEY); LOGGER.info("Local metadata directory: {}", metadataDirectory); managers.add(new FileBasedImageManager(metadataDirectory)); @@ -266,7 +269,17 @@ public void init() throws Exception { LOGGER.info("Using default directory for local metadata."); managers.add(new FileBasedImageManager()); } - managers.add(new GitlabBasedImageManager()); + boolean useGitlab = true; + if (System.getenv().containsKey(USE_GITLAB_KEY)) { + try { + useGitlab = Boolean.parseBoolean(System.getenv().get(USE_GITLAB_KEY)); + } catch (Exception e) { + LOGGER.error("Couldn't parse value of " + USE_GITLAB_KEY + ". It will be ignored."); + } + } + if (useGitlab) { + managers.add(new GitlabBasedImageManager()); + } imageManager = new ImageManagerFacade(managers); LOGGER.debug("Image manager initialized."); @@ -332,13 +345,11 @@ public void closeExpRabbitMQConnector() { *
  • {@link Commands#DOCKER_CONTAINER_STOP}
    • * * - * @param command - * command to be executed - * @param data - * byte-encoded supplementary json for the command + * @param command command to be executed + * @param data byte-encoded supplementary json for the command * - * 0 - start container 1 - stop container Data format for each - * command: Start container: + * 0 - start container 1 - stop container Data format for each + * command: Start container: */ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.BasicProperties props) { String replyTo = null; @@ -365,7 +376,8 @@ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.Bas containerName = createContainer(startParams); } else { LOGGER.error( - "Got a request to start a container for experiment \"{}\" which is either not running or was already stopped. Returning null.", sessionId); + "Got a request to start a container for experiment \"{}\" which is either not running or was already stopped. Returning null.", + sessionId); } if (replyTo != null) { @@ -374,8 +386,7 @@ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.Bas propsBuilder.deliveryMode(2); propsBuilder.correlationId(props.getCorrelationId()); AMQP.BasicProperties replyProps = propsBuilder.build(); - publishToCmdChannel("", replyTo, replyProps, - RabbitMQUtils.writeString(containerName)); + publishToCmdChannel("", replyTo, replyProps, RabbitMQUtils.writeString(containerName)); } catch (IOException e) { StringBuilder errMsgBuilder = new StringBuilder(); errMsgBuilder.append("Error, couldn't sent response after creation of container ("); @@ -463,8 +474,7 @@ private StartCommandData deserializeStartCommandData(byte[] data) { * Creates and starts a container based on the given {@link StartCommandData} * instance. * - * @param data - * the data needed to start the container + * @param data the data needed to start the container * @return the name of the created container */ private String createContainer(StartCommandData data) { @@ -492,8 +502,7 @@ private String createContainer(StartCommandData data) { /** * Stops the container with the given container name. * - * @param containerName - * name of the container that should be stopped + * @param containerName name of the container that should be stopped */ public void stopContainer(String containerName) { String containerId = containerManager.getContainerId(containerName); @@ -587,14 +596,10 @@ public void analyzeExperiment(String uri) throws IOException { * Sends the given command to the command queue with the given data appended and * using the given properties. * - * @param address - * address for the message - * @param command - * the command that should be sent - * @param data - * data that should be appended to the command - * @param props - * properties that should be used for the message + * @param address address for the message + * @param command the command that should be sent + * @param data data that should be appended to the command + * @param props properties that should be used for the message * @throws IOException */ protected void sendToCmdQueue(String address, byte command, byte data[], BasicProperties props) throws IOException { @@ -619,7 +624,8 @@ protected void sendToCmdQueue(String address, byte command, byte data[], BasicPr /** * A wrapper around basicPublish. */ - private void publishToCmdChannel(String exchange, String routingKey, BasicProperties props, byte[] body) throws IOException { + private void publishToCmdChannel(String exchange, String routingKey, BasicProperties props, byte[] body) + throws IOException { if (rabbitMQConnector != null) { rabbitMQConnector.basicPublish(exchange, routingKey, props, body); } else { @@ -774,11 +780,9 @@ public void handleFrontEndCmd(byte bytes[], String replyTo, BasicProperties repl * Retrieves model for the given challenge from the given graph (or without * selecting a certain graph if the graphUri is {@code null}). * - * @param challengeUri - * the URI for which the model should be retrieved - * @param graphUri - * the URI from which the data should be retrieved or {@code null} if - * all graphs should be taken into account. + * @param challengeUri the URI for which the model should be retrieved + * @param graphUri the URI from which the data should be retrieved or + * {@code null} if all graphs should be taken into account. * @return the RDF model of the challenge */ protected Model getChallengeFromUri(String challengeUri, String graphUri) { @@ -833,8 +837,7 @@ private List getChallengeTasksFromUri(String challengeU /** * Inserts the configured experiments of a challenge into the queue. * - * @param challengeUri - * the URI of the challenge + * @param challengeUri the URI of the challenge */ private void executeChallengeExperiments(String challengeUri) { // get experiments from the challenge @@ -855,12 +858,9 @@ private void executeChallengeExperiments(String challengeUri) { * Schedules the date of next execution for a repeatable challenge, or closes * it. * - * @param storage - * storage - * @param challengeUri - * challenge URI - * @param now - * time to use as current when scheduling + * @param storage storage + * @param challengeUri challenge URI + * @param now time to use as current when scheduling */ protected static synchronized void scheduleDateOfNextExecution(StorageServiceClient storage, String challengeUri, Calendar now) { @@ -933,10 +933,8 @@ protected static synchronized void scheduleDateOfNextExecution(StorageServiceCli /** * Copies the challenge from challenge definition graph to public graph. * - * @param storage - * storage - * @param challengeUri - * challenge URI + * @param storage storage + * @param challengeUri challenge URI */ protected static synchronized boolean copyChallengeToPublicResultGraph(StorageServiceClient storage, String challengeUri) { @@ -951,8 +949,7 @@ protected static synchronized boolean copyChallengeToPublicResultGraph(StorageSe * Closes the challenge with the given URI by adding the "closed" triple to its * graph and inserting the configured experiments into the queue. * - * @param challengeUri - * the URI of the challenge that should be closed + * @param challengeUri the URI of the challenge that should be closed */ private void closeChallenge(String challengeUri) { LOGGER.info("Closing challenge {}...", challengeUri); @@ -1153,18 +1150,15 @@ private Model createExpModelForChallengeTask(Model model, String challengeTaskUr * Adds a new experiment with the given benchmark, system and benchmark * parameter to the queue. * - * @param benchmarkUri - * the URI of the benchmark - * @param systemUri - * the URI of the system - * @param userName - * the name of the user who requested the creation of the experiment - * @param serializedBenchParams - * the serialized benchmark parameters - * @param executionDate - * the date at which this experiment should be executed as part of a - * challenge. Should be set to null if it is not part of - * a challenge. + * @param benchmarkUri the URI of the benchmark + * @param systemUri the URI of the system + * @param userName the name of the user who requested the creation + * of the experiment + * @param serializedBenchParams the serialized benchmark parameters + * @param executionDate the date at which this experiment should be + * executed as part of a challenge. Should be set + * to null if it is not part of a + * challenge. * @return the Id of the created experiment */ protected String addExperimentToQueue(String benchmarkUri, String systemUri, String userName, @@ -1245,8 +1239,7 @@ private synchronized String generateExperimentId() { * Generates an experiment URI using the given id and the experiment URI * namespace defined by {@link Constants#EXPERIMENT_URI_NS}. * - * @param id - * the id of the experiment + * @param id the id of the experiment * @return the experiment URI */ @Deprecated From 536e7b42e8dc916c74cd14f2444b1451a12cfde4 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Tue, 10 Oct 2023 16:27:35 +0200 Subject: [PATCH 05/52] Build gui-serverbackend in Docker See #541, #556. --- Makefile | 4 ++-- hobbit-gui/gui-serverbackend/Dockerfile | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index a4844b79..6f50b6e8 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ build-java: install-parent-pom build-controller build-storage build-analysis bui build-gui: cd hobbit-gui/gui-client && sh -c 'test "$$TRAVIS" = "true" && npm ci; true' && sh -c 'test "$$TRAVIS" = "true" || npm install; true' && npm run build-prod - cd hobbit-gui/gui-serverbackend && mvn clean package + # see hobbit-gui/gui-serverbackend/Dockerfile build-controller: cd platform-controller && make build @@ -49,7 +49,7 @@ build-dev-platform-controller-image: docker build -t hobbitproject/hobbit-platform-controller:dev ./platform-controller build-dev-gui-image: - docker build -t hobbitproject/hobbit-gui:dev ./hobbit-gui/gui-serverbackend + docker build -t hobbitproject/hobbit-gui:dev --file hobbit-gui/gui-serverbackend/Dockerfile . build-dev-analysis-image: docker build -t hobbitproject/hobbit-analysis-component:dev ./analysis-component diff --git a/hobbit-gui/gui-serverbackend/Dockerfile b/hobbit-gui/gui-serverbackend/Dockerfile index 08671041..4c887f82 100644 --- a/hobbit-gui/gui-serverbackend/Dockerfile +++ b/hobbit-gui/gui-serverbackend/Dockerfile @@ -1,3 +1,13 @@ +FROM maven:3-eclipse-temurin-11 AS build +WORKDIR /usr/src/hobbit-platform +COPY parent-pom/pom.xml ./parent-pom/ +COPY hobbit-gui/gui-serverbackend/pom.xml ./hobbit-gui/gui-serverbackend/ +RUN mvn --file ./hobbit-gui/gui-serverbackend/ dependency:go-offline +COPY hobbit-gui/gui-serverbackend/src ./hobbit-gui/gui-serverbackend/src +RUN mvn --file ./hobbit-gui/gui-serverbackend/ test +COPY hobbit-gui/gui-client/dist ./hobbit-gui/gui-client/dist +RUN mvn --file ./hobbit-gui/gui-serverbackend/ package + FROM jetty:9.3-jre8 RUN cd $JETTY_BASE && \ @@ -6,6 +16,6 @@ RUN cd $JETTY_BASE && \ rm -f keycloak-jetty93-adapter-for-hobbit-dist-2.4.0.Final.zip && \ java -jar $JETTY_HOME/start.jar --add-to-startd=keycloak -ADD ./messages /var/lib/jetty/webapps/messages +COPY hobbit-gui/gui-serverbackend/messages /var/lib/jetty/webapps/messages -ADD ./target/gui-serverbackend.war $JETTY_BASE/webapps/ROOT.war +COPY --from=build /usr/src/hobbit-platform/hobbit-gui/gui-serverbackend/target/gui-serverbackend.war $JETTY_BASE/webapps/ROOT.war From 2726095ba7b90238a5b7911619ca6a3cbaec8cc4 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Wed, 11 Oct 2023 12:17:34 +0200 Subject: [PATCH 06/52] Build gui-client in Docker See #425, #556. --- Makefile | 1 - hobbit-gui/gui-serverbackend/Dockerfile | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6f50b6e8..f1c04a12 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,6 @@ build: build-java build-dev-images build-java: install-parent-pom build-controller build-storage build-analysis build-gui build-gui: - cd hobbit-gui/gui-client && sh -c 'test "$$TRAVIS" = "true" && npm ci; true' && sh -c 'test "$$TRAVIS" = "true" || npm install; true' && npm run build-prod # see hobbit-gui/gui-serverbackend/Dockerfile build-controller: diff --git a/hobbit-gui/gui-serverbackend/Dockerfile b/hobbit-gui/gui-serverbackend/Dockerfile index 4c887f82..7e6db235 100644 --- a/hobbit-gui/gui-serverbackend/Dockerfile +++ b/hobbit-gui/gui-serverbackend/Dockerfile @@ -1,3 +1,11 @@ +# https://nodejs.org/en/docs/guides/nodejs-docker-webapp +FROM node:16 AS build-client +WORKDIR /usr/src/hobbit-platform +COPY hobbit-gui/gui-client/package*.json ./hobbit-gui/gui-client/ +RUN npm --prefix hobbit-gui/gui-client ci --omit=dev +COPY hobbit-gui/gui-client ./hobbit-gui/gui-client +RUN npm --prefix hobbit-gui/gui-client run build-prod + FROM maven:3-eclipse-temurin-11 AS build WORKDIR /usr/src/hobbit-platform COPY parent-pom/pom.xml ./parent-pom/ @@ -5,7 +13,7 @@ COPY hobbit-gui/gui-serverbackend/pom.xml ./hobbit-gui/gui-serverbackend/ RUN mvn --file ./hobbit-gui/gui-serverbackend/ dependency:go-offline COPY hobbit-gui/gui-serverbackend/src ./hobbit-gui/gui-serverbackend/src RUN mvn --file ./hobbit-gui/gui-serverbackend/ test -COPY hobbit-gui/gui-client/dist ./hobbit-gui/gui-client/dist +COPY --from=build-client /usr/src/hobbit-platform/hobbit-gui/gui-client/dist ./hobbit-gui/gui-client/dist RUN mvn --file ./hobbit-gui/gui-serverbackend/ package FROM jetty:9.3-jre8 From 149088e162a1477d8e339f1658c09d17b7b2a6ad Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Tue, 10 Oct 2023 18:38:50 +0200 Subject: [PATCH 07/52] Use a directory instead of a war file --- hobbit-gui/gui-serverbackend/Dockerfile | 2 +- hobbit-gui/gui-serverbackend/pom.xml | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/hobbit-gui/gui-serverbackend/Dockerfile b/hobbit-gui/gui-serverbackend/Dockerfile index 7e6db235..667eecc4 100644 --- a/hobbit-gui/gui-serverbackend/Dockerfile +++ b/hobbit-gui/gui-serverbackend/Dockerfile @@ -26,4 +26,4 @@ RUN cd $JETTY_BASE && \ COPY hobbit-gui/gui-serverbackend/messages /var/lib/jetty/webapps/messages -COPY --from=build /usr/src/hobbit-platform/hobbit-gui/gui-serverbackend/target/gui-serverbackend.war $JETTY_BASE/webapps/ROOT.war +COPY --from=build /usr/src/hobbit-platform/hobbit-gui/gui-serverbackend/target/gui-serverbackend $JETTY_BASE/webapps/ROOT diff --git a/hobbit-gui/gui-serverbackend/pom.xml b/hobbit-gui/gui-serverbackend/pom.xml index 97b437a0..a6bc95a3 100644 --- a/hobbit-gui/gui-serverbackend/pom.xml +++ b/hobbit-gui/gui-serverbackend/pom.xml @@ -148,6 +148,19 @@ org.apache.maven.plugins maven-war-plugin 3.0.0 + + + default-war + none + + + war-exploded + package + + exploded + + + From 972385c1ff458e22568192509e6545d9b64010a7 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Tue, 10 Oct 2023 16:41:25 +0200 Subject: [PATCH 08/52] Always resolve or reject the Promise in getToken --- hobbit-gui/gui-client/src/app/auth/keycloak.service.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hobbit-gui/gui-client/src/app/auth/keycloak.service.ts b/hobbit-gui/gui-client/src/app/auth/keycloak.service.ts index ef1d58b9..313c603c 100644 --- a/hobbit-gui/gui-client/src/app/auth/keycloak.service.ts +++ b/hobbit-gui/gui-client/src/app/auth/keycloak.service.ts @@ -66,6 +66,8 @@ export class KeycloakService { .error(() => { reject('Failed to refresh token'); }); + } else { + reject('No token'); } }); } From dc82184b27c60ae96d7605f2756dd125d48e8bc6 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Wed, 11 Oct 2023 11:40:48 +0200 Subject: [PATCH 09/52] Add `USE_UI_AUTH` option to disable access control and Keycloak requirement Resolves #555. --- config/jetty/web-without-ui-auth.xml | 103 ++++++++++++++++++ docker-compose-dev.yml | 20 ++-- .../src/app/auth/keycloak.service.ts | 12 ++ .../gui-client/src/app/custom-http.service.ts | 7 +- .../hobbit/gui/rest/InternalResources.java | 38 +++++++ 5 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 config/jetty/web-without-ui-auth.xml diff --git a/config/jetty/web-without-ui-auth.xml b/config/jetty/web-without-ui-auth.xml new file mode 100644 index 00000000..6d5139ad --- /dev/null +++ b/config/jetty/web-without-ui-auth.xml @@ -0,0 +1,103 @@ + + + + + de.usu.research.hobbit.gui.rest.Application + + + + IndexServer + de.usu.research.hobbit.gui.util.IndexServer + + + + + de.usu.research.hobbit.gui.rest.Application + /rest/* + + + + IndexServer + /benchmarks + /benchmarks/* + /challenges + /challenges/* + /experiments + /experiments/* + /home + /home/* + /reports + /reports/* + /upload + /upload/* + + + + + webapi + /rest/internal/keycloak-config + + + + + + + webapi + /rest/* + GET + POST + PUT + DELETE + + + + + + BASIC + this is ignored currently + + + + system-provider + + + guest + + + challenge-organiser + + + + + cors + de.usu.research.hobbit.gui.util.CorsFilter + + + cache + de.usu.research.hobbit.gui.util.CacheFilter + + + mqconn + de.usu.research.hobbit.gui.util.ConnectionShutdownFilter + + + + cors + /rest/* + + + + cache + /rest/* + + + + mqconn + /rest/* + + diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index ce25d944..694e359c 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -36,20 +36,22 @@ services: - KEYCLOAK_DIRECT_URL=http://keycloak:8080/auth - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_HTTP_PORT=9200 - #volumes: + - USE_UI_AUTH=false + volumes: + - ./config/jetty/web-without-ui-auth.xml:/var/lib/jetty/webapps/ROOT/WEB-INF/web.xml #- /data/docker/messages/global.html:/var/lib/jetty/webapps/messages/global.html #- /data/docker/messages/benchmark.html:/var/lib/jetty/webapps/messages/benchmark.html #- /data/docker/messages/status.html:/var/lib/jetty/webapps/messages/status.html # Keycloak user management (used by the GUI) - keycloak: - image: hobbitproject/hobbit-keycloak:latest - ports: - - "8181:8080" - networks: - - hobbit - volumes: - - ./config/keycloak:/opt/jboss/keycloak/standalone/data/db + # keycloak: + # image: hobbitproject/hobbit-keycloak:latest + # ports: + # - "8181:8080" + # networks: + # - hobbit + # volumes: + # - ./config/keycloak:/opt/jboss/keycloak/standalone/data/db # HOBBIT Analysis component analysis: diff --git a/hobbit-gui/gui-client/src/app/auth/keycloak.service.ts b/hobbit-gui/gui-client/src/app/auth/keycloak.service.ts index 313c603c..2a1e2199 100644 --- a/hobbit-gui/gui-client/src/app/auth/keycloak.service.ts +++ b/hobbit-gui/gui-client/src/app/auth/keycloak.service.ts @@ -18,6 +18,14 @@ export class KeycloakService { req.onload = (e) => { const keycloakConfig = JSON.parse(req.responseText); + if (keycloakConfig.clientId === 'DISABLED') { + console.log('UI auth is disabled'); + KeycloakService.auth.loggedIn = true; + KeycloakService.auth.authz = true; + resolve(null); + return; + } + const keycloakAuth: any = Keycloak(keycloakConfig); keycloakAuth.init({ onLoad: 'login-required' }) .success(() => { @@ -67,6 +75,10 @@ export class KeycloakService { reject('Failed to refresh token'); }); } else { + if (KeycloakService.auth.authz === true) { + resolve(null); + return; + } reject('No token'); } }); diff --git a/hobbit-gui/gui-client/src/app/custom-http.service.ts b/hobbit-gui/gui-client/src/app/custom-http.service.ts index 2e28d76e..1bead0dd 100644 --- a/hobbit-gui/gui-client/src/app/custom-http.service.ts +++ b/hobbit-gui/gui-client/src/app/custom-http.service.ts @@ -93,14 +93,17 @@ export class CustomHttp { const obs = Observable.fromPromise(this.keycloakService.getToken()).map(token => { const requestUrl = environment.backendUrl + url.substr(environment.backendPrefix.length); - const headers = { 'Authorization': 'bearer ' + token }; + const headers = {}; + if (token !== null) { + Object.assign(headers, {Authorization: 'bearer ' + token}); + } let requestOptions = options; if (!options) requestOptions = { 'headers': new HttpHeaders(headers) }; else if (!options.headers) requestOptions.headers = new HttpHeaders(headers); else - requestOptions.headers = requestOptions.headers.set('Authorization', headers['Authorization']); + Object.assign(requestOptions.headers, headers); return { url: requestUrl, options: requestOptions }; }); return obs; diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/InternalResources.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/InternalResources.java index 20533d53..944def76 100644 --- a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/InternalResources.java +++ b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/InternalResources.java @@ -64,6 +64,11 @@ public class InternalResources { private static final Logger LOGGER = LoggerFactory.getLogger(InternalResources.class); + /** + * Environmental variable key for the UI authorization (keycloak) usage flag. + */ + private static final String USE_UI_AUTH_KEY = "USE_UI_AUTH"; + private static volatile KeycloakConfigBean cachedBean; private static Cache userInfoCache = CacheBuilder.newBuilder().maximumSize(100) .expireAfterWrite(1, TimeUnit.HOURS).build(); @@ -76,6 +81,22 @@ public Response getKeycloakConfig(@Context ServletContext servletContext) { return Response.ok(cachedBean).build(); } + boolean useAuth = true; + if (System.getenv().containsKey(USE_UI_AUTH_KEY)) { + try { + useAuth = Boolean.parseBoolean(System.getenv().get(USE_UI_AUTH_KEY)); + } catch (Exception e) { + LOGGER.error("Couldn't parse value of %s. It will be ignored.", USE_UI_AUTH_KEY); + } + } + if (!useAuth) { + KeycloakConfigBean bean = cachedBean = new KeycloakConfigBean(); + bean.setRealm(""); + bean.setUrl(""); + bean.setClientId("DISABLED"); + return Response.ok(bean).build(); + } + try (InputStream is = servletContext.getResourceAsStream("/WEB-INF/jetty-web.xml")) { KeycloakConfigBean bean = cachedBean; if (bean == null) { @@ -102,6 +123,23 @@ public Response userInfo(@Context SecurityContext sc) { } public static UserInfoBean getUserInfoBean(SecurityContext sc) { + boolean useAuth = true; + if (System.getenv().containsKey(USE_UI_AUTH_KEY)) { + try { + useAuth = Boolean.parseBoolean(System.getenv().get(USE_UI_AUTH_KEY)); + } catch (Exception e) { + LOGGER.error("Couldn't parse value of %s. It will be ignored.", USE_UI_AUTH_KEY); + } + } + if (!useAuth) { + UserInfoBean bean = new UserInfoBean(); + bean.setRoles(Arrays.asList("challenge-organiser", "guest", "system-provider")); + bean.setUserPrincipalName("guest"); + bean.setPreferredUsername("guest"); + bean.setName("Guest"); + return bean; + } + Principal userPrincipal = sc.getUserPrincipal(); String userPrincipalName = userPrincipal.getName(); From 3ad6b201730b0447475cb4798fe5bd5013dafdbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Sat, 14 Oct 2023 17:43:30 +0200 Subject: [PATCH 10/52] Implemented the hardware constraint processing. Fixed several compiler warnings. --- .../ExperimentHardwareInformationTest.java | 25 +++++++--------- .../controller/ExperimentTimeoutTest.java | 11 +++---- .../controller/PlatformControllerTest.java | 30 +++++++++++-------- .../docker/ContainerManagerImplTest.java | 16 ++++++---- .../docker/MetaDataFactoryTest.java | 4 --- .../hobbit/controller/utils/WaitingTest.java | 2 +- 6 files changed, 42 insertions(+), 46 deletions(-) diff --git a/platform-controller/src/test/java/org/hobbit/controller/ExperimentHardwareInformationTest.java b/platform-controller/src/test/java/org/hobbit/controller/ExperimentHardwareInformationTest.java index 2ee4d89c..04c69df2 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/ExperimentHardwareInformationTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/ExperimentHardwareInformationTest.java @@ -1,29 +1,26 @@ package org.hobbit.controller; -import org.apache.jena.sparql.vocabulary.DOAP; -import org.apache.jena.vocabulary.RDF; -import org.apache.jena.vocabulary.RDFS; -import org.hobbit.vocab.MEXCORE; -import org.apache.jena.rdf.model.RDFNode; import java.util.List; -import org.apache.jena.rdf.model.Resource; -import org.hobbit.controller.PlatformController; -import org.hobbit.controller.ExperimentManager; import org.apache.commons.compress.utils.IOUtils; import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.sparql.vocabulary.DOAP; +import org.apache.jena.vocabulary.RDF; +import org.apache.jena.vocabulary.RDFS; import org.hobbit.controller.data.ExperimentConfiguration; import org.hobbit.controller.docker.ResourceInformationCollectorImpl; -import org.hobbit.controller.mocks.DummyPlatformController; import org.hobbit.controller.mocks.DummyImageManager; +import org.hobbit.controller.mocks.DummyPlatformController; import org.hobbit.controller.mocks.DummyStorageServiceClient; +import org.hobbit.utils.config.HobbitConfiguration; import org.hobbit.vocab.HOBBIT; +import org.hobbit.vocab.MEXCORE; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.contrib.java.lang.system.EnvironmentVariables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,15 +41,13 @@ public class ExperimentHardwareInformationTest extends DockerBasedTest { private ExperimentManager manager; private PlatformController controller; - @Rule - public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); - @Before public void init() throws Exception { controller = new DummyPlatformController(); controller.resInfoCollector = new ResourceInformationCollectorImpl(controller.containerManager); controller.queue.add(new ExperimentConfiguration(EXPERIMENT_ID, DummyImageManager.BENCHMARK_NAME, "{}", DummyImageManager.SYSTEM_URI)); - manager = new ExperimentManager(controller, 1000, 1000); + HobbitConfiguration configuration = new HobbitConfiguration(); + manager = new ExperimentManager(controller, configuration, 1000L, 1000L); controller.expManager = manager; } diff --git a/platform-controller/src/test/java/org/hobbit/controller/ExperimentTimeoutTest.java b/platform-controller/src/test/java/org/hobbit/controller/ExperimentTimeoutTest.java index 2b461422..215ea534 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/ExperimentTimeoutTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/ExperimentTimeoutTest.java @@ -5,19 +5,18 @@ import org.apache.commons.compress.utils.IOUtils; import org.apache.jena.rdf.model.Model; import org.hobbit.controller.data.ExperimentConfiguration; -import org.hobbit.controller.mocks.DummyPlatformController; import org.hobbit.controller.mocks.DummyImageManager; +import org.hobbit.controller.mocks.DummyPlatformController; import org.hobbit.controller.mocks.DummyStorageServiceClient; import org.hobbit.core.data.status.ControllerStatus; +import org.hobbit.utils.config.HobbitConfiguration; import org.hobbit.vocab.HOBBIT; import org.hobbit.vocab.HobbitErrors; import org.hobbit.vocab.HobbitExperiments; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.contrib.java.lang.system.EnvironmentVariables; /** * A simple test that uses a dummy {@link PlatformController} to simulate an @@ -35,16 +34,14 @@ public class ExperimentTimeoutTest { private PlatformController controller; private Semaphore benchmarkControllerTerminated = new Semaphore(0); - @Rule - public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); - @Before public void init() { // set max execution time to 1s System.setProperty("MAX_EXECUTION_TIME", "1000"); controller = new DummyPlatformController(benchmarkControllerTerminated); controller.queue.add(new ExperimentConfiguration(EXPERIMENT_ID, DummyImageManager.BENCHMARK_NAME, "{}", DummyImageManager.SYSTEM_URI)); - manager = new ExperimentManager(controller, 1000, 1000); + HobbitConfiguration configuration = new HobbitConfiguration(); + manager = new ExperimentManager(controller, configuration, 1000, 1000); controller.expManager = manager; } diff --git a/platform-controller/src/test/java/org/hobbit/controller/PlatformControllerTest.java b/platform-controller/src/test/java/org/hobbit/controller/PlatformControllerTest.java index 40479a56..5d8a287e 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/PlatformControllerTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/PlatformControllerTest.java @@ -21,10 +21,13 @@ import static org.junit.Assert.assertTrue; import java.nio.charset.StandardCharsets; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.configuration2.MapConfiguration; import org.apache.commons.io.IOUtils; import org.hobbit.controller.data.ExperimentConfiguration; import org.hobbit.controller.data.ExperimentStatus; @@ -33,12 +36,12 @@ import org.hobbit.controller.docker.ContainerManagerImpl; import org.hobbit.core.Commands; import org.hobbit.core.Constants; +import org.hobbit.utils.config.HobbitConfiguration; import org.hobbit.utils.docker.DockerHelper; +import org.hobbit.vocab.HobbitExperiments; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.contrib.java.lang.system.EnvironmentVariables; import com.spotify.docker.client.messages.swarm.Service; import com.spotify.docker.client.messages.swarm.Task; @@ -51,9 +54,6 @@ public class PlatformControllerTest extends ContainerManagerBasedTest { private static final String RABBIT_HOST_NAME = DockerHelper.getHost(); private static final String SESSION_ID = "test-session"; - @Rule - public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); - private PlatformController controller; private void assertDockerImageEquals(String message, String expected, String got) throws Exception { @@ -64,12 +64,16 @@ private void assertDockerImageEquals(String message, String expected, String got @Before public void init() throws Exception { - environmentVariables.set(Constants.RABBIT_MQ_HOST_NAME_KEY, RABBIT_HOST_NAME); - environmentVariables.set(Constants.GENERATOR_ID_KEY, "0"); - environmentVariables.set(Constants.GENERATOR_COUNT_KEY, "1"); - environmentVariables.set(Constants.HOBBIT_SESSION_ID_KEY, "0"); + Map envVariables = new HashMap<>(); + envVariables.put(Constants.RABBIT_MQ_HOST_NAME_KEY, RABBIT_HOST_NAME); + envVariables.put(Constants.GENERATOR_ID_KEY, "0"); + envVariables.put(Constants.GENERATOR_COUNT_KEY, "1"); + envVariables.put(Constants.HOBBIT_SESSION_ID_KEY, "0"); + + HobbitConfiguration configuration = new HobbitConfiguration(); + configuration.addConfiguration(new MapConfiguration(envVariables)); - controller = new PlatformController(new LocalExperimentManager(null, SESSION_ID)); + controller = new PlatformController(new LocalExperimentManager(null, configuration, SESSION_ID)); try { controller.init(); } catch (Exception e) { @@ -142,11 +146,11 @@ public void receiveCreateContainerCommand() throws Exception { protected static class LocalExperimentManager extends ExperimentManager { - public LocalExperimentManager(PlatformController controller, String session) { - super(controller); + public LocalExperimentManager(PlatformController controller, HobbitConfiguration config, String session) { + super(controller, config); experimentStatus = new ExperimentStatus( new ExperimentConfiguration(session, "TestBenchmark", "", "TestSytem"), - Constants.EXPERIMENT_URI_NS + session); + HobbitExperiments.getExperimentURI(session)); experimentStatus.setState(States.STARTED); } diff --git a/platform-controller/src/test/java/org/hobbit/controller/docker/ContainerManagerImplTest.java b/platform-controller/src/test/java/org/hobbit/controller/docker/ContainerManagerImplTest.java index 0c5f6746..2fc27180 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/docker/ContainerManagerImplTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/docker/ContainerManagerImplTest.java @@ -16,10 +16,12 @@ */ package org.hobbit.controller.docker; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import static org.junit.Assert.*; -import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.nio.file.Paths; import java.util.ArrayList; @@ -31,6 +33,8 @@ import org.junit.Assert; import org.junit.Assume; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableMap; import com.spotify.docker.client.DockerClient; @@ -38,8 +42,8 @@ import com.spotify.docker.client.exceptions.ServiceNotFoundException; import com.spotify.docker.client.messages.Container; import com.spotify.docker.client.messages.ContainerConfig; -import com.spotify.docker.client.messages.Image; import com.spotify.docker.client.messages.HostConfig; +import com.spotify.docker.client.messages.Image; import com.spotify.docker.client.messages.PortBinding; import com.spotify.docker.client.messages.swarm.Service; import com.spotify.docker.client.messages.swarm.Task; @@ -103,7 +107,7 @@ public void startContainer() throws Exception { final Service serviceInfo = dockerClient.inspectService(containerId); assertNotNull("Service inspection response from docker", serviceInfo); - assertThat("Container ID as seen by the platform (names are used in place of IDs)", containerId, not(equalTo(serviceInfo.id()))); + assertNotEquals("Container ID as seen by the platform (names are used in place of IDs)", containerId, serviceInfo.id()); assertEquals("Type label of created swarm service", serviceInfo.spec().labels().get(ContainerManagerImpl.LABEL_TYPE), Constants.CONTAINER_TYPE_SYSTEM); diff --git a/platform-controller/src/test/java/org/hobbit/controller/docker/MetaDataFactoryTest.java b/platform-controller/src/test/java/org/hobbit/controller/docker/MetaDataFactoryTest.java index 17152455..a67778cf 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/docker/MetaDataFactoryTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/docker/MetaDataFactoryTest.java @@ -4,10 +4,8 @@ import java.io.InputStream; import java.util.Arrays; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import org.apache.commons.compress.utils.IOUtils; import org.apache.log4j.lf5.util.StreamUtils; @@ -34,8 +32,6 @@ public void testModelToBenchmarkMetaData() throws IOException { expectedMetaData.usedImages.add("hobbit/gerbil_taskgen"); expectedMetaData.date = new Date(10); expectedMetaData.source = "test"; - Map constraints = new HashMap<>(); - expectedMetaData.setSystemHardwareConstraints(constraints); InputStream input = ParameterForwardingTest.class.getClassLoader() .getResourceAsStream("org/hobbit/controller/benchmark.ttl"); diff --git a/platform-controller/src/test/java/org/hobbit/controller/utils/WaitingTest.java b/platform-controller/src/test/java/org/hobbit/controller/utils/WaitingTest.java index 12a82d2a..2718f879 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/utils/WaitingTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/utils/WaitingTest.java @@ -1,8 +1,8 @@ package org.hobbit.controller.utils; import org.junit.Test; +import org.junit.Assert; -import junit.framework.Assert; public class WaitingTest { From f0828662f240bf583f53e3dfb8387517490af9c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Sat, 14 Oct 2023 17:43:58 +0200 Subject: [PATCH 11/52] Implemented the hardware constraint processing. Fixed several compiler warnings. --- .../hobbit/controller/ExperimentManager.java | 28 ++++++++++++++++++- .../hobbit/controller/PlatformController.java | 11 ++++++-- .../data/ExperimentConfiguration.java | 1 + .../controller/docker/ContainerManager.java | 3 ++ .../docker/ContainerManagerImpl.java | 2 -- 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java index 9a290f4f..7f0fc932 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java @@ -20,7 +20,9 @@ import java.io.IOException; import java.io.StringWriter; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.Timer; @@ -40,6 +42,7 @@ import org.hobbit.controller.data.ExperimentStatus.States; import org.hobbit.controller.data.SetupHardwareInformation; import org.hobbit.controller.docker.ClusterManager; +import org.hobbit.controller.docker.ContainerManager; import org.hobbit.controller.docker.MetaDataFactory; import org.hobbit.controller.execute.ExperimentAbortTimerTask; import org.hobbit.controller.utils.RabbitMQConnector; @@ -270,7 +273,7 @@ public void createNextExperiment() { Constants.RABBIT_MQ_HOST_NAME_KEY + "=" + experimentStatus.getRabbitMQContainer(), Constants.HOBBIT_SESSION_ID_KEY + "=" + config.id, Constants.SYSTEM_PARAMETERS_MODEL_KEY + "=" + serializedSystemParams }, - null, null, config.id, benchmark.getSystemHardwareConstraints()); + null, null, config.id, getHardwareConstraints(config.serializedBenchParams)); if (containerId == null) { LOGGER.error("Couldn't start the system. Trying to cancel the benchmark."); forceBenchmarkTerminate_unsecured(HobbitErrors.SystemCreationError); @@ -291,6 +294,29 @@ public void createNextExperiment() { } } + protected static Map getHardwareConstraints(String serializedBenchParams) { + // If the serialized model could contain the hobbit:maxHardware property + if (serializedBenchParams.contains("maxHardware")) { + Model model = RabbitMQUtils.readModel(serializedBenchParams); + Resource hardware = RdfHelper.getObjectResource(model, HobbitExperiments.New, HOBBIT.maxHardware); + if (hardware != null) { + Map constraints = new HashMap<>(); + Integer cpuCount = RdfHelper.getIntValue(model, hardware, HOBBIT.hasCPUTypeCount); + if (cpuCount != null) { + // We need the CPU count in nano CPU seconds (i.e., we have to multiply the + // count with 10^9) + constraints.put(ContainerManager.NANO_CPU_LIMIT_CONSTRAINT, cpuCount * 1000000000L); + } + Long memory = RdfHelper.getLongValue(model, hardware, HOBBIT.hasMemory); + if (memory != null) { + constraints.put(ContainerManager.MEMORY_LIMIT_CONSTRAINT, memory); + } + return constraints; + } + } + return Collections.emptyMap(); + } + private void createRabbitMQ(ExperimentConfiguration config) throws Exception { String rabbitMQAddress = hobbitConfig.getString(RABBIT_MQ_EXPERIMENTS_HOST_NAME_KEY, (String) null); if (rabbitMQAddress == null) { diff --git a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java index c137945e..8fda3809 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java +++ b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java @@ -20,12 +20,15 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.time.Duration; import java.util.ArrayList; import java.util.Calendar; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.Timer; @@ -33,7 +36,6 @@ import java.util.concurrent.Semaphore; import org.apache.commons.configuration2.EnvironmentConfiguration; -import org.apache.commons.io.Charsets; import org.apache.commons.io.IOUtils; import org.apache.jena.query.Dataset; import org.apache.jena.query.DatasetFactory; @@ -636,7 +638,7 @@ protected void handleCmd(byte bytes[], AMQP.BasicProperties props) { int idLength = buffer.getInt(); byte sessionIdBytes[] = new byte[idLength]; buffer.get(sessionIdBytes); - String sessionId = new String(sessionIdBytes, Charsets.UTF_8); + String sessionId = new String(sessionIdBytes, StandardCharsets.UTF_8); byte command = buffer.get(); byte remainingData[]; if (buffer.remaining() > 0) { @@ -699,6 +701,10 @@ public void handleFrontEndCmd(byte bytes[], String replyTo, BasicProperties repl String systemUri = RabbitMQUtils.readString(buffer); String serializedBenchParams = RabbitMQUtils.readString(buffer); String userName = RabbitMQUtils.readString(buffer); + + Map maxHardwareConstraints = new HashMap<>(); + // TODO get maximum hardware constraints from the UI + // TODO deserialize hardware constraint map String experimentId = addExperimentToQueue(benchmarkUri, systemUri, userName, serializedBenchParams, null, null, null); response = RabbitMQUtils.writeString(experimentId); @@ -804,6 +810,7 @@ private List getChallengeTasksFromUri(String challengeU // get benchmark information String benchmarkUri = RdfHelper.getStringValue(model, challengeTask, HOBBIT.involvesBenchmark); String experimentId, systemUri, serializedBenchParams; + // TODO Read maximum hardware constraints from the benchmarking task data // iterate participating system instances NodeIterator systemInstanceIterator = model.listObjectsOfProperty(challengeTask, HOBBIT.involvesSystemInstance); diff --git a/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentConfiguration.java b/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentConfiguration.java index f18f5363..15465f3e 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentConfiguration.java +++ b/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentConfiguration.java @@ -17,6 +17,7 @@ package org.hobbit.controller.data; import java.util.Calendar; +import java.util.Map; /** * This data structure contains the information about a planned experiment. diff --git a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java index c623e295..a67d6727 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java @@ -31,6 +31,9 @@ * */ public interface ContainerManager { + + public static final String MEMORY_LIMIT_CONSTRAINT = "memory-limit"; + public static final String NANO_CPU_LIMIT_CONSTRAINT = "nanoCPU-limit"; /** * Exit code of containers diff --git a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java index 12ee49be..e06c1262 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java +++ b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java @@ -76,8 +76,6 @@ public class ContainerManagerImpl implements ContainerManager { public static final String USER_EMAIL_KEY = "GITLAB_EMAIL"; public static final String USER_PASSWORD_KEY = GitlabControllerImpl.GITLAB_TOKEN_KEY; public static final String REGISTRY_URL_KEY = "REGISTRY_URL"; - public static final String MEMORY_LIMIT_CONSTRAINT = "memory-limit"; - public static final String NANO_CPU_LIMIT_CONSTRAINT = "nanoCPU-limit"; private static final int DOCKER_MAX_NAME_LENGTH = 63; From e3d7034643d9acc58ad695fcb8c887c97a10bd91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Sat, 14 Oct 2023 17:44:29 +0200 Subject: [PATCH 12/52] Implemented the processing of hardware constraints. Fixed some smaller issues and warnings. --- hobbit-gui/gui-serverbackend/pom.xml | 2 +- .../rabbitmq/PlatformControllerClient.java | 129 +++++++++++------- .../gui/rabbitmq/StorageServiceChannel.java | 93 ------------- .../hobbit/gui/rest/LicenseResources.java | 11 +- .../rest/beans/HardwareConstraintBean.java | 79 +++++++++++ .../hobbit/gui/rest/beans/LicenseBean.java | 2 - .../gui/rest/beans/SubmitModelBean.java | 14 +- 7 files changed, 171 insertions(+), 159 deletions(-) delete mode 100644 hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/StorageServiceChannel.java create mode 100644 hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/HardwareConstraintBean.java diff --git a/hobbit-gui/gui-serverbackend/pom.xml b/hobbit-gui/gui-serverbackend/pom.xml index 97b437a0..4f216ccd 100644 --- a/hobbit-gui/gui-serverbackend/pom.xml +++ b/hobbit-gui/gui-serverbackend/pom.xml @@ -14,7 +14,7 @@ org.hobbit parent - ${hobbitplatform.version} + 2.0.17-SNAPSHOT ../../parent-pom gui-serverbackend diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/PlatformControllerClient.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/PlatformControllerClient.java index 10059c5f..bab1dbdf 100644 --- a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/PlatformControllerClient.java +++ b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/PlatformControllerClient.java @@ -18,7 +18,6 @@ import java.io.Closeable; import java.io.IOException; -import java.io.StringWriter; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -30,6 +29,7 @@ import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.Resource; import org.apache.jena.vocabulary.RDF; +import org.apache.jena.vocabulary.RDFS; import org.apache.jena.vocabulary.XSD; import org.hobbit.core.Constants; import org.hobbit.core.FrontEndApiCommands; @@ -53,6 +53,7 @@ import de.usu.research.hobbit.gui.rest.Datatype; import de.usu.research.hobbit.gui.rest.beans.BenchmarkBean; import de.usu.research.hobbit.gui.rest.beans.ConfigurationParamValueBean; +import de.usu.research.hobbit.gui.rest.beans.HardwareConstraintBean; import de.usu.research.hobbit.gui.rest.beans.SubmitModelBean; import de.usu.research.hobbit.gui.rest.beans.SystemBean; import de.usu.research.hobbit.gui.rest.beans.UserInfoBean; @@ -114,8 +115,7 @@ public void close() throws IOException { * @throws IOException * @throws InterruptedException * @throws ConsumerCancelledException - * @throws ShutdownSignalException - * If something goes wrong with the request + * @throws ShutdownSignalException If something goes wrong with the request */ public List requestBenchmarks() throws IOException, ShutdownSignalException, ConsumerCancelledException, InterruptedException { @@ -153,11 +153,11 @@ public List requestBenchmarks() /** * Retrieves the benchmark details from the HOBBIT PlatformControler * - * @param benchmarkUri - * the URI of the benchmark for which the details should be retrieved - * @param user - * information about the requesting user which will be used to filter - * the systems that can be used with the requested benchmark. + * @param benchmarkUri the URI of the benchmark for which the details should be + * retrieved + * @param user information about the requesting user which will be used + * to filter the systems that can be used with the requested + * benchmark. * @return * @throws GUIBackendException * @throws IOException @@ -215,16 +215,14 @@ public BenchmarkBean requestBenchmarkDetails(String benchmarkUri, UserInfoBean u * experiment will be started with the chosen system and benchmark * configuration. * - * @param benchmarkConf - * the benchmark configuration with which an experiment should be - * started - * @param userName - * the name of the user who submitted the benchmark configuration + * @param benchmarkConf the benchmark configuration with which an experiment + * should be started + * @param userName the name of the user who submitted the benchmark + * configuration * @return The ID of the created experiment - * @throws GUIBackendException - * If the given benchmark configuration is not valid - * @throws IOException - * If there is a problem during the receiving of the response + * @throws GUIBackendException If the given benchmark configuration is not valid + * @throws IOException If there is a problem during the receiving of the + * response */ public String submitBenchmark(SubmitModelBean benchmarkConf, String userName) throws GUIBackendException, IOException { @@ -257,6 +255,13 @@ public String submitBenchmark(SubmitModelBean benchmarkConf, String userName) throw new GUIBackendException("Please check your parameter definitions."); } + try { + addHardwareConstraint(model, HobbitExperiments.New, benchmarkConf.getMaxHardwareConstraints()); + } catch (Exception e) { + LOGGER.error("Got an exception while processing the hardware constraints.", e); + throw new GUIBackendException("Please check your parameter definitions."); + } + byte[] data = RabbitMQUtils.writeByteArrays(new byte[] { FrontEndApiCommands.ADD_EXPERIMENT_CONFIGURATION }, new byte[][] { RabbitMQUtils.writeString(benchmarkUri), RabbitMQUtils.writeString(systemUri), RabbitMQUtils.writeModel(model), RabbitMQUtils.writeString(userName) }, @@ -265,7 +270,7 @@ public String submitBenchmark(SubmitModelBean benchmarkConf, String userName) LOGGER.info("Sending request..."); data = client.request(data); if (data == null) { - throw new IOException("Didn't got a response."); + throw new IOException("Didn't get a response."); } String id = RabbitMQUtils.readString(data); @@ -279,15 +284,14 @@ public String submitBenchmark(SubmitModelBean benchmarkConf, String userName) * Adds the given list of parameters to the given RDF model by creating triples * using the given benchmark resource. * - * @param model - * the RDF model to which the parameter should be added - * @param benchmarkInstanceResource - * the resource of the benchmark inside the given RDF model - * @param list - * the list of parameters that should be added + * @param model the RDF model to which the parameter should be + * added + * @param experimentInstance the resource of the new experiment inside the given + * RDF model + * @param list the list of parameters that should be added * @return the updated model */ - protected static Model addParameters(Model model, Resource benchmarkInstanceResource, + protected static Model addParameters(Model model, Resource experimentInstance, List list) { for (ConfigurationParamValueBean paramValue : list) { String uri = paramValue.getId(); @@ -296,21 +300,49 @@ protected static Model addParameters(Model model, Resource benchmarkInstanceReso String range = paramValue.getRange(); if (range == null) { - model.add(benchmarkInstanceResource, model.createProperty(uri), + model.add(experimentInstance, model.createProperty(uri), model.createTypedLiteral(value, expandedXsdId(datatype))); } else { if (range.startsWith(XSD.NS)) { - model.add(benchmarkInstanceResource, model.createProperty(uri), - model.createTypedLiteral(value, range)); + model.add(experimentInstance, model.createProperty(uri), model.createTypedLiteral(value, range)); } else { - model.add(benchmarkInstanceResource, model.createProperty(uri), model.createResource(value)); + model.add(experimentInstance, model.createProperty(uri), model.createResource(value)); } } } + return model; + } - StringWriter writer = new StringWriter(); - model.write(writer, "Turtle"); - + /** + * Add hardware constraints defined in the given bean to the given model. + * + * @param model the RDF model to which the constraints should be + * added + * @param experimentInstance the resource of the new experiment for which the + * constraint should hold inside the given RDF model + * @param hardwareConstraint the constraint (or {@code null} if such a + * constraint doesn't exist) + * @return the given, updated model + */ + protected static Model addHardwareConstraint(Model model, Resource experimentInstance, + HardwareConstraintBean hardwareConstraint) { + if ((hardwareConstraint != null) && (hardwareConstraint.isValidConstraint())) { + Resource constraintsResource = model.createResource(hardwareConstraint.getIri()); + model.add(constraintsResource, RDF.type, HOBBIT.Hardware); + String label = hardwareConstraint.getLabel(); + if ((label != null) && (!label.isEmpty())) { + model.addLiteral(constraintsResource, RDFS.label, model.createLiteral(label)); + } + int cpuCount = hardwareConstraint.getCpuCount(); + if (cpuCount > 0) { + model.addLiteral(constraintsResource, HOBBIT.hasCPUTypeCount, cpuCount); + } + long memory = hardwareConstraint.getMemory(); + if (memory > 0) { + model.addLiteral(constraintsResource, HOBBIT.hasMemory, memory); + } + model.add(experimentInstance, HOBBIT.maxHardware, constraintsResource); + } return model; } @@ -318,11 +350,11 @@ protected static Model addParameters(Model model, Resource benchmarkInstanceReso * Requests the status of the controller. * * @return the status of the controller - * @throws IOException - * If no response has been received + * @throws IOException If no response has been received */ public ControllerStatus requestStatus(String userName) throws IOException { - byte[] data = client.request(RabbitMQUtils.writeByteArrays(new byte[] { FrontEndApiCommands.LIST_CURRENT_STATUS }, + byte[] data = client + .request(RabbitMQUtils.writeByteArrays(new byte[] { FrontEndApiCommands.LIST_CURRENT_STATUS }, new byte[][] { RabbitMQUtils.writeString(userName) }, null)); if (data == null) { throw new IOException("Didn't get a response."); @@ -339,10 +371,8 @@ public ControllerStatus requestStatus(String userName) throws IOException { /** * Closes the challenge with the given URI. * - * @param challengeUri - * the URI of the challenge that should be closed - * @throws IOException - * If the controller does not responses + * @param challengeUri the URI of the challenge that should be closed + * @throws IOException If the controller does not responses */ public void closeChallenge(String challengeUri) throws IOException { LOGGER.info("Sending request..."); @@ -359,9 +389,8 @@ public void closeChallenge(String challengeUri) throws IOException { * Requests the systems for the user with the given user mail address. Returns * an empty list if an error occurs. * - * @param userMail - * the mail address of the user for which the systems should be - * requested + * @param userMail the mail address of the user for which the systems should be + * requested * @return the systems for the given user or an empty list if an error occurred */ public List requestSystemsOfUser(String userMail) { @@ -401,10 +430,8 @@ protected void transformSysMetaDataToBean(Collection systems, Li * Requests the deletion of the experiment with the given experiment id with the * access rights of the given user. * - * @param id - * id of the experiment that should be removed - * @param userName - * name of the user requesting the deletion + * @param id id of the experiment that should be removed + * @param userName name of the user requesting the deletion * @return {@code true} if the deletion was successful, else {@code false} */ public boolean requestExperimentDeletion(String id, String userName) { @@ -497,13 +524,11 @@ private static String expandedXsdId(String id) { * Sends a request to the platform controller to terminate the experiment with * the given ID using the access rights of the given user. * - * @param experimentId - * the id of the experiment that should be terminated. - * @param preferredUsername - * the name of the user who wants to terminate the experiment + * @param experimentId the id of the experiment that should be terminated. + * @param preferredUsername the name of the user who wants to terminate the + * experiment * @return {@code true} if the termination was successful, else {@code false} - * @throws IOException - * If communication problems arise. + * @throws IOException If communication problems arise. */ public boolean terminateExperiment(String experimentId, String preferredUsername) throws IOException { byte[] res = client.request(RabbitMQUtils.writeByteArrays(new byte[] { FrontEndApiCommands.REMOVE_EXPERIMENT }, diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/StorageServiceChannel.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/StorageServiceChannel.java deleted file mode 100644 index c90002fd..00000000 --- a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/StorageServiceChannel.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * This file is part of gui-serverbackend. - * - * gui-serverbackend is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * gui-serverbackend is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with gui-serverbackend. If not, see . - */ -package de.usu.research.hobbit.gui.rabbitmq; - - -import java.util.UUID; - -import org.hobbit.core.Constants; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.rabbitmq.client.AMQP.BasicProperties; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.QueueingConsumer; - -@Deprecated -public class StorageServiceChannel implements AutoCloseable { - - @SuppressWarnings("unused") - private static final Logger LOGGER = LoggerFactory.getLogger(StorageServiceChannel.class); - - /** - * Maximum number of retries that are executed to connect to RabbitMQ. - */ - public static final int NUMBER_OF_RETRIES_TO_CONNECT_TO_RABBIT_MQ = 5; - /** - * Time, the system waits before retrying to connect to RabbitMQ. Note that - * this time will be multiplied with the number of already failed tries. - */ - public static final long START_WAITING_TIME_BEFORE_RETRY = 5000; - - private Channel channel; - private String requestQueueName = Constants.STORAGE_QUEUE_NAME; - private String replyQueueName; - private QueueingConsumer consumer; - - public StorageServiceChannel(RabbitMQConnection connection) throws Exception { - channel = connection.createChannel(); - - replyQueueName = channel.queueDeclare().getQueue(); - consumer = new QueueingConsumer(channel); - channel.basicConsume(replyQueueName, true, consumer); - } - - public String call(String message) throws Exception { - String response; - String corrId = UUID.randomUUID().toString(); - - BasicProperties props = new BasicProperties - .Builder() - .correlationId(corrId) - .replyTo(replyQueueName) - .build(); - - channel.basicPublish("", requestQueueName, props, message.getBytes("UTF-8")); - - while (true) { - QueueingConsumer.Delivery delivery = consumer.nextDelivery(); - if (delivery.getProperties().getCorrelationId().equals(corrId)) { - response = new String(delivery.getBody(), "UTF-8"); - break; - } - } - - return response; - } - - @Override - public void close() throws Exception { - if (channel != null) { - try { - channel.close(); - } catch (Exception e) { - // ignore - } - } - } - -} diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/LicenseResources.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/LicenseResources.java index 1663f0ac..3fc32da6 100644 --- a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/LicenseResources.java +++ b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/LicenseResources.java @@ -16,11 +16,6 @@ */ package de.usu.research.hobbit.gui.rest; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Objects; - import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -36,7 +31,6 @@ import org.apache.jena.vocabulary.DCAT; import org.apache.jena.vocabulary.RDF; import org.hobbit.core.Constants; -import org.hobbit.core.data.status.ControllerStatus; import org.hobbit.storage.client.StorageServiceClient; import org.hobbit.storage.queries.SparqlQueries; import org.hobbit.utils.rdf.RdfHelper; @@ -45,11 +39,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.usu.research.hobbit.gui.rabbitmq.GUIBackendException; -import de.usu.research.hobbit.gui.rabbitmq.PlatformControllerClient; -import de.usu.research.hobbit.gui.rabbitmq.PlatformControllerClientSingleton; import de.usu.research.hobbit.gui.rabbitmq.StorageServiceClientSingleton; -import de.usu.research.hobbit.gui.rest.beans.*; +import de.usu.research.hobbit.gui.rest.beans.LicenseBean; @Path("license") public class LicenseResources { diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/HardwareConstraintBean.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/HardwareConstraintBean.java new file mode 100644 index 00000000..fc7332e1 --- /dev/null +++ b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/HardwareConstraintBean.java @@ -0,0 +1,79 @@ +package de.usu.research.hobbit.gui.rest.beans; + +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +public class HardwareConstraintBean { + + private String iri; + private String label; + private int cpuCount = -1; + private long memory = -1; + + /** + * @return the iri + */ + public String getIri() { + return iri; + } + + /** + * @param iri the iri to set + */ + public void setIri(String iri) { + this.iri = iri; + } + + /** + * @return the label + */ + public String getLabel() { + return label; + } + + /** + * @param label the label to set + */ + public void setLabel(String label) { + this.label = label; + } + + /** + * @return the cpuCount + */ + public int getCpuCount() { + return cpuCount; + } + + /** + * @param cpuCount the cpuCount to set + */ + public void setCpuCount(int cpuCount) { + this.cpuCount = cpuCount; + } + + /** + * @return the memory + */ + public long getMemory() { + return memory; + } + + /** + * @param memory the memory to set + */ + public void setMemory(long memory) { + this.memory = memory; + } + + /** + * Returns true if this constraint is valid, i.e., if it has an IRI assigned and + * it contains at least one requirement that can be processed by the platform. + * + * @return {@code true} if the constraint is valid, otherwise {@code false} + */ + public boolean isValidConstraint() { + return (iri != null) && (!iri.isEmpty()) && (cpuCount > 0) || (memory > 0); + } + +} diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/LicenseBean.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/LicenseBean.java index 21645114..da4543a7 100644 --- a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/LicenseBean.java +++ b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/LicenseBean.java @@ -1,7 +1,5 @@ package de.usu.research.hobbit.gui.rest.beans; -import java.util.List; - import javax.xml.bind.annotation.XmlRootElement; /** diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/SubmitModelBean.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/SubmitModelBean.java index 1c46b6a5..9209a1da 100644 --- a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/SubmitModelBean.java +++ b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/SubmitModelBean.java @@ -25,7 +25,7 @@ public class SubmitModelBean { private String benchmark; private String system; private List configurationParams; - + private HardwareConstraintBean maxHardwareConstraints; public List getConfigurationParams() { return configurationParams; @@ -45,4 +45,16 @@ public String getSystem() { public void setSystem(String system) { this.system = system; } + /** + * @return the maxHardwareConstraints + */ + public HardwareConstraintBean getMaxHardwareConstraints() { + return maxHardwareConstraints; + } + /** + * @param maxHardwareConstraints the maxHardwareConstraints to set + */ + public void setMaxHardwareConstraints(HardwareConstraintBean maxHardwareConstraints) { + this.maxHardwareConstraints = maxHardwareConstraints; + } } From 4dde753a933478acaa0a8612023d1046403f74bf Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Mon, 6 Nov 2023 11:18:31 +0100 Subject: [PATCH 13/52] Remove a temporary file --- hobbit-gui/gui-client/.package.json.swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 hobbit-gui/gui-client/.package.json.swp diff --git a/hobbit-gui/gui-client/.package.json.swp b/hobbit-gui/gui-client/.package.json.swp deleted file mode 100644 index c0c2605bbfdfccfa7d4f1afdbbe5712911100914..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2J#Q387{`Z{5O8GbNSjT|&b`_PE)e2HU_}U|V5CqO&F;;e*S9;fnVs|bY+=fj zR5X#2e1o8fhOa<_l7{pY^!@^f|Ln^hvCfw!P0(}F&wJiyp2yGpX7;3$Cu~04yzBLN zm*MplW7#i%{~*{^w)=>&G>&$JyZV5xzz9R!!pZ*|(OrW^OI|BZKXb!%w~ z1<3#zAOmE843GgbKnBPF8Th;m*z62@iYa|jO^a98GbgUquXG^;WPl8i0Wv@a$N(82 z17v^!@A;5B#&eg@CM5fESltbk+m_!hhZ zzk#2?2z(Fj0XK&(=m2_=0Wv@a$N(8217v^%8fg33i&vJTh@+UCvRPcX-FYRK@ZnR^|OQ+;7kC2TJ#4klS~iF*pW+Y)rr@NHX5_4=#Q^AXHlVs<3yN7;@;o{Zjei}7e_@0+g;wp zY^s*8)k7nlX6VMav%5tvrwl8j=#sA&HT=HZyD2lN`cg%)%#nKt8&9hG2U@xFzt!w6 z^H#U=X!fhSYiMUSugt`$RKu!)_9p?IWzy7@TD;ZfYxNw|a6bsOwz)MTD^gqh$~;vK z+W{6PEkws^7=)s<4GnVs=11)@`jCAD2EB-u From d169b44eb7a045454b47ba6b70e8eadc86ee69c9 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Mon, 6 Nov 2023 11:29:14 +0100 Subject: [PATCH 14/52] Fix gui-client build --- hobbit-gui/gui-serverbackend/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hobbit-gui/gui-serverbackend/Dockerfile b/hobbit-gui/gui-serverbackend/Dockerfile index 667eecc4..9376102f 100644 --- a/hobbit-gui/gui-serverbackend/Dockerfile +++ b/hobbit-gui/gui-serverbackend/Dockerfile @@ -1,5 +1,9 @@ # https://nodejs.org/en/docs/guides/nodejs-docker-webapp -FROM node:16 AS build-client + +# https://www.npmjs.com/package/node-sass#node-version-support-policy +# node-sass 4.10.0 (../gui-client/package-lock.json) ⇒ node <= 11 + +FROM node:11 AS build-client WORKDIR /usr/src/hobbit-platform COPY hobbit-gui/gui-client/package*.json ./hobbit-gui/gui-client/ RUN npm --prefix hobbit-gui/gui-client ci --omit=dev From 8c9719834fd4782adf38dec1efb6c7bda14a9356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Sat, 11 Nov 2023 12:41:58 +0100 Subject: [PATCH 15/52] Adapted file persmissions. --- config/db/storage-init.sh | 3 --- migrate-kibana-index.sh | 0 platform-controller/test_cmd.sh | 0 platform-controller/test_request_benchmark.sh | 0 platform-controller/test_request_benchmark_details.sh | 0 platform-controller/test_request_controller_status.sh | 0 platform-controller/test_request_system_resource_usage.sh | 0 platform-controller/test_start_benchmark.sh | 0 platform-controller/test_trigger_all_correlation_analysis.sh | 0 rabbitmq-cluster/cluster-entrypoint.sh | 0 run-storage-init.sh | 2 +- update-logstash-index.sh | 0 12 files changed, 1 insertion(+), 4 deletions(-) mode change 100755 => 100644 migrate-kibana-index.sh mode change 100755 => 100644 platform-controller/test_cmd.sh mode change 100755 => 100644 platform-controller/test_request_benchmark.sh mode change 100755 => 100644 platform-controller/test_request_benchmark_details.sh mode change 100755 => 100644 platform-controller/test_request_controller_status.sh mode change 100755 => 100644 platform-controller/test_request_system_resource_usage.sh mode change 100755 => 100644 platform-controller/test_start_benchmark.sh mode change 100755 => 100644 platform-controller/test_trigger_all_correlation_analysis.sh mode change 100755 => 100644 rabbitmq-cluster/cluster-entrypoint.sh mode change 100755 => 100644 update-logstash-index.sh diff --git a/config/db/storage-init.sh b/config/db/storage-init.sh index e73bbbcd..5d30c028 100644 --- a/config/db/storage-init.sh +++ b/config/db/storage-init.sh @@ -1,9 +1,6 @@ # Initialization script for the Storage of the HOBBIT Platform. # Controls access rights for Platform-specific RDF graphs. -echo "Waiting for port to open..." -while ! nc -q 1 localhost 1111 Date: Sat, 11 Nov 2023 12:43:11 +0100 Subject: [PATCH 16/52] Fixed issues with the hardware constraints. --- .../rabbitmq/PlatformControllerClient.java | 30 +++++++++++++++ .../hobbit/gui/rest/BenchmarksResources.java | 38 +++++++++++++++---- .../rest/beans/HardwareConstraintBean.java | 5 +++ 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/PlatformControllerClient.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/PlatformControllerClient.java index bab1dbdf..e3f24d50 100644 --- a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/PlatformControllerClient.java +++ b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rabbitmq/PlatformControllerClient.java @@ -39,6 +39,7 @@ import org.hobbit.core.data.status.QueuedExperiment; import org.hobbit.core.rabbit.RabbitMQUtils; import org.hobbit.core.rabbit.RabbitRpcClient; +import org.hobbit.utils.rdf.RdfHelper; import org.hobbit.vocab.HOBBIT; import org.hobbit.vocab.HobbitExperiments; import org.slf4j.Logger; @@ -210,6 +211,31 @@ public BenchmarkBean requestBenchmarkDetails(String benchmarkUri, UserInfoBean u return benchmarkDetails; } + /** + * Sends the given RDF model containing a benchmark configuration to the + * platform controller where an experiment will be started based on that model. + * + * @param model the RDF model containing all necessary information to start an + * experiment. + * @throws IOException If there is a problem during the receiving of the + * response + */ + public String submitBenchmark(Model model, String userName) throws GUIBackendException, IOException { + Resource benchmark = RdfHelper.getObjectResource(model, HobbitExperiments.New, HOBBIT.involvesBenchmark); + if ((benchmark == null) || (!benchmark.isURIResource())) { + String msg = "The given experiment configuration does not contain a benchmark IRI. Aborting."; + LOGGER.error(msg); + throw new GUIBackendException(msg); + } + Resource system = RdfHelper.getObjectResource(model, HobbitExperiments.New, HOBBIT.involvesSystemInstance); + if ((system == null) || (!system.isURIResource())) { + String msg = "The given experiment configuration does not contain a system instance IRI. Aborting."; + LOGGER.error(msg); + throw new GUIBackendException(msg); + } + return submitBenchmark(benchmark.getURI(), system.getURI(), model, userName); + } + /** * Sends the given benchmark configuration to the platform controller where an * experiment will be started with the chosen system and benchmark @@ -261,7 +287,11 @@ public String submitBenchmark(SubmitModelBean benchmarkConf, String userName) LOGGER.error("Got an exception while processing the hardware constraints.", e); throw new GUIBackendException("Please check your parameter definitions."); } + return submitBenchmark(benchmarkUri, systemUri, model, userName); + } + protected String submitBenchmark(String benchmarkUri, String systemUri, Model model, String userName) + throws IOException { byte[] data = RabbitMQUtils.writeByteArrays(new byte[] { FrontEndApiCommands.ADD_EXPERIMENT_CONFIGURATION }, new byte[][] { RabbitMQUtils.writeString(benchmarkUri), RabbitMQUtils.writeString(systemUri), RabbitMQUtils.writeModel(model), RabbitMQUtils.writeString(userName) }, diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/BenchmarksResources.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/BenchmarksResources.java index a027344e..c3c401d6 100644 --- a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/BenchmarksResources.java +++ b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/BenchmarksResources.java @@ -65,12 +65,13 @@ public Response listAll() { throw new GUIBackendException("Couldn't connect to platform controller."); } benchmarks = client.requestBenchmarks(); - } - catch (Exception ex) { - return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(InfoBean.withMessage(ex.getMessage())).build(); + } catch (Exception ex) { + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(InfoBean.withMessage(ex.getMessage())).build(); } } - return Response.ok(new GenericEntity>(benchmarks){}).build(); + return Response.ok(new GenericEntity>(benchmarks) { + }).build(); } @GET @@ -100,9 +101,9 @@ public Response getById(@Context SecurityContext sc, @PathParam("id") String id) benchmarkDetails.setSystems(new ArrayList<>(0)); } return Response.ok(benchmarkDetails).build(); - } - catch (Exception ex) { - return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(InfoBean.withMessage(ex.getMessage())).build(); + } catch (Exception ex) { + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(InfoBean.withMessage(ex.getMessage())).build(); } } } @@ -123,7 +124,28 @@ public Response submitBenchmark(@Context SecurityContext sc, SubmitModelBean mod return Response.ok(new SubmitResponseBean(id)).build(); } catch (Exception e) { LOGGER.warn("Failed to submit benchmark: " + e.getMessage()); - return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(InfoBean.withMessage(e.getMessage())).build(); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(InfoBean.withMessage(e.getMessage())) + .build(); } } + + // TODO: we should also accept RDF data to start an experiment +// @POST +// @Consumes("text/turtle") +// @Produces(MediaType.APPLICATION_JSON) +// public Response submitBenchmark(@Context SecurityContext sc, String modelString) { +// try { +// UserInfoBean userInfo = InternalResources.getUserInfoBean(sc); +// PlatformControllerClient client = PlatformControllerClientSingleton.getInstance(); +// if (client == null) { +// throw new GUIBackendException("Couldn't connect to platform controller."); +// } +// String id = client.submitBenchmark(model, userInfo.getPreferredUsername()); +// return Response.ok(new SubmitResponseBean(id)).build(); +// } catch (Exception e) { +// LOGGER.warn("Failed to submit benchmark: " + e.getMessage()); +// return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(InfoBean.withMessage(e.getMessage())).build(); +// } +// } + } diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/HardwareConstraintBean.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/HardwareConstraintBean.java index fc7332e1..27a8faa5 100644 --- a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/HardwareConstraintBean.java +++ b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/beans/HardwareConstraintBean.java @@ -1,6 +1,9 @@ package de.usu.research.hobbit.gui.rest.beans; +import java.beans.Transient; + import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; @XmlRootElement public class HardwareConstraintBean { @@ -72,6 +75,8 @@ public void setMemory(long memory) { * * @return {@code true} if the constraint is valid, otherwise {@code false} */ + @XmlTransient + @Transient public boolean isValidConstraint() { return (iri != null) && (!iri.isEmpty()) && (cpuCount > 0) || (memory > 0); } From 98c9d66f8b04dec1765283760d8c837076821094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Mon, 13 Nov 2023 10:55:43 +0100 Subject: [PATCH 17/52] Moved the web.xml used in cases where no authentication is needed into the GUI docker container. Added a small script that exchanges the two xml files in case the UI authentication flag is set to false. --- hobbit-gui/gui-serverbackend/Dockerfile | 13 ++++++++++--- .../main/webapp/WEB-INF}/web-without-ui-auth.xml | 0 .../src/main/webapp/WEB-INF/web.xml | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) rename {config/jetty => hobbit-gui/gui-serverbackend/src/main/webapp/WEB-INF}/web-without-ui-auth.xml (100%) diff --git a/hobbit-gui/gui-serverbackend/Dockerfile b/hobbit-gui/gui-serverbackend/Dockerfile index 9376102f..7ad7f619 100644 --- a/hobbit-gui/gui-serverbackend/Dockerfile +++ b/hobbit-gui/gui-serverbackend/Dockerfile @@ -3,6 +3,7 @@ # https://www.npmjs.com/package/node-sass#node-version-support-policy # node-sass 4.10.0 (../gui-client/package-lock.json) ⇒ node <= 11 +# Build web pages FROM node:11 AS build-client WORKDIR /usr/src/hobbit-platform COPY hobbit-gui/gui-client/package*.json ./hobbit-gui/gui-client/ @@ -10,6 +11,7 @@ RUN npm --prefix hobbit-gui/gui-client ci --omit=dev COPY hobbit-gui/gui-client ./hobbit-gui/gui-client RUN npm --prefix hobbit-gui/gui-client run build-prod +# Build Java backend FROM maven:3-eclipse-temurin-11 AS build WORKDIR /usr/src/hobbit-platform COPY parent-pom/pom.xml ./parent-pom/ @@ -20,14 +22,19 @@ RUN mvn --file ./hobbit-gui/gui-serverbackend/ test COPY --from=build-client /usr/src/hobbit-platform/hobbit-gui/gui-client/dist ./hobbit-gui/gui-client/dist RUN mvn --file ./hobbit-gui/gui-serverbackend/ package +# Set up web server FROM jetty:9.3-jre8 - +# Add script to register keycloak +COPY hobbit-gui/gui-serverbackend/configure-auth.sh $JETTY_BASE/configure-auth.sh +# Download the keycloak adapter and add it RUN cd $JETTY_BASE && \ curl -L -O http://hobbitdata.informatik.uni-leipzig.de/hobbit/keycloak-jetty93-adapter-for-hobbit-dist-2.4.0.Final.zip && \ unzip keycloak-jetty93-adapter-for-hobbit-dist-2.4.0.Final.zip && \ rm -f keycloak-jetty93-adapter-for-hobbit-dist-2.4.0.Final.zip && \ java -jar $JETTY_HOME/start.jar --add-to-startd=keycloak - +# Copy message definitions COPY hobbit-gui/gui-serverbackend/messages /var/lib/jetty/webapps/messages +# Copy Java backend +COPY --chown=jetty --chmod=777 --from=build /usr/src/hobbit-platform/hobbit-gui/gui-serverbackend/target/gui-serverbackend $JETTY_BASE/webapps/ROOT -COPY --from=build /usr/src/hobbit-platform/hobbit-gui/gui-serverbackend/target/gui-serverbackend $JETTY_BASE/webapps/ROOT +ENTRYPOINT ./configure-auth.sh && /docker-entrypoint.sh diff --git a/config/jetty/web-without-ui-auth.xml b/hobbit-gui/gui-serverbackend/src/main/webapp/WEB-INF/web-without-ui-auth.xml similarity index 100% rename from config/jetty/web-without-ui-auth.xml rename to hobbit-gui/gui-serverbackend/src/main/webapp/WEB-INF/web-without-ui-auth.xml diff --git a/hobbit-gui/gui-serverbackend/src/main/webapp/WEB-INF/web.xml b/hobbit-gui/gui-serverbackend/src/main/webapp/WEB-INF/web.xml index d38ea12b..1e4a3c49 100644 --- a/hobbit-gui/gui-serverbackend/src/main/webapp/WEB-INF/web.xml +++ b/hobbit-gui/gui-serverbackend/src/main/webapp/WEB-INF/web.xml @@ -1,3 +1,4 @@ + From c9da48f3578e1a20bef8778603632107146e2d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Mon, 13 Nov 2023 10:57:00 +0100 Subject: [PATCH 18/52] Added the missing configure-auth script. --- hobbit-gui/gui-serverbackend/configure-auth.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 hobbit-gui/gui-serverbackend/configure-auth.sh diff --git a/hobbit-gui/gui-serverbackend/configure-auth.sh b/hobbit-gui/gui-serverbackend/configure-auth.sh new file mode 100755 index 00000000..4646bf4f --- /dev/null +++ b/hobbit-gui/gui-serverbackend/configure-auth.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Runs the GUI serverbackend of the HOBBIT Platform. +USE_KEYCLOAK="${USE_UI_AUTH:-true}" + +if [ "${USE_KEYCLOAK}" == "false" ]; then + echo "Replacing web.xml with web-without-ui-auth.xml" + cp /var/lib/jetty/webapps/ROOT/WEB-INF/web-without-ui-auth.xml /var/lib/jetty/webapps/ROOT/WEB-INF/web.xml +fi + From 2823711755906cb4c11ed8d49f17d55037f72f73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 17 Nov 2023 17:44:24 +0100 Subject: [PATCH 19/52] Fixed pom version problem. --- analysis-component/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analysis-component/pom.xml b/analysis-component/pom.xml index 01a7e44a..cb0cd257 100644 --- a/analysis-component/pom.xml +++ b/analysis-component/pom.xml @@ -14,7 +14,7 @@ org.hobbit parent - ${hobbitplatform.version} + 2.0.17-SNAPSHOT ../parent-pom analysis-component From d3cb4ba26605d763fd1d72a6f2ad928de2315a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 17 Nov 2023 17:44:43 +0100 Subject: [PATCH 20/52] Removed user information that is not needed for retrieving the license information. --- .../java/de/usu/research/hobbit/gui/rest/LicenseResources.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/LicenseResources.java b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/LicenseResources.java index 3fc32da6..7f481202 100644 --- a/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/LicenseResources.java +++ b/hobbit-gui/gui-serverbackend/src/main/java/de/usu/research/hobbit/gui/rest/LicenseResources.java @@ -48,7 +48,7 @@ public class LicenseResources { @GET @Produces(MediaType.APPLICATION_JSON) - public Response getLicense(@Context SecurityContext sc) throws Exception { + public Response getLicense() throws Exception { LOGGER.info("Requesting dataset license..."); LicenseBean info = new LicenseBean(); String query = SparqlQueries.getLicenseOfDataset(Constants.PUBLIC_RESULT_GRAPH_URI); From 18614df56ed56c9639112f78d0411c696ef5647e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 17 Nov 2023 17:45:07 +0100 Subject: [PATCH 21/52] Fixed pom version problem. --- platform-storage/storage-service/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform-storage/storage-service/pom.xml b/platform-storage/storage-service/pom.xml index c637b1a4..ee3f98f6 100644 --- a/platform-storage/storage-service/pom.xml +++ b/platform-storage/storage-service/pom.xml @@ -23,7 +23,7 @@ org.hobbit parent - ${hobbitplatform.version} + 2.0.17-SNAPSHOT ../../parent-pom storage-service From 36365aec36dbeaacb439510b67a466f30fcd58c8 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Mon, 20 Nov 2023 13:32:44 +0100 Subject: [PATCH 22/52] Do not run tests on build --- hobbit-gui/gui-serverbackend/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hobbit-gui/gui-serverbackend/Dockerfile b/hobbit-gui/gui-serverbackend/Dockerfile index 9376102f..b04abf57 100644 --- a/hobbit-gui/gui-serverbackend/Dockerfile +++ b/hobbit-gui/gui-serverbackend/Dockerfile @@ -16,9 +16,8 @@ COPY parent-pom/pom.xml ./parent-pom/ COPY hobbit-gui/gui-serverbackend/pom.xml ./hobbit-gui/gui-serverbackend/ RUN mvn --file ./hobbit-gui/gui-serverbackend/ dependency:go-offline COPY hobbit-gui/gui-serverbackend/src ./hobbit-gui/gui-serverbackend/src -RUN mvn --file ./hobbit-gui/gui-serverbackend/ test COPY --from=build-client /usr/src/hobbit-platform/hobbit-gui/gui-client/dist ./hobbit-gui/gui-client/dist -RUN mvn --file ./hobbit-gui/gui-serverbackend/ package +RUN mvn --file ./hobbit-gui/gui-serverbackend/ -Dmaven.test.skip=true package FROM jetty:9.3-jre8 From 56df812acb515fa2ba13c940dd17fc9087087959 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Mon, 20 Nov 2023 13:33:35 +0100 Subject: [PATCH 23/52] Build platform-controller in Docker See #556. --- Makefile | 4 ++-- platform-controller/Dockerfile | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f1c04a12..e9c5bcf9 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ build-gui: # see hobbit-gui/gui-serverbackend/Dockerfile build-controller: - cd platform-controller && make build + # see platform-controller/Dockerfile build-storage: cd platform-storage/storage-service && mvn clean package -U @@ -45,7 +45,7 @@ build-analysis: build-dev-images: build-dev-platform-controller-image build-dev-gui-image build-dev-analysis-image build-dev-storage-image build-dev-platform-controller-image: - docker build -t hobbitproject/hobbit-platform-controller:dev ./platform-controller + docker build -t hobbitproject/hobbit-platform-controller:dev --file platform-controller/Dockerfile . build-dev-gui-image: docker build -t hobbitproject/hobbit-gui:dev --file hobbit-gui/gui-serverbackend/Dockerfile . diff --git a/platform-controller/Dockerfile b/platform-controller/Dockerfile index e5a8f9c9..2d9f6626 100644 --- a/platform-controller/Dockerfile +++ b/platform-controller/Dockerfile @@ -1,8 +1,17 @@ -FROM eclipse-temurin:11-focal +FROM maven:3-eclipse-temurin-11 AS build +WORKDIR /usr/src/hobbit-platform +COPY parent-pom/pom.xml parent-pom/ +ARG project=platform-controller +COPY ${project}/pom.xml ${project}/ +RUN mvn --file ${project} dependency:go-offline +COPY ${project}/src ${project}/src +RUN mvn --file ${project} -Dmaven.test.skip=true package + +FROM eclipse-temurin:11 # Create an empty metadata directory (it will be used as default by the file-based image manager) RUN mkdir -p /usr/src/app/metadata -COPY target/platform-controller.jar platform-controller.jar +COPY --from=build /usr/src/hobbit-platform/platform-controller/target/platform-controller.jar . CMD ["java", "-cp", "platform-controller.jar", "org.hobbit.core.run.ComponentStarter", "org.hobbit.controller.PlatformController"] From cf844589a2d61c3828d00adc4a7c703761d6a653 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Mon, 20 Nov 2023 15:26:54 +0100 Subject: [PATCH 24/52] Build storage-service in Docker See #556. --- Makefile | 4 ++-- platform-storage/storage-service/Dockerfile | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index e9c5bcf9..15a9dae0 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ build-controller: # see platform-controller/Dockerfile build-storage: - cd platform-storage/storage-service && mvn clean package -U + # see platform-storage/storage-service/Dockerfile build-analysis: cd analysis-component && mvn clean package -U @@ -54,7 +54,7 @@ build-dev-analysis-image: docker build -t hobbitproject/hobbit-analysis-component:dev ./analysis-component build-dev-storage-image: - docker build -t hobbitproject/hobbit-storage-service:dev ./platform-storage/storage-service + docker build -t hobbitproject/hobbit-storage-service:dev --file ./platform-storage/storage-service/Dockerfile . create-networks: @docker network inspect hobbit >/dev/null || (docker network create -d overlay --attachable --subnet 172.16.100.0/24 hobbit && echo "Created network: hobbit") diff --git a/platform-storage/storage-service/Dockerfile b/platform-storage/storage-service/Dockerfile index 299475f3..a3af1ec0 100644 --- a/platform-storage/storage-service/Dockerfile +++ b/platform-storage/storage-service/Dockerfile @@ -1,5 +1,12 @@ -FROM eclipse-temurin:11-focal - -COPY target/storage-service.jar storage-service.jar +FROM maven:3-eclipse-temurin-11 AS build +WORKDIR /usr/src/hobbit-platform +COPY parent-pom/pom.xml parent-pom/ +ARG project=platform-storage/storage-service +COPY ${project}/pom.xml ${project}/ +RUN mvn --file ${project} dependency:go-offline +COPY ${project}/src ${project}/src +RUN mvn --file ${project} -Dmaven.test.skip=true package +FROM eclipse-temurin:11 +COPY --from=build /usr/src/hobbit-platform/platform-storage/storage-service/target/storage-service.jar . CMD ["java", "-cp", "storage-service.jar", "org.hobbit.core.run.ComponentStarter", "org.hobbit.storage.service.StorageService"] From d17d2b09e4eda98bfd6405dd2b8833d4006fffc1 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Mon, 20 Nov 2023 15:35:39 +0100 Subject: [PATCH 25/52] Build analysis-component in Docker See #556. --- Makefile | 4 ++-- analysis-component/Dockerfile | 15 +++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 15a9dae0..0b670583 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ build-storage: # see platform-storage/storage-service/Dockerfile build-analysis: - cd analysis-component && mvn clean package -U + # see analysis-component/Dockerfile build-dev-images: build-dev-platform-controller-image build-dev-gui-image build-dev-analysis-image build-dev-storage-image @@ -51,7 +51,7 @@ build-dev-gui-image: docker build -t hobbitproject/hobbit-gui:dev --file hobbit-gui/gui-serverbackend/Dockerfile . build-dev-analysis-image: - docker build -t hobbitproject/hobbit-analysis-component:dev ./analysis-component + docker build -t hobbitproject/hobbit-analysis-component:dev --file ./analysis-component/Dockerfile . build-dev-storage-image: docker build -t hobbitproject/hobbit-storage-service:dev --file ./platform-storage/storage-service/Dockerfile . diff --git a/analysis-component/Dockerfile b/analysis-component/Dockerfile index c60531dd..627245fd 100644 --- a/analysis-component/Dockerfile +++ b/analysis-component/Dockerfile @@ -1,5 +1,12 @@ -FROM eclipse-temurin:11-focal +FROM maven:3-eclipse-temurin-11 AS build +WORKDIR /usr/src/hobbit-platform +COPY parent-pom/pom.xml parent-pom/ +ARG project=analysis-component +COPY ${project}/pom.xml ${project}/ +RUN mvn --file ${project} dependency:go-offline +COPY ${project}/src ${project}/src +RUN mvn --file ${project} -Dmaven.test.skip=true package -COPY target/analysis-component.jar . - -CMD java -cp analysis-component.jar org.hobbit.core.run.ComponentStarter org.hobbit.analysis.AnalysisComponent +FROM eclipse-temurin:11 +COPY --from=build /usr/src/hobbit-platform/analysis-component/target/analysis-component.jar . +CMD ["java", "-cp", "analysis-component.jar", "org.hobbit.core.run.ComponentStarter", "org.hobbit.analysis.AnalysisComponent"] From 2b79fef33fd5a1e6ffb1e60d24bc1376021d88e1 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Mon, 20 Nov 2023 17:36:33 +0100 Subject: [PATCH 26/52] Install parent-pom before building components --- analysis-component/Dockerfile | 1 + hobbit-gui/gui-serverbackend/Dockerfile | 1 + platform-controller/Dockerfile | 1 + platform-storage/storage-service/Dockerfile | 1 + 4 files changed, 4 insertions(+) diff --git a/analysis-component/Dockerfile b/analysis-component/Dockerfile index 627245fd..7019524f 100644 --- a/analysis-component/Dockerfile +++ b/analysis-component/Dockerfile @@ -1,6 +1,7 @@ FROM maven:3-eclipse-temurin-11 AS build WORKDIR /usr/src/hobbit-platform COPY parent-pom/pom.xml parent-pom/ +RUN mvn --file parent-pom -Dmaven.test.skip=true install ARG project=analysis-component COPY ${project}/pom.xml ${project}/ RUN mvn --file ${project} dependency:go-offline diff --git a/hobbit-gui/gui-serverbackend/Dockerfile b/hobbit-gui/gui-serverbackend/Dockerfile index b04abf57..6e98f83c 100644 --- a/hobbit-gui/gui-serverbackend/Dockerfile +++ b/hobbit-gui/gui-serverbackend/Dockerfile @@ -13,6 +13,7 @@ RUN npm --prefix hobbit-gui/gui-client run build-prod FROM maven:3-eclipse-temurin-11 AS build WORKDIR /usr/src/hobbit-platform COPY parent-pom/pom.xml ./parent-pom/ +RUN mvn --file parent-pom -Dmaven.test.skip=true install COPY hobbit-gui/gui-serverbackend/pom.xml ./hobbit-gui/gui-serverbackend/ RUN mvn --file ./hobbit-gui/gui-serverbackend/ dependency:go-offline COPY hobbit-gui/gui-serverbackend/src ./hobbit-gui/gui-serverbackend/src diff --git a/platform-controller/Dockerfile b/platform-controller/Dockerfile index 2d9f6626..da6db936 100644 --- a/platform-controller/Dockerfile +++ b/platform-controller/Dockerfile @@ -1,6 +1,7 @@ FROM maven:3-eclipse-temurin-11 AS build WORKDIR /usr/src/hobbit-platform COPY parent-pom/pom.xml parent-pom/ +RUN mvn --file parent-pom -Dmaven.test.skip=true install ARG project=platform-controller COPY ${project}/pom.xml ${project}/ RUN mvn --file ${project} dependency:go-offline diff --git a/platform-storage/storage-service/Dockerfile b/platform-storage/storage-service/Dockerfile index a3af1ec0..072e57ca 100644 --- a/platform-storage/storage-service/Dockerfile +++ b/platform-storage/storage-service/Dockerfile @@ -1,6 +1,7 @@ FROM maven:3-eclipse-temurin-11 AS build WORKDIR /usr/src/hobbit-platform COPY parent-pom/pom.xml parent-pom/ +RUN mvn --file parent-pom -Dmaven.test.skip=true install ARG project=platform-storage/storage-service COPY ${project}/pom.xml ${project}/ RUN mvn --file ${project} dependency:go-offline From 766971018437893b85905ca101c0c10f1a033e1c Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Mon, 20 Nov 2023 17:37:16 +0100 Subject: [PATCH 27/52] Cleanup Makefile --- Makefile | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 0b670583..53cfc5ff 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ -# build platform components -default: build +build: build-dev-images deploy: create-networks start @@ -26,22 +25,6 @@ start-dev-platform: start-dev-elk: docker-compose -f docker-compose-elk.yml up -d -build: build-java build-dev-images - -build-java: install-parent-pom build-controller build-storage build-analysis build-gui - -build-gui: - # see hobbit-gui/gui-serverbackend/Dockerfile - -build-controller: - # see platform-controller/Dockerfile - -build-storage: - # see platform-storage/storage-service/Dockerfile - -build-analysis: - # see analysis-component/Dockerfile - build-dev-images: build-dev-platform-controller-image build-dev-gui-image build-dev-analysis-image build-dev-storage-image build-dev-platform-controller-image: From 627b58cde5e8b22f1a7dd06a0463b69490264ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 8 Dec 2023 18:33:36 +0100 Subject: [PATCH 28/52] Added the handling of error reports. Errors are added to the result model using PROV-O properties. --- .../hobbit/controller/ExperimentManager.java | 66 ++++++ .../hobbit/controller/PlatformController.java | 57 ++++-- .../controller/data/ExperimentStatus.java | 1 - .../controller/docker/ContainerManager.java | 193 +++++++----------- .../docker/ContainerManagerImpl.java | 10 + .../mocks/DummyContainerManager.java | 5 + 6 files changed, 200 insertions(+), 132 deletions(-) diff --git a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java index 7f0fc932..ca3665be 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java @@ -27,15 +27,19 @@ import java.util.Set; import java.util.Timer; import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import org.apache.commons.io.IOUtils; import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.NodeIterator; import org.apache.jena.rdf.model.Property; import org.apache.jena.rdf.model.ResIterator; import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.ResourceFactory; import org.apache.jena.vocabulary.RDF; +import org.apache.jena.vocabulary.RDFS; import org.hobbit.controller.config.HobbitConfig; import org.hobbit.controller.data.ExperimentConfiguration; import org.hobbit.controller.data.ExperimentStatus; @@ -49,15 +53,18 @@ import org.hobbit.core.Commands; import org.hobbit.core.Constants; import org.hobbit.core.data.BenchmarkMetaData; +import org.hobbit.core.data.ErrorData; import org.hobbit.core.data.SystemMetaData; import org.hobbit.core.data.status.ControllerStatus; import org.hobbit.core.data.status.RunningExperiment; import org.hobbit.core.rabbit.RabbitMQUtils; import org.hobbit.utils.config.HobbitConfiguration; import org.hobbit.utils.rdf.RdfHelper; +import org.hobbit.vocab.Algorithm; import org.hobbit.vocab.HOBBIT; import org.hobbit.vocab.HobbitErrors; import org.hobbit.vocab.HobbitExperiments; +import org.hobbit.vocab.PROV; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,6 +103,10 @@ public class ExperimentManager implements Closeable { * to start. */ public static final long CHECK_FOR_NEW_EXPERIMENT = 10000; + /** + * IRI name space used for the generation of IRIs for error instances. + */ + public static final String ERROR_INSTANCE_NAMESPACE = "http://project-hobbit.org/error-instances/"; /** * Default time an experiment has to terminate after it has been started. */ @@ -118,6 +129,10 @@ public class ExperimentManager implements Closeable { * Timer used to trigger the creation of the next benchmark. */ protected Timer expStartTimer; + /** + * The last error Id that has been used for the currently running session. + */ + private AtomicInteger lastErrorReportId; /** * The configuration of this platform. */ @@ -789,4 +804,55 @@ public boolean isExpRunning(String sessionId) { public void setController(PlatformController controller) { this.controller = controller; } + + /** + * Add reported error to the experiment result model if the experiment with the + * given session is still running. + * + * @param sessionId the session ID of the container that reported the + * error + * @param errorData the data of the reported error + * @param isBenchmarkContainer a flag that shows whether the error came from a + * container that belongs to the benchmark + * ({@code true}) or to the benchmarked system + * ({@code false}). + */ + public void handleErrorReport(String sessionId, ErrorData errorData, boolean isBenchmarkContainer) { + // Transform the error data into RDF + Model resultModel = ModelFactory.createDefaultModel(); + // Generate IRI for this error + Resource error = ResourceFactory.createResource(new StringBuilder(ERROR_INSTANCE_NAMESPACE).append(sessionId) + .append('_').append(lastErrorReportId.getAndIncrement()).toString()); + // Add the error type + Resource errorType = (errorData.getErrorType() == null) ? HobbitErrors.UnspecifiedError + : ResourceFactory.createResource(errorData.getErrorType()); + resultModel.add(error, RDF.type, errorType); + // Add connection to experiment + resultModel.add(error, PROV.wasGeneratedBy, ResourceFactory.createResource(experimentStatus.experimentUri)); + + // Add source + if (isBenchmarkContainer) { + resultModel.add(error, PROV.wasAttributedTo, + ResourceFactory.createResource(experimentStatus.getConfig().benchmarkUri)); + } else { + resultModel.add(error, PROV.wasAttributedTo, + ResourceFactory.createResource(experimentStatus.getConfig().systemUri)); + } + // Add details + if (errorData.getLabel() != null) { + resultModel.add(error, RDFS.label, errorData.getLabel()); + } + if (errorData.getDescription() != null) { + resultModel.add(error, RDFS.comment, errorData.getDescription()); + } + if (errorData.getDetails() != null) { + resultModel.add(error, Algorithm.errorDetails, errorData.getLabel()); + } + synchronized (experimentMutex) { + // Check again whether we are still working on the same experiment + if (isExpRunning(sessionId)) { + setResultModel_unsecured(resultModel); + } + } + } } diff --git a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java index c1bbae92..16b82352 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java +++ b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java @@ -71,6 +71,7 @@ import org.hobbit.core.FrontEndApiCommands; import org.hobbit.core.components.AbstractComponent; import org.hobbit.core.data.BenchmarkMetaData; +import org.hobbit.core.data.ErrorData; import org.hobbit.core.data.StartCommandData; import org.hobbit.core.data.StopCommandData; import org.hobbit.core.data.SystemMetaData; @@ -80,6 +81,7 @@ import org.hobbit.core.data.usage.ResourceUsageInformation; import org.hobbit.core.rabbit.DataSender; import org.hobbit.core.rabbit.DataSenderImpl; +import org.hobbit.core.rabbit.GsonUtils; import org.hobbit.core.rabbit.RabbitMQUtils; import org.hobbit.storage.client.StorageServiceClient; import org.hobbit.storage.queries.SparqlQueries; @@ -379,7 +381,7 @@ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.Bas String containerName = ""; if (expManager.isExpRunning(sessionId)) { // Convert data byte array to config data structure - startParams = deserializeStartCommandData(data); + startParams = GsonUtils.deserializeObjectWithGson(gson, data, StartCommandData.class); // trigger creation containerName = createContainer(startParams); } else { @@ -411,7 +413,7 @@ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.Bas } case Commands.DOCKER_CONTAINER_STOP: { // get containerId from params - StopCommandData stopParams = deserializeStopCommandData(data); + StopCommandData stopParams = GsonUtils.deserializeObjectWithGson(gson, data, StopCommandData.class); // trigger stop stopContainer(stopParams.containerName); break; @@ -459,23 +461,27 @@ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.Bas } } } + case Commands.REPORT_ERROR: { + // Ensure that the container belongs to the current Experiment + if (expManager.isExpRunning(sessionId)) { + ErrorData errorData = GsonUtils.deserializeObjectWithGson(gson, data, ErrorData.class); + if (errorData != null) { + try { + handleErrorReport(sessionId, errorData); + } catch (Exception e) { + LOGGER.error("Exception while handling error report. It will be ignored.", e); + } + } else { + LOGGER.error("Couldn't parse error command received for experiment \"{}\". It will be ignored.", + sessionId); + } + } else { + LOGGER.error( + "Got an error report of the experiment \"{}\" which is either not running or was already stopped.", + sessionId); + } } - } - - private StopCommandData deserializeStopCommandData(byte[] data) { - if (data == null) { - return null; - } - String dataString = RabbitMQUtils.readString(data); - return gson.fromJson(dataString, StopCommandData.class); - } - - private StartCommandData deserializeStartCommandData(byte[] data) { - if (data == null) { - return null; } - String dataString = RabbitMQUtils.readString(data); - return gson.fromJson(dataString, StartCommandData.class); } /** @@ -788,6 +794,23 @@ public void handleFrontEndCmd(byte bytes[], String replyTo, BasicProperties repl LOGGER.debug("Finished handling of front end request."); } + private void handleErrorReport(String sessionId, ErrorData errorData) { + // Identify whether the container belongs to the benchmark or the system + if (errorData.getContainerId() == null) { + LOGGER.error("Got an error report without container ID. It will be ignored."); + return; + } + String containerType = containerManager.getContainerType(errorData.getContainerId()); + boolean isBenchmarkContainer = Constants.CONTAINER_TYPE_BENCHMARK.equals(containerType); + if (!isBenchmarkContainer && (!Constants.CONTAINER_TYPE_SYSTEM.equals(containerType))) { + LOGGER.error( + "Got an error report from a container with type \"{}\" which is neither a benchmark nor a system container. It will be ignored."); + return; + } + // Give the error report to the experiment manager + expManager.handleErrorReport(sessionId, errorData, isBenchmarkContainer); + } + /** * Retrieves model for the given challenge from the given graph (or without * selecting a certain graph if the graphUri is {@code null}). diff --git a/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentStatus.java b/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentStatus.java index 4b1a96a9..a74fd0d6 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentStatus.java +++ b/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentStatus.java @@ -37,7 +37,6 @@ import org.hobbit.controller.docker.ImageManager; import org.hobbit.controller.docker.MetaDataFactory; import org.hobbit.controller.execute.ExperimentAbortTimerTask; -import org.hobbit.core.Constants; import org.hobbit.core.rabbit.RabbitMQUtils; import org.hobbit.vocab.HOBBIT; import org.hobbit.vocab.HobbitExperiments; diff --git a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java index a67d6727..3daf465e 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManager.java @@ -19,6 +19,8 @@ import java.util.List; import java.util.Map; +import org.hobbit.core.Constants; + import com.spotify.docker.client.exceptions.DockerException; import com.spotify.docker.client.messages.ContainerStats; import com.spotify.docker.client.messages.swarm.Service; @@ -31,13 +33,12 @@ * */ public interface ContainerManager { - + public static final String MEMORY_LIMIT_CONSTRAINT = "memory-limit"; public static final String NANO_CPU_LIMIT_CONSTRAINT = "nanoCPU-limit"; /** - * Exit code of containers - * where process was terminated with SIGKILL (number 9). + * Exit code of containers where process was terminated with SIGKILL (number 9). */ public static long DOCKER_EXITCODE_SIGKILL = 128 + 9; @@ -53,8 +54,7 @@ public interface ContainerManager { /** * Start container with container type Benchmark and no parent * - * @param imageName - * Name of the image to start + * @param imageName Name of the image to start * * @return container id * @deprecated because the method tries to create a container with type=null and @@ -68,10 +68,8 @@ public interface ContainerManager { /** * Start container with container type Benchmark and no parent * - * @param imageName - * name of the image to start - * @param command - * command to be executed + * @param imageName name of the image to start + * @param command command to be executed * * @return container id * @deprecated because the method tries to create a container with type=null and @@ -85,12 +83,9 @@ public interface ContainerManager { /** * Start container with given image, type and parent * - * @param imageName - * name of the image to start - * @param type - * container type - * @param parent - * parent id + * @param imageName name of the image to start + * @param type container type + * @param parent parent id * * * @return container id @@ -100,14 +95,10 @@ public interface ContainerManager { /** * Starts the container with the given image name. * - * @param imageName - * name of the image to be started - * @param containerType - * type to be assigned to container - * @param parentId - * id of the parent container - * @param command - * commands that should be executed + * @param imageName name of the image to be started + * @param containerType type to be assigned to container + * @param parentId id of the parent container + * @param command commands that should be executed * * @return container Id or null if an error occurred. */ @@ -116,16 +107,11 @@ public interface ContainerManager { /** * Starts the container with the given image name. * - * @param imageName - * name of the image to be started - * @param containerType - * type to be assigned to container - * @param parentId - * id of the parent container - * @param env - * environment variables of the schema "key=value" - * @param command - * commands that should be executed + * @param imageName name of the image to be started + * @param containerType type to be assigned to container + * @param parentId id of the parent container + * @param env environment variables of the schema "key=value" + * @param command commands that should be executed * * @return container Id or null if an error occurred. */ @@ -135,18 +121,12 @@ public String startContainer(String imageName, String containerType, String pare /** * Starts the container with the given image name. * - * @param imageName - * name of the image to be started - * @param containerType - * type to be assigned to container - * @param parentId - * id of the parent container - * @param env - * environment variables of the schema "key=value" - * @param netAliases - * network aliases for this container - * @param command - * commands that should be executed + * @param imageName name of the image to be started + * @param containerType type to be assigned to container + * @param parentId id of the parent container + * @param env environment variables of the schema "key=value" + * @param netAliases network aliases for this container + * @param command commands that should be executed * * @return container Id or null if an error occurred. */ @@ -156,66 +136,45 @@ public String startContainer(String imageName, String containerType, String pare /** * Starts the container with the given image name. * - * @param imageName - * name of the image to be started - * @param containerType - * type to be assigned to container - * @param parentId - * id of the parent container - * @param env - * environment variables of the schema "key=value" - * @param command - * commands that should be executed - * @param pullImage - * whether the image needs to be prefetched + * @param imageName name of the image to be started + * @param containerType type to be assigned to container + * @param parentId id of the parent container + * @param env environment variables of the schema "key=value" + * @param command commands that should be executed + * @param pullImage whether the image needs to be prefetched * * @return container Id or null if an error occurred. */ public String startContainer(String imageName, String containerType, String parentId, String[] env, - String[] command, boolean pullImage); + String[] command, boolean pullImage); /** * Starts the container with the given image name. * - * @param imageName - * name of the image to be started - * @param containerType - * type to be assigned to container - * @param parentId - * id of the parent container - * @param env - * environment variables of the schema "key=value" - * @param netAliases - * network aliases for this container - * @param command - * commands that should be executed - * @param pullImage - * whether the image needs to be prefetched - * @param constraints - * Additional constraints for the container + * @param imageName name of the image to be started + * @param containerType type to be assigned to container + * @param parentId id of the parent container + * @param env environment variables of the schema "key=value" + * @param netAliases network aliases for this container + * @param command commands that should be executed + * @param pullImage whether the image needs to be prefetched + * @param constraints Additional constraints for the container * * @return container Id or null if an error occurred. */ public String startContainer(String imageName, String containerType, String parentId, String[] env, - String[] netAliases, String[] command, boolean pullImage, Map constraints); + String[] netAliases, String[] command, boolean pullImage, Map constraints); /** * Starts the container with the given image name. * - * @param imageName - * name of the image to be started - * @param containerType - * type to be assigned to container - * @param parentId - * id of the parent container - * @param env - * environment variables of the schema "key=value" - * @param command - * commands that should be executed - * @param experimentId - * experimentId to add to GELF tag - * @param constraints - * Additional constraints for the container + * @param imageName name of the image to be started + * @param containerType type to be assigned to container + * @param parentId id of the parent container + * @param env environment variables of the schema "key=value" + * @param command commands that should be executed + * @param experimentId experimentId to add to GELF tag + * @param constraints Additional constraints for the container * * @return container Id or null if an error occurred. */ @@ -225,8 +184,7 @@ public String startContainer(String imageName, String containerType, String pare /** * Stops the container with the given container Id. * - * @param containerId - * id of the container that should be stopped + * @param containerId id of the container that should be stopped * @deprecated use {@link #removeContainer(String)} instead. */ @Deprecated @@ -235,16 +193,14 @@ public String startContainer(String imageName, String containerType, String pare /** * Removes the container with the given container Id. * - * @param containerId - * id of the container that should be removed + * @param containerId id of the container that should be removed */ public void removeContainer(String serviceName); /** * Stops the parent container and all its children given the parent id * - * @param parentId - * id of the parent container + * @param parentId id of the parent container * @deprecated use {@link #removeParentAndChildren(String)} instead. */ @Deprecated @@ -253,8 +209,7 @@ public String startContainer(String imageName, String containerType, String pare /** * Removes the parent container and all its children given the parent id * - * @param parent - * id of the parent container + * @param parent id of the parent container */ public void removeParentAndChildren(String parent); @@ -282,26 +237,26 @@ public default List getContainers() { /** * Get a list of services which fulfill the given filter criteria. * - * @Service.Criteria criteria - * service criteria for filtering the list of services + * @Service.Criteria criteria service criteria for filtering the list of + * services */ public List getContainers(Service.Criteria criteria); /** - * @deprecated Platform uses names as IDs. - * Retrieves the container Id for the container with the given name or null if - * no such container could be found. + * @deprecated Platform uses names as IDs. Retrieves the container Id for the + * container with the given name or null if no such container could + * be found. */ - @Deprecated + @Deprecated public String getContainerId(String name); /** - * @deprecated Platform uses names as IDs. - * Returns the name of the container with the given Id or {@code null} if such a - * container can not be found + * @deprecated Platform uses names as IDs. Returns the name of the container + * with the given Id or {@code null} if such a container can not be + * found * - * @param containerId - * the Id of the container for which the name should be retrieved + * @param containerId the Id of the container for which the name should be + * retrieved * @return the name of the container with the given Id or {@code null} if such a * container can not be found */ @@ -311,16 +266,15 @@ public default List getContainers() { /** * Adds the given observer to the list of internal observers. * - * @param containerObserver - * the observer that should be added to the internal list + * @param containerObserver the observer that should be added to the internal + * list */ public void addContainerObserver(ContainerStateObserver containerObserver); /** * Pulls the image with the given name. * - * @param imageName - * the name of the image that should be pulled + * @param imageName the name of the image that should be pulled */ public void pullImage(String imageName); @@ -328,10 +282,21 @@ public default List getContainers() { * Returns statistics of the container with the given Id or {@code null} if the * container can not be found or an error occurs. * - * @param containerId - * the Id of the container for which statistics should be requested + * @param containerId the Id of the container for which statistics should be + * requested * @return statistics of the container with the given Id or {@code null} if the * container can not be found or an error occurs. */ public ContainerStats getStats(String containerId); + + /** + * Returns the type of the container as string. The type is typically one of + * {@link Constants#CONTAINER_TYPE_BENCHMARK}, + * {@link Constants#CONTAINER_TYPE_DATABASE} or + * {@link Constants#CONTAINER_TYPE_SYSTEM}. + * + * @param containerId + * @return + */ + String getContainerType(String containerId); } diff --git a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java index e06c1262..0d109e6c 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java +++ b/platform-controller/src/main/java/org/hobbit/controller/docker/ContainerManagerImpl.java @@ -777,4 +777,14 @@ public ContainerStats getStats(String containerId) { } return stats; } + + @Override + public String getContainerType(String containerId) { + Service container = null; + try { + container = getContainerInfo(containerId); + } catch (Exception e) { + } + return (container == null) ? null : container.spec().labels().get(LABEL_TYPE); + } } diff --git a/platform-controller/src/test/java/org/hobbit/controller/mocks/DummyContainerManager.java b/platform-controller/src/test/java/org/hobbit/controller/mocks/DummyContainerManager.java index cfcdd7ac..24e8ffe5 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/mocks/DummyContainerManager.java +++ b/platform-controller/src/test/java/org/hobbit/controller/mocks/DummyContainerManager.java @@ -148,4 +148,9 @@ public ContainerStats getStats(String containerId) { return null; } + @Override + public String getContainerType(String containerId) { + return null; + } + } From 1eb5c711df427af844b8fec9c6f8280d5689c157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Thu, 14 Dec 2023 14:08:44 +0100 Subject: [PATCH 29/52] Included the new blocking rabbitmq receiver in the storage service. --- platform-storage/storage-service/pom.xml | 1 - .../storage/service/DeliveryProcessing.java | 5 ++-- .../storage/service/StorageService.java | 26 ++++++++----------- .../hobbit/storage/service/TestRPCClient.java | 14 +++++----- 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/platform-storage/storage-service/pom.xml b/platform-storage/storage-service/pom.xml index ee3f98f6..14b8c3bf 100644 --- a/platform-storage/storage-service/pom.xml +++ b/platform-storage/storage-service/pom.xml @@ -76,7 +76,6 @@ org.slf4j slf4j-log4j12 - ${slf4j.version} diff --git a/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/DeliveryProcessing.java b/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/DeliveryProcessing.java index c5b40375..1154fc36 100644 --- a/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/DeliveryProcessing.java +++ b/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/DeliveryProcessing.java @@ -23,16 +23,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.rabbitmq.client.QueueingConsumer; import com.rabbitmq.client.AMQP.BasicProperties; -import com.rabbitmq.client.QueueingConsumer.Delivery; +import com.rabbitmq.client.Delivery; public class DeliveryProcessing implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(DeliveryProcessing.class); private StorageService storage; - private QueueingConsumer.Delivery delivery; + private Delivery delivery; private RabbitQueue queue; public DeliveryProcessing(StorageService storage, Delivery delivery, RabbitQueue queue) { diff --git a/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/StorageService.java b/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/StorageService.java index 51ee0272..99a37490 100644 --- a/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/StorageService.java +++ b/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/StorageService.java @@ -49,10 +49,11 @@ import org.hobbit.core.Constants; import org.hobbit.core.components.AbstractComponent; import org.hobbit.core.data.RabbitQueue; +import org.hobbit.core.rabbit.QueueingConsumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.rabbitmq.client.QueueingConsumer; +import com.rabbitmq.client.Delivery; /** * @@ -127,11 +128,10 @@ public class StorageService extends AbstractComponent implements CredentialsProv /** * Calls the SPARQL Endpoint denoted by the URL, to execute the queryString. * - * @param queryString - * The query to be executed + * @param queryString The query to be executed * @return Returns the queryString results serialized in JSON - * @throws Exception - * If endpoint not reachable, exception while executing query, etc. + * @throws Exception If endpoint not reachable, exception while executing query, + * etc. */ public String callSparqlEndpoint(String queryString) throws Exception { String response = null; @@ -236,7 +236,7 @@ public void run() throws Exception { LOGGER.info("[Storage Service] Awaiting Storage Service requests"); ExecutorService executor = Executors.newFixedThreadPool(MAX_NUMBER_PARALLEL_REQUESTS); while (true) { - QueueingConsumer.Delivery delivery = consumer.nextDelivery(); + Delivery delivery = consumer.getDeliveryQueue().poll(); executor.execute(new DeliveryProcessing(this, delivery, queue)); } } @@ -246,8 +246,7 @@ public void run() throws Exception { * outside of brackets, i.e.,parts that match {{@code ...}} and {@code <...>} * are removed. It can be used to make sure that only keywords are processed. * - * @param query - * the SPARQL query that should be reduced + * @param query the SPARQL query that should be reduced * @return the reduced SPARQL query */ protected static String reduceQueryToKeyWords(String query) { @@ -278,16 +277,13 @@ protected static String reduceQueryToKeyWords(String query) { * true an {@link IllegalStateException} is thrown. If the flag is * false, null is returned. * - * @param key - * the name of the environmental variable - * @param essential - * a flag indicating whether the value must be retrievable + * @param key the name of the environmental variable + * @param essential a flag indicating whether the value must be retrievable * @return the value of the environmental variable or null if the * variable couldn't be found and the essential flag is * false. - * @throws IllegalStateException - * if the variable couldn't be found and the essential flag is - * true. + * @throws IllegalStateException if the variable couldn't be found and the + * essential flag is true. */ protected String getEnvValue(String key, boolean essential) { String value = null; diff --git a/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/TestRPCClient.java b/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/TestRPCClient.java index 95ca1ed8..35a19c0d 100644 --- a/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/TestRPCClient.java +++ b/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/TestRPCClient.java @@ -21,11 +21,6 @@ */ package org.hobbit.storage.service; -import com.rabbitmq.client.ConnectionFactory; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.QueueingConsumer; -import com.rabbitmq.client.AMQP.BasicProperties; import java.util.UUID; import org.apache.commons.io.IOUtils; @@ -34,10 +29,17 @@ import org.apache.jena.vocabulary.RDF; import org.apache.jena.vocabulary.RDFS; import org.hobbit.core.Constants; +import org.hobbit.core.rabbit.QueueingConsumer; import org.hobbit.storage.client.StorageServiceClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.rabbitmq.client.AMQP.BasicProperties; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; +import com.rabbitmq.client.Delivery; + /** * * @author Milos Jovanovik (mjovanovik@openlinksw.com) @@ -113,7 +115,7 @@ public String call(String message) throws Exception { channel.basicPublish("", requestQueueName, props, message.getBytes("UTF-8")); while (true) { - QueueingConsumer.Delivery delivery = consumer.nextDelivery(); + Delivery delivery = consumer.getDeliveryQueue().poll(); if (delivery.getProperties().getCorrelationId().equals(corrId)) { response = new String(delivery.getBody(), "UTF-8"); break; From 7cbb4e7a7682e4429fe3564b8a15056ebbf73c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Thu, 14 Dec 2023 14:09:04 +0100 Subject: [PATCH 30/52] Included the new blocking rabbitmq receiver in the analysis component. --- .../hobbit/analysis/AnalysisComponent.java | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java b/analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java index 7715c165..77c7c901 100644 --- a/analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java +++ b/analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java @@ -16,22 +16,41 @@ */ package org.hobbit.analysis; -import java.util.stream.Stream; -import java.util.stream.Collectors; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.aksw.palmetto.evaluate.correlation.PearsonsSampleCorrelationCoefficient; +import org.apache.commons.io.IOUtils; import org.apache.jena.datatypes.RDFDatatype; import org.apache.jena.datatypes.xsd.XSDDatatype; -import org.apache.jena.rdf.model.*; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.ResIterator; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.ResourceFactory; +import org.apache.jena.rdf.model.Statement; +import org.apache.jena.rdf.model.StmtIterator; import org.apache.jena.vocabulary.DCTerms; import org.apache.jena.vocabulary.RDF; import org.apache.jena.vocabulary.XSD; -import org.apache.commons.io.IOUtils; import org.hobbit.core.Constants; import org.hobbit.core.components.AbstractComponent; import org.hobbit.core.data.RabbitQueue; +import org.hobbit.core.rabbit.QueueingConsumer; import org.hobbit.core.rabbit.RabbitMQUtils; import org.hobbit.storage.client.StorageServiceClient; import org.hobbit.storage.queries.SparqlQueries; @@ -42,13 +61,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.rabbitmq.client.QueueingConsumer; +import com.rabbitmq.client.Delivery; -import weka.attributeSelection.*; import weka.attributeSelection.AttributeSelection; +import weka.attributeSelection.CfsSubsetEval; +import weka.attributeSelection.GreedyStepwise; import weka.classifiers.functions.LinearRegression; import weka.clusterers.SimpleKMeans; -import weka.core.*; +import weka.core.Attribute; +import weka.core.DenseInstance; +import weka.core.Instance; +import weka.core.Instances; +import weka.core.SparseInstance; /** @@ -82,9 +106,9 @@ public void init() throws Exception { @Override public void run() throws Exception { LOGGER.info("Awaiting requests"); - QueueingConsumer.Delivery delivery; + Delivery delivery; while (true) { - delivery = consumer.nextDelivery(); + delivery = consumer.getDeliveryQueue().poll(); if (delivery != null) { LOGGER.info("Received a request. Processing..."); String expUri = RabbitMQUtils.readString(delivery.getBody()); From 5d5eae99c17770ec8c0d3719d67780f4620e8348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 15 Dec 2023 14:43:18 +0100 Subject: [PATCH 31/52] Updated dev docker-compose. --- docker-compose-dev.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 694e359c..9d1197ba 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -9,16 +9,20 @@ services: HOBBIT_RABBIT_IMAGE: "rabbitmq:management" HOBBIT_RABBIT_HOST: "rabbit" HOBBIT_REDIS_HOST: "redis" - DEPLOY_ENV: "testing" - GITLAB_USER: "${GITLAB_USER}" - GITLAB_EMAIL: "${GITLAB_EMAIL}" - GITLAB_TOKEN: "${GITLAB_TOKEN}" - LOGGING_GELF_ADDRESS: "udp://localhost:12201" + DEPLOY_ENV: "develop" + #GITLAB_USER: "${GITLAB_USER}" + #GITLAB_EMAIL: "${GITLAB_EMAIL}" + #GITLAB_TOKEN: "${GITLAB_TOKEN}" + #LOGGING_GELF_ADDRESS: "udp://localhost:12201" SWARM_NODE_NUMBER: "1" PROMETHEUS_HOST: prometheus PROMETHEUS_PORT: 9090 + USE_GITLAB: "false" + LOCAL_METADATA_DIRECTORY: "/metadata" + DOCKER_AUTOPULL: "0" volumes: - /var/run/docker.sock:/var/run/docker.sock + - ./metadata:/metadata # HOBBIT GUI gui: @@ -37,8 +41,8 @@ services: - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_HTTP_PORT=9200 - USE_UI_AUTH=false - volumes: - - ./config/jetty/web-without-ui-auth.xml:/var/lib/jetty/webapps/ROOT/WEB-INF/web.xml + #volumes: + #- ./config/jetty/web-without-ui-auth.xml:/var/lib/jetty/webapps/ROOT/WEB-INF/web.xml #- /data/docker/messages/global.html:/var/lib/jetty/webapps/messages/global.html #- /data/docker/messages/benchmark.html:/var/lib/jetty/webapps/messages/benchmark.html #- /data/docker/messages/status.html:/var/lib/jetty/webapps/messages/status.html @@ -92,7 +96,8 @@ services: stop_signal: SIGINT stop_grace_period: 2m volumes: - - ./config/db:/opt/virtuoso-opensource/database + - ./config/db:/opt/virtuoso-opensource/var/lib/virtuoso/db + #- ./config/db:/opt/virtuoso-opensource/database networks: - hobbit-core ports: From 861895fefbbcbaf872d9c2bb4f098423d470189d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Sat, 16 Dec 2023 18:13:31 +0100 Subject: [PATCH 32/52] Fixed problems with the queue-based consumer. --- .../hobbit/analysis/AnalysisComponent.java | 26 ++++++++++++++----- .../storage/service/StorageService.java | 22 ++++++++++++++-- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java b/analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java index 77c7c901..399a4daa 100644 --- a/analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java +++ b/analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java @@ -28,6 +28,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -80,6 +81,12 @@ * TODO:: !!REFACTOR INTO A MORE GENERIC DESIGN!! */ public class AnalysisComponent extends AbstractComponent { + /** + * The time the main receiver thread will sleep if it didn't receive any message + * (in seconds). + */ + private static final int WAITING_TIME_BEFORE_CHECKING_STATUS = 60; + private static final Logger LOGGER = LoggerFactory.getLogger(AnalysisComponent.class); private static final String GRAPH_URI = Constants.PUBLIC_RESULT_GRAPH_URI; protected RabbitQueue controller2AnalysisQueue; @@ -108,14 +115,22 @@ public void run() throws Exception { LOGGER.info("Awaiting requests"); Delivery delivery; while (true) { - delivery = consumer.getDeliveryQueue().poll(); + delivery = null; + // Let's wait for a delivery for 60 seconds + try { + delivery = consumer.getDeliveryQueue().poll(WAITING_TIME_BEFORE_CHECKING_STATUS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // interrupted; just continue + } if (delivery != null) { LOGGER.info("Received a request. Processing..."); String expUri = RabbitMQUtils.readString(delivery.getBody()); handleRequest(expUri); + } else { + // This would be the place at which we could react to signals, e.g., terminate + // the service if needed. } } - } protected void handleRequest(String expUri) { @@ -259,18 +274,17 @@ public void close() throws IOException { * The model of the experiment * @return Returns the analyzed model */ - - /** + /* private AnalysisModel analyseExperiment(Model experimentModel, String expUri){ AnalysisModel analysisModel = new AnalysisModel(experimentModel, expUri); analysisModel.analyse(); return analysisModel; }*/ - private void notifyQueue(){ + /*private void notifyQueue(){ //TODO:: return the status of component - } + }*/ /** * This class implements the functionality of the Analysis Model which diff --git a/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/StorageService.java b/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/StorageService.java index 99a37490..66f50db0 100644 --- a/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/StorageService.java +++ b/platform-storage/storage-service/src/main/java/org/hobbit/storage/service/StorageService.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.aksw.jena_sparql_api.core.QueryExecutionFactory; import org.aksw.jena_sparql_api.http.QueryExecutionFactoryHttp; @@ -65,6 +66,11 @@ public class StorageService extends AbstractComponent implements CredentialsProv private static final Logger LOGGER = LoggerFactory.getLogger(StorageService.class); private static final int MAX_NUMBER_PARALLEL_REQUESTS = 10; + /** + * The time the main receiver thread will sleep if it didn't receive any message + * (in seconds). + */ + private static final int WAITING_TIME_BEFORE_CHECKING_STATUS = 60; /** * Maximum result size used for pagination. @@ -235,9 +241,21 @@ public void init() throws Exception { public void run() throws Exception { LOGGER.info("[Storage Service] Awaiting Storage Service requests"); ExecutorService executor = Executors.newFixedThreadPool(MAX_NUMBER_PARALLEL_REQUESTS); + Delivery delivery; while (true) { - Delivery delivery = consumer.getDeliveryQueue().poll(); - executor.execute(new DeliveryProcessing(this, delivery, queue)); + delivery = null; + // Let's wait for a delivery for 60 seconds + try { + delivery = consumer.getDeliveryQueue().poll(WAITING_TIME_BEFORE_CHECKING_STATUS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // interrupted; just continue + } + if (delivery != null) { + executor.execute(new DeliveryProcessing(this, delivery, queue)); + } else { + // This would be the place at which we could react to signals, e.g., terminate + // the service if needed. + } } } From 2e630e57db1ecdf07c302cd28ffba9fda86439e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Sat, 16 Dec 2023 18:14:12 +0100 Subject: [PATCH 33/52] Finished implementation of error report handling. --- .../hobbit/controller/ExperimentManager.java | 12 +- .../hobbit/controller/PlatformController.java | 19 +-- .../controller/data/ExperimentStatus.java | 124 ++++++++---------- .../controller/utils/RabbitMQConnector.java | 8 +- 4 files changed, 78 insertions(+), 85 deletions(-) diff --git a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java index ca3665be..4416b348 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java @@ -27,7 +27,6 @@ import java.util.Set; import java.util.Timer; import java.util.TimerTask; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import org.apache.commons.io.IOUtils; @@ -129,10 +128,6 @@ public class ExperimentManager implements Closeable { * Timer used to trigger the creation of the next benchmark. */ protected Timer expStartTimer; - /** - * The last error Id that has been used for the currently running session. - */ - private AtomicInteger lastErrorReportId; /** * The configuration of this platform. */ @@ -208,6 +203,7 @@ public void createNextExperiment() { BenchmarkMetaData benchmark = controller.imageManager().getBenchmark(config.benchmarkUri); if ((benchmark == null) || (benchmark.mainImage == null)) { + // Think about reusing the existing object created above experimentStatus = new ExperimentStatus(config, HobbitExperiments.getExperimentURI(config.id), this, defaultMaxExecutionTime); experimentStatus.addError(HobbitErrors.BenchmarkImageMissing); @@ -216,6 +212,7 @@ public void createNextExperiment() { SystemMetaData system = controller.imageManager().getSystem(config.systemUri); if ((system == null) || (system.mainImage == null)) { + // Think about reusing the existing object created above experimentStatus = new ExperimentStatus(config, HobbitExperiments.getExperimentURI(config.id), this, defaultMaxExecutionTime); experimentStatus.addError(HobbitErrors.SystemImageMissing); @@ -556,6 +553,7 @@ private synchronized void handleExperimentTermination_unsecured() { */ private void forceBenchmarkTerminate_unsecured(Resource error) { if (experimentStatus != null) { + experimentStatus.setState(States.STOPPED); String parent = experimentStatus.getRootContainer(); controller.containerManager.removeParentAndChildren(parent); if (error != null) { @@ -822,7 +820,7 @@ public void handleErrorReport(String sessionId, ErrorData errorData, boolean isB Model resultModel = ModelFactory.createDefaultModel(); // Generate IRI for this error Resource error = ResourceFactory.createResource(new StringBuilder(ERROR_INSTANCE_NAMESPACE).append(sessionId) - .append('_').append(lastErrorReportId.getAndIncrement()).toString()); + .append('_').append(experimentStatus.getNextErrorReportId()).toString()); // Add the error type Resource errorType = (errorData.getErrorType() == null) ? HobbitErrors.UnspecifiedError : ResourceFactory.createResource(errorData.getErrorType()); @@ -846,7 +844,7 @@ public void handleErrorReport(String sessionId, ErrorData errorData, boolean isB resultModel.add(error, RDFS.comment, errorData.getDescription()); } if (errorData.getDetails() != null) { - resultModel.add(error, Algorithm.errorDetails, errorData.getLabel()); + resultModel.add(error, Algorithm.errorDetails, errorData.getDetails()); } synchronized (experimentMutex) { // Check again whether we are still working on the same experiment diff --git a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java index 16b82352..ea1ffde6 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java +++ b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java @@ -351,15 +351,18 @@ public void closeExpRabbitMQConnector() { *

    * Commands handled by this method: *

      + *
    • {@link Commands#BENCHMARK_FINISHED_SIGNAL}
      • + *
    • {@link Commands#BENCHMARK_READY_SIGNAL}
      • *
    • {@link Commands#DOCKER_CONTAINER_START}
      • *
    • {@link Commands#DOCKER_CONTAINER_STOP}
      • + *
    • {@link Commands#REPORT_ERROR}
      • + *
    • {@link Commands#REQUEST_SYSTEM_RESOURCES_USAGE}
      • + *
    • {@link Commands#SYSTEM_READY_SIGNAL}
      • + *
    • {@link Commands#TASK_GENERATION_FINISHED}
      • *
    * * @param command command to be executed * @param data byte-encoded supplementary json for the command - * - * 0 - start container 1 - stop container Data format for each - * command: Start container: */ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.BasicProperties props) { String replyTo = null; @@ -373,15 +376,14 @@ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.Bas } else { LOGGER.info("received command: session={}, command={}", sessionId, Commands.toString(command)); } - // This command will receive data from Rabbit - // determine the command + // Determine the command switch (command) { case Commands.DOCKER_CONTAINER_START: { StartCommandData startParams = null; String containerName = ""; if (expManager.isExpRunning(sessionId)) { // Convert data byte array to config data structure - startParams = GsonUtils.deserializeObjectWithGson(gson, data, StartCommandData.class); + startParams = GsonUtils.deserializeObjectWithGson(gson, data, StartCommandData.class, false); // trigger creation containerName = createContainer(startParams); } else { @@ -413,7 +415,7 @@ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.Bas } case Commands.DOCKER_CONTAINER_STOP: { // get containerId from params - StopCommandData stopParams = GsonUtils.deserializeObjectWithGson(gson, data, StopCommandData.class); + StopCommandData stopParams = GsonUtils.deserializeObjectWithGson(gson, data, StopCommandData.class, false); // trigger stop stopContainer(stopParams.containerName); break; @@ -462,9 +464,10 @@ public void receiveCommand(byte command, byte[] data, String sessionId, AMQP.Bas } } case Commands.REPORT_ERROR: { + LOGGER.warn("Received error report for session {}.", sessionId); // Ensure that the container belongs to the current Experiment if (expManager.isExpRunning(sessionId)) { - ErrorData errorData = GsonUtils.deserializeObjectWithGson(gson, data, ErrorData.class); + ErrorData errorData = GsonUtils.deserializeObjectWithGson(gson, data, ErrorData.class, false); if (errorData != null) { try { handleErrorReport(sessionId, errorData); diff --git a/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentStatus.java b/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentStatus.java index a74fd0d6..6ee359fb 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentStatus.java +++ b/platform-controller/src/main/java/org/hobbit/controller/data/ExperimentStatus.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.Timer; import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; @@ -150,15 +151,17 @@ private States(String description) { * Timer used to abort the experiment if it takes too much time. */ private final Timer abortTimer; + /** + * The next error Id that will be used for the currently running session. + */ + private AtomicInteger nextErrorReportId = new AtomicInteger(0); /** * Creates an experiment status with the given experiment config, the given * experiment URI and the current system time as start time. * - * @param config - * the configuration of the experiment - * @param experimentUri - * the URI of the experiment + * @param config the configuration of the experiment + * @param experimentUri the URI of the experiment */ public ExperimentStatus(ExperimentConfiguration config, String experimentUri) { this(config, experimentUri, null, 0, System.currentTimeMillis()); @@ -170,12 +173,9 @@ public ExperimentStatus(ExperimentConfiguration config, String experimentUri) { * timer using the given maximum runtime of the experiment and the experiment * manager which will be used to abort the experiment if the time is exceeded. * - * @param config - * the configuration of the experiment - * @param experimentUri - * the URI of the experiment - * @param startTimeStamp - * the time stamp at which the experiment is started. + * @param config the configuration of the experiment + * @param experimentUri the URI of the experiment + * @param startTimeStamp the time stamp at which the experiment is started. */ public ExperimentStatus(ExperimentConfiguration config, String experimentUri, long startTimeStamp) { this(config, experimentUri, null, 0, System.currentTimeMillis()); @@ -188,16 +188,12 @@ public ExperimentStatus(ExperimentConfiguration config, String experimentUri, lo * experiment manager which will be used to abort the experiment if the time is * exceeded. * - * @param config - * the configuration of the experiment - * @param experimentUri - * the URI of the experiment - * @param manager - * experiment manager which is used if the maximum runtime is - * exceeded - * @param timeUntilAborting - * the maximum runtime for this experiment which is used to configure - * the internal timer. + * @param config the configuration of the experiment + * @param experimentUri the URI of the experiment + * @param manager experiment manager which is used if the maximum + * runtime is exceeded + * @param timeUntilAborting the maximum runtime for this experiment which is + * used to configure the internal timer. */ public ExperimentStatus(ExperimentConfiguration config, String experimentUri, ExperimentManager manager, long timeUntilAborting) { @@ -210,18 +206,13 @@ public ExperimentStatus(ExperimentConfiguration config, String experimentUri, Ex * timer using the given maximum runtime of the experiment and the experiment * manager which will be used to abort the experiment if the time is exceeded. * - * @param config - * the configuration of the experiment - * @param experimentUri - * the URI of the experiment - * @param manager - * experiment manager which is used if the maximum runtime is - * exceeded - * @param timeUntilAborting - * the maximum runtime for this experiment which is used to configure - * the internal timer. - * @param startTimeStamp - * the time stamp at which the experiment is started. + * @param config the configuration of the experiment + * @param experimentUri the URI of the experiment + * @param manager experiment manager which is used if the maximum + * runtime is exceeded + * @param timeUntilAborting the maximum runtime for this experiment which is + * used to configure the internal timer. + * @param startTimeStamp the time stamp at which the experiment is started. */ public ExperimentStatus(ExperimentConfiguration config, String experimentUri, ExperimentManager manager, long timeUntilAborting, long startTimeStamp) { @@ -306,8 +297,7 @@ public long getAbortionTimeStamp() { /** * Adds an image to the set of images used in this experiment. * - * @param image - * image name to add + * @param image image name to add */ public void addImage(String image) { usedImages.add(image); @@ -327,12 +317,10 @@ public Set getUsedImages() { * and the experiment manager which will be used to abort the experiment if the * time is exceeded. * - * @param manager - * experiment manager which is used if the maximum runtime is - * exceeded - * @param timeUntilAborting - * the maximum runtime for this experiment which is used to configure - * the internal timer. + * @param manager experiment manager which is used if the maximum + * runtime is exceeded + * @param timeUntilAborting the maximum runtime for this experiment which is + * used to configure the internal timer. */ public void startAbortionTimer(ExperimentManager manager, long timeUntilAborting) { abortionTimeStamp = System.currentTimeMillis() + timeUntilAborting; @@ -345,9 +333,8 @@ public void startAbortionTimer(ExperimentManager manager, long timeUntilAborting * benchmark is ready and returns true if internally both have the * state of being ready. * - * @param systemReportedReady - * true if the system is ready or false if - * the benchmark is ready + * @param systemReportedReady true if the system is ready or + * false if the benchmark is ready * @return true if system and benchmark are ready */ public synchronized boolean setReadyAndCheck(boolean systemReportedReady) { @@ -371,8 +358,7 @@ public Model getResultModel() { * This method is thread-safe. *

    * - * @param resultModel - * the new result model + * @param resultModel the new result model */ public void setOrMergeResultModel(Model resultModel) { try { @@ -400,8 +386,7 @@ public void setOrMergeResultModel(Model resultModel) { * This method is thread-safe. *

    * - * @param resultModel - * the new result model + * @param resultModel the new result model */ public void setResultModel(Model resultModel) { try { @@ -426,8 +411,7 @@ public void setResultModel(Model resultModel) { * This method is thread-safe. *

    * - * @param error - * the error that should be added to the result model + * @param error the error that should be added to the result model */ public void addErrorIfNonPresent(Resource error) { try { @@ -457,8 +441,7 @@ public void addErrorIfNonPresent(Resource error) { * This method is thread-safe. *

    * - * @param error - * the error that should be added to the result model + * @param error the error that should be added to the result model */ public void addError(Resource error) { try { @@ -484,8 +467,7 @@ public void addError(Resource error) { * This method is not thread-safe. *

    * - * @param error - * the error that should be added to the result model + * @param error the error that should be added to the result model */ private void addError_Unsecured(Resource error) { if (this.resultModel == null) { @@ -530,8 +512,7 @@ private void addBasicInformation_Unsecured() { if (config.serializedBenchParams != null) { try { Model benchmarkParamModel = RabbitMQUtils.readModel(config.serializedBenchParams); - StmtIterator iterator = benchmarkParamModel.listStatements( - HobbitExperiments.New, null, (RDFNode) null); + StmtIterator iterator = benchmarkParamModel.listStatements(HobbitExperiments.New, null, (RDFNode) null); Statement statement; while (iterator.hasNext()) { statement = iterator.next(); @@ -555,15 +536,14 @@ private void addBasicInformation_Unsecured() { * Uses the given {@link ImageManager} instance to add additional meta data * regarding the benchmark and the system to the experiment result model. * - * @param imageManager - * used to get RDF models for the benchmark and the system of this - * experiment - * @param endTimeStamp - * point in time at which the experiment ended - * @param hardwareInformation - * hardware information on which experiment was carried out - */ - public void addMetaDataToResult(ImageManager imageManager, long endTimeStamp, SetupHardwareInformation hardwareInformation) { + * @param imageManager used to get RDF models for the benchmark and the + * system of this experiment + * @param endTimeStamp point in time at which the experiment ended + * @param hardwareInformation hardware information on which experiment was + * carried out + */ + public void addMetaDataToResult(ImageManager imageManager, long endTimeStamp, + SetupHardwareInformation hardwareInformation) { try { modelMutex.acquire(); } catch (InterruptedException e) { @@ -594,11 +574,13 @@ public void addMetaDataToResult(ImageManager imageManager, long endTimeStamp, Se // Add end date Calendar endDate = Calendar.getInstance(); endDate.setTimeInMillis(endTimeStamp); - resultModel.add(resultModel.getResource(experimentUri), HOBBIT.endTime, resultModel.createTypedLiteral(endDate)); + resultModel.add(resultModel.getResource(experimentUri), HOBBIT.endTime, + resultModel.createTypedLiteral(endDate)); // Add hardware information if (hardwareInformation != null) { - resultModel.add(resultModel.getResource(experimentUri), HOBBIT.wasCarriedOutOn, hardwareInformation.addToModel(resultModel)); + resultModel.add(resultModel.getResource(experimentUri), HOBBIT.wasCarriedOutOn, + hardwareInformation.addToModel(resultModel)); } // Remove statements that shouldn't be part of the result model. @@ -613,6 +595,16 @@ public void addMetaDataToResult(ImageManager imageManager, long endTimeStamp, Se } } + /** + * Returns the next error report ID for this experiment. Note that each call of + * this method increases the error ID counter internally. + * + * @return the next error report ID for this experiment + */ + public int getNextErrorReportId() { + return nextErrorReportId.getAndIncrement(); + } + @Override public void close() throws IOException { if (abortTimer != null) { diff --git a/platform-controller/src/main/java/org/hobbit/controller/utils/RabbitMQConnector.java b/platform-controller/src/main/java/org/hobbit/controller/utils/RabbitMQConnector.java index 542421f5..78f3b849 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/utils/RabbitMQConnector.java +++ b/platform-controller/src/main/java/org/hobbit/controller/utils/RabbitMQConnector.java @@ -16,11 +16,11 @@ */ package org.hobbit.controller.utils; -import org.hobbit.controller.PlatformController; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; -import org.apache.commons.io.Charsets; +import org.hobbit.controller.PlatformController; import org.hobbit.core.components.AbstractCommandReceivingComponent; import com.rabbitmq.client.AMQP; @@ -60,7 +60,7 @@ protected void handleCmd(byte bytes[], AMQP.BasicProperties props) { int idLength = buffer.getInt(); byte sessionIdBytes[] = new byte[idLength]; buffer.get(sessionIdBytes); - String sessionId = new String(sessionIdBytes, Charsets.UTF_8); + String sessionId = new String(sessionIdBytes, StandardCharsets.UTF_8); byte command = buffer.get(); byte remainingData[]; if (buffer.remaining() > 0) { @@ -78,7 +78,7 @@ public void basicPublish(String exchange, String routingKey, BasicProperties pro @Override public String toString() { - return String.format("{rabbitMQHostName=%s}", this.rabbitMQHostName); + return String.format("RabbitMQConnector{rabbitMQHostName=%s}", this.rabbitMQHostName); } ///// There are some methods that shouldn't be used by the controller and From 1c5c314ab45772079eb8af0c4891eed105575e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Sun, 17 Dec 2023 16:24:41 +0100 Subject: [PATCH 34/52] Removed the errorDetails property since it is not used anymore. --- .../src/main/java/org/hobbit/controller/ExperimentManager.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java index 4416b348..4e795edf 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java @@ -843,9 +843,6 @@ public void handleErrorReport(String sessionId, ErrorData errorData, boolean isB if (errorData.getDescription() != null) { resultModel.add(error, RDFS.comment, errorData.getDescription()); } - if (errorData.getDetails() != null) { - resultModel.add(error, Algorithm.errorDetails, errorData.getDetails()); - } synchronized (experimentMutex) { // Check again whether we are still working on the same experiment if (isExpRunning(sessionId)) { From 25ab24dc7bb1bfb3b5adce1858798cb7f2035666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Sun, 17 Dec 2023 16:44:56 +0100 Subject: [PATCH 35/52] Prepared release of 2.0.17. --- analysis-component/pom.xml | 2 +- hobbit-gui/gui-serverbackend/pom.xml | 2 +- parent-pom/pom.xml | 4 ++-- platform-controller/pom.xml | 2 +- platform-storage/storage-service/pom.xml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/analysis-component/pom.xml b/analysis-component/pom.xml index cb0cd257..0d1b559b 100644 --- a/analysis-component/pom.xml +++ b/analysis-component/pom.xml @@ -14,7 +14,7 @@ org.hobbit parent - 2.0.17-SNAPSHOT + 2.0.17 ../parent-pom analysis-component diff --git a/hobbit-gui/gui-serverbackend/pom.xml b/hobbit-gui/gui-serverbackend/pom.xml index 1a703f86..4b0ac327 100644 --- a/hobbit-gui/gui-serverbackend/pom.xml +++ b/hobbit-gui/gui-serverbackend/pom.xml @@ -14,7 +14,7 @@ org.hobbit parent - 2.0.17-SNAPSHOT + 2.0.17 ../../parent-pom gui-serverbackend diff --git a/parent-pom/pom.xml b/parent-pom/pom.xml index 35902a5c..03de3256 100644 --- a/parent-pom/pom.xml +++ b/parent-pom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.hobbit parent - 2.0.17-SNAPSHOT + 2.0.17 pom @@ -67,7 +67,7 @@ org.hobbit core - 1.0.22-SNAPSHOT + 1.0.22 diff --git a/platform-controller/pom.xml b/platform-controller/pom.xml index bf303218..f3ee6fbd 100644 --- a/platform-controller/pom.xml +++ b/platform-controller/pom.xml @@ -22,7 +22,7 @@ org.hobbit parent - 2.0.17-SNAPSHOT + 2.0.17 ../parent-pom platform-controller diff --git a/platform-storage/storage-service/pom.xml b/platform-storage/storage-service/pom.xml index 14b8c3bf..ef5daaa1 100644 --- a/platform-storage/storage-service/pom.xml +++ b/platform-storage/storage-service/pom.xml @@ -23,7 +23,7 @@ org.hobbit parent - 2.0.17-SNAPSHOT + 2.0.17 ../../parent-pom storage-service From f173e7fb0123af5f857eb0f8c4eb59957885657f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Mon, 18 Dec 2023 11:46:42 +0100 Subject: [PATCH 36/52] Removed faulty file. --- .../main/java/org/hobbit/controller/Temp.java | 68 ------------------- 1 file changed, 68 deletions(-) delete mode 100644 platform-controller/src/main/java/org/hobbit/controller/Temp.java diff --git a/platform-controller/src/main/java/org/hobbit/controller/Temp.java b/platform-controller/src/main/java/org/hobbit/controller/Temp.java deleted file mode 100644 index d736945d..00000000 --- a/platform-controller/src/main/java/org/hobbit/controller/Temp.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.hobbit.controller; - -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.List; - -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Property; -import org.apache.jena.rdf.model.RDFNode; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.vocabulary.RDF; -import org.hobbit.utils.rdf.RdfHelper; -import org.hobbit.vocab.HOBBIT; -import org.hobbit.vocab.HobbitExperiments; - -public class Temp { - - public static void main(String[] args) { - Model model = ModelFactory.createDefaultModel(); - try (InputStream in = new FileInputStream("a2kb-results.ttl")) { - model.read(in, "", "TTL"); - } catch (Exception e) { - e.printStackTrace(); - return; - } - Property dataset = model.getProperty("http://w3id.org/gerbil/hobbit/vocab#hasDataset"); - Property micPre = model.getProperty("http://w3id.org/gerbil/vocab#microPrecision"); - Property micRec = model.getProperty("http://w3id.org/gerbil/vocab#microRecall"); - Property micF1 = model.getProperty("http://w3id.org/gerbil/vocab#microF1"); - Property macPre = model.getProperty("http://w3id.org/gerbil/vocab#macroPrecision"); - Property macRec = model.getProperty("http://w3id.org/gerbil/vocab#macroRecall"); - Property macF1 = model.getProperty("http://w3id.org/gerbil/vocab#macroF1"); - Property errors = model.getProperty("http://w3id.org/gerbil/vocab#errorCount"); - Property avgMillis = model.getProperty("http://w3id.org/gerbil/vocab#avgMillisPerDoc"); - System.out.println("\"ID\",\"system\",\"dataset\",\"mic. P\",\"mic. R\",\"mic. F1\",\"mac. P\", \"mac. R\", \"mac. F1\",\"errors\",\"avg millis/doc\""); - List experiments = RdfHelper.getSubjectResources(model, RDF.type, HOBBIT.Experiment); - for (Resource experiment : experiments) { - System.out.print('"'); - System.out.print(HobbitExperiments.getExperimentId(experiment)); - System.out.print("\",\""); - System.out.print(RdfHelper.getLabel(model, RdfHelper.getObjectResource(model, experiment, HOBBIT.involvesSystemInstance))); - System.out.print("\",\""); - System.out.print(RdfHelper.getObjectResource(model, experiment, dataset).getLocalName()); - System.out.print("\",\""); - if (model.contains(experiment, HOBBIT.terminatedWithError, (RDFNode) null)) { - System.out.println("\""); - } else { - System.out.print(RdfHelper.getStringValue(model, experiment, micPre)); - System.out.print("\",\""); - System.out.print(RdfHelper.getStringValue(model, experiment, micRec)); - System.out.print("\",\""); - System.out.print(RdfHelper.getStringValue(model, experiment, micF1)); - System.out.print("\",\""); - System.out.print(RdfHelper.getStringValue(model, experiment, macPre)); - System.out.print("\",\""); - System.out.print(RdfHelper.getStringValue(model, experiment, macRec)); - System.out.print("\",\""); - System.out.print(RdfHelper.getStringValue(model, experiment, macF1)); - System.out.print("\",\""); - System.out.print(RdfHelper.getStringValue(model, experiment, errors)); - System.out.print("\",\""); - System.out.print(RdfHelper.getStringValue(model, experiment, avgMillis)); - System.out.println("\""); - } - } - } -} From ff5d0c1dc5262e11142e24da678a3cf913f3b0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Mon, 18 Dec 2023 11:50:14 +0100 Subject: [PATCH 37/52] Applied changes according to review. --- config/db/storage-init.sh | 3 +++ docker-compose-dev.yml | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/config/db/storage-init.sh b/config/db/storage-init.sh index 5d30c028..e73bbbcd 100644 --- a/config/db/storage-init.sh +++ b/config/db/storage-init.sh @@ -1,6 +1,9 @@ # Initialization script for the Storage of the HOBBIT Platform. # Controls access rights for Platform-specific RDF graphs. +echo "Waiting for port to open..." +while ! nc -q 1 localhost 1111 Date: Wed, 3 Jan 2024 13:03:04 +0100 Subject: [PATCH 38/52] cAdvisor image moved to another registry --- docker-compose-dev.yml | 2 +- platform-controller/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0591c640..63b254f2 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -119,7 +119,7 @@ services: - hobbit-core cadvisor: - image: google/cadvisor + image: gcr.io/cadvisor/cadvisor networks: - hobbit-core volumes: diff --git a/platform-controller/Makefile b/platform-controller/Makefile index 91d8bbf5..96e74f0c 100644 --- a/platform-controller/Makefile +++ b/platform-controller/Makefile @@ -48,7 +48,7 @@ cAdvisor: -v /sys:/sys:ro \ -v /var/lib/docker/:/var/lib/docker:ro \ -v /dev/disk:/dev/disk:ro \ - google/cadvisor + gcr.io/cadvisor/cadvisor trigger-all-correlation-analysis: docker run --network hobbit-core --volume $(CURDIR):/data:ro -e "HOBBIT_RABBIT_HOST=rabbit" java:alpine /data/test_trigger_all_correlation_analysis.sh From 3428f5be9c492a34954a921a5c88ce6d029c4c14 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Thu, 4 Jan 2024 14:08:04 +0100 Subject: [PATCH 39/52] Run CI workflow with ubuntu-22.04 --- .github/workflows/make.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index c6a312e6..71ef0059 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -2,7 +2,8 @@ name: make on: push jobs: make: - runs-on: ubuntu-20.04 + # https://docs.github.com/en/actions/using-jobs/choosing-the-runner-for-a-job + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v1 From 1a0a839b245d88989008560e1d313f9955323776 Mon Sep 17 00:00:00 2001 From: Denis Kuchelev Date: Thu, 4 Jan 2024 14:21:39 +0100 Subject: [PATCH 40/52] Configure networks as in new docker compose documentation https://docs.docker.com/compose/compose-file/06-networks/ --- docker-compose-dev.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 63b254f2..61a26556 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -144,8 +144,6 @@ services: networks: hobbit: - external: - name: hobbit + external: true hobbit-core: - external: - name: hobbit-core + external: true From 5162fc4c40556da41cb1b8e434f8f1e38e741847 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 5 Jan 2024 11:45:03 +0100 Subject: [PATCH 41/52] Fixed execution permissions for shell scripts. --- migrate-kibana-index.sh | 0 platform-controller/test_cmd.sh | 0 platform-controller/test_request_benchmark.sh | 0 platform-controller/test_request_benchmark_details.sh | 0 platform-controller/test_request_controller_status.sh | 0 platform-controller/test_request_system_resource_usage.sh | 0 platform-controller/test_start_benchmark.sh | 0 platform-controller/test_trigger_all_correlation_analysis.sh | 0 rabbitmq-cluster/cluster-entrypoint.sh | 0 update-logstash-index.sh | 0 10 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 migrate-kibana-index.sh mode change 100644 => 100755 platform-controller/test_cmd.sh mode change 100644 => 100755 platform-controller/test_request_benchmark.sh mode change 100644 => 100755 platform-controller/test_request_benchmark_details.sh mode change 100644 => 100755 platform-controller/test_request_controller_status.sh mode change 100644 => 100755 platform-controller/test_request_system_resource_usage.sh mode change 100644 => 100755 platform-controller/test_start_benchmark.sh mode change 100644 => 100755 platform-controller/test_trigger_all_correlation_analysis.sh mode change 100644 => 100755 rabbitmq-cluster/cluster-entrypoint.sh mode change 100644 => 100755 update-logstash-index.sh diff --git a/migrate-kibana-index.sh b/migrate-kibana-index.sh old mode 100644 new mode 100755 diff --git a/platform-controller/test_cmd.sh b/platform-controller/test_cmd.sh old mode 100644 new mode 100755 diff --git a/platform-controller/test_request_benchmark.sh b/platform-controller/test_request_benchmark.sh old mode 100644 new mode 100755 diff --git a/platform-controller/test_request_benchmark_details.sh b/platform-controller/test_request_benchmark_details.sh old mode 100644 new mode 100755 diff --git a/platform-controller/test_request_controller_status.sh b/platform-controller/test_request_controller_status.sh old mode 100644 new mode 100755 diff --git a/platform-controller/test_request_system_resource_usage.sh b/platform-controller/test_request_system_resource_usage.sh old mode 100644 new mode 100755 diff --git a/platform-controller/test_start_benchmark.sh b/platform-controller/test_start_benchmark.sh old mode 100644 new mode 100755 diff --git a/platform-controller/test_trigger_all_correlation_analysis.sh b/platform-controller/test_trigger_all_correlation_analysis.sh old mode 100644 new mode 100755 diff --git a/rabbitmq-cluster/cluster-entrypoint.sh b/rabbitmq-cluster/cluster-entrypoint.sh old mode 100644 new mode 100755 diff --git a/update-logstash-index.sh b/update-logstash-index.sh old mode 100644 new mode 100755 From 8ee941602ba68364b713fdd62660b5586b22e7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 5 Jan 2024 12:51:50 +0100 Subject: [PATCH 42/52] Fixed badge in README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 95a18b7b..2ea820b3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/hobbit-project/platform.svg?branch=master)](https://travis-ci.org/hobbit-project/platform) +[![make](https://github.com/hobbit-project/platform/actions/workflows/make.yml/badge.svg?branch=master)](https://github.com/hobbit-project/platform/actions/workflows/make.yml) # HOBBIT platform From 7188ad371aae470bbfe7f23a74772e16bb9cc200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 5 Jan 2024 12:53:04 +0100 Subject: [PATCH 43/52] Added a utilities class for connectivity assumptions. Based the GitlabControllerTest on the assumption that the Hobbit gitlab instance is available. --- .../ConnectivityAssumptionUtils.java | 34 +++++++++++++++++++ .../gitlab/GitlabControllerImplTest.java | 4 +++ 2 files changed, 38 insertions(+) create mode 100644 platform-controller/src/test/java/org/hobbit/controller/ConnectivityAssumptionUtils.java diff --git a/platform-controller/src/test/java/org/hobbit/controller/ConnectivityAssumptionUtils.java b/platform-controller/src/test/java/org/hobbit/controller/ConnectivityAssumptionUtils.java new file mode 100644 index 00000000..e406b01d --- /dev/null +++ b/platform-controller/src/test/java/org/hobbit/controller/ConnectivityAssumptionUtils.java @@ -0,0 +1,34 @@ +package org.hobbit.controller; + +import java.net.HttpURLConnection; +import java.net.URL; + +import org.junit.Assume; + +/** + * A simple class that offers utility methods to assume the connectivity within + * JUnit tests. + * + * @author Michael Röder (michael.roeder@uni-paderborn.de) + * + */ +public class ConnectivityAssumptionUtils { + + public static void assumeConnectivity(String httpUrl) { + try { + URL pingUrl = new URL(httpUrl); + HttpURLConnection connection = (HttpURLConnection) pingUrl.openConnection(); + + Assume.assumeTrue( + "Got a wrong status (" + connection.getResponseCode() + + ") code while checking the connectivity to \"" + httpUrl + + "\". I will assume that I cannot connect to this endpoint.", + connection.getResponseCode() < 400); + } catch (Exception e) { + Assume.assumeNoException( + "Exception while checking connectivity to \"" + httpUrl + + "\". I will assume that I cannot connect to this endpoint. Exception: " + e.getMessage(), + e); + } + } +} diff --git a/platform-controller/src/test/java/org/hobbit/controller/gitlab/GitlabControllerImplTest.java b/platform-controller/src/test/java/org/hobbit/controller/gitlab/GitlabControllerImplTest.java index a3bb73ae..fdd8e34d 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/gitlab/GitlabControllerImplTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/gitlab/GitlabControllerImplTest.java @@ -28,6 +28,8 @@ import org.apache.jena.rdf.model.impl.ResourceImpl; import org.gitlab.api.GitlabAPI; import org.gitlab.api.models.*; +import org.hobbit.controller.ConnectivityAssumptionUtils; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -37,6 +39,7 @@ /** * Created by Timofey Ermilov on 17/10/2016. + * Updated by Michael Röder to run only if the test instance is available- */ public class GitlabControllerImplTest { private static final String GITLAB_URL = "https://git.project-hobbit.eu/"; @@ -47,6 +50,7 @@ public class GitlabControllerImplTest { @Before public void init() throws InterruptedException { + ConnectivityAssumptionUtils.assumeConnectivity(GITLAB_URL); gitlabProject = new GitlabProject(); gitlabProject.setId(526); gitlabProject.setName("testing-benchmark"); From 79b563c8d88955e7785504749e7f3a91aae7e206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 5 Jan 2024 13:52:48 +0100 Subject: [PATCH 44/52] Added timeouts to Gitlab-based tests. Added an additional assumption to the image manager test. --- .../controller/docker/GitlabBasedImageManagerTest.java | 5 ++++- .../controller/gitlab/GitlabControllerImplTest.java | 10 +++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/platform-controller/src/test/java/org/hobbit/controller/docker/GitlabBasedImageManagerTest.java b/platform-controller/src/test/java/org/hobbit/controller/docker/GitlabBasedImageManagerTest.java index 0253cadd..ee935ea9 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/docker/GitlabBasedImageManagerTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/docker/GitlabBasedImageManagerTest.java @@ -25,6 +25,8 @@ import org.apache.jena.rdf.model.Model; import org.apache.jena.vocabulary.RDF; +import org.hobbit.controller.ConnectivityAssumptionUtils; +import org.hobbit.controller.gitlab.GitlabControllerImplTest; import org.hobbit.core.data.BenchmarkMetaData; import org.hobbit.core.data.SystemMetaData; import org.hobbit.core.rabbit.RabbitMQUtils; @@ -45,6 +47,7 @@ public class GitlabBasedImageManagerTest { @Before public void initObserver() { + ConnectivityAssumptionUtils.assumeConnectivity(GitlabControllerImplTest.GITLAB_URL); imageManager = new GitlabBasedImageManager(); } @@ -59,7 +62,7 @@ public void initObserver() { // "http://example.org/GerbilBenchmark"); // } - @Test + @Test (timeout = 120000) public void getBenchmarks() throws Exception { // use future to make test wait for async stuff (sigh, java) diff --git a/platform-controller/src/test/java/org/hobbit/controller/gitlab/GitlabControllerImplTest.java b/platform-controller/src/test/java/org/hobbit/controller/gitlab/GitlabControllerImplTest.java index fdd8e34d..0dbf9a1a 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/gitlab/GitlabControllerImplTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/gitlab/GitlabControllerImplTest.java @@ -42,7 +42,7 @@ * Updated by Michael Röder to run only if the test instance is available- */ public class GitlabControllerImplTest { - private static final String GITLAB_URL = "https://git.project-hobbit.eu/"; + public static final String GITLAB_URL = "https://git.project-hobbit.eu/"; GitlabControllerImpl controller; GitlabProject gitlabProject; GitlabBranch gitlabBranch; @@ -97,7 +97,7 @@ public void init() throws InterruptedException { // wait for controller to fetch projects } - @Test + @Test (timeout = 120000) public void getAllProjects() { controller.fetchProjects(); List projects = controller.getAllProjects(); @@ -106,7 +106,7 @@ public void getAllProjects() { assertTrue("There are more than 10 projects", projects.size() > 10); } - @Test + @Test (timeout = 120000) public void getProjectsOfUnknownUser() throws IOException { Set projects = controller.getProjectsOfUser("nonexisting@example.com"); assertEquals("Empty project list for unknown user", 0, projects.size()); @@ -123,7 +123,7 @@ public void getProjectsOfRegularUser() throws IOException { } */ - @Test + @Test (timeout = 120000) public void gitlabToProject() { Project project = controller.gitlabToProject(gitlabProject); assertNotNull("Project shouldn't be null", project); @@ -131,7 +131,7 @@ public void gitlabToProject() { "gitadmin / testing-benchmark", project.getName()); } - @Test + @Test (timeout = 120000) public void getCheckedModel() throws IOException { byte[] benchmarkCfgBytes = api.getRawFileContent(gitlabProject.getId(), gitlabBranch.getCommit().getId(), "benchmark.ttl"); Model checkedModel = controller.getCheckedModel(benchmarkCfgBytes, "benchmark", gitlabProject.getWebUrl()); From ea0da3a63c285e6f21ae6930d083881be923dd9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 5 Jan 2024 13:52:59 +0100 Subject: [PATCH 45/52] Fixed a comment. --- platform-controller/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform-controller/Makefile b/platform-controller/Makefile index 96e74f0c..4a32db0d 100644 --- a/platform-controller/Makefile +++ b/platform-controller/Makefile @@ -16,7 +16,7 @@ run: org.hobbit.controller.PlatformController test: - # Without HOBBIT_RABBIT_EXPERIMENTS_HOST we will be not be able to connect to the RabbitMQ platform will create. + # Without HOBBIT_RABBIT_EXPERIMENTS_HOST we won't be able to connect to RabbitMQ docker-compose --file=../docker-compose-dev.yml up -d cadvisor node-exporter prometheus rabbit redis HOBBIT_RABBIT_IMAGE=rabbitmq:management \ HOBBIT_RABBIT_HOST=localhost \ From 223f67150afbd4b32cbde6ff512ff337e5e869f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 5 Jan 2024 13:53:31 +0100 Subject: [PATCH 46/52] Removed an assertion that made the closing of an experiment fail although it wasn't necessary. --- .../java/org/hobbit/controller/PlatformController.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java index ea1ffde6..419dbb1b 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java +++ b/platform-controller/src/main/java/org/hobbit/controller/PlatformController.java @@ -340,9 +340,12 @@ public void setExpRabbitMQConnector(RabbitMQConnector rabbitMQConnector) { */ public void closeExpRabbitMQConnector() { LOGGER.info("Closing experiment's RabbitMQ connector for the command queue: {}", rabbitMQConnector); - assert rabbitMQConnector != null : "RabbitMQ connector shouldn't be null"; - IOUtils.closeQuietly(rabbitMQConnector); - rabbitMQConnector = null; + if(rabbitMQConnector != null) { + IOUtils.closeQuietly(rabbitMQConnector); + rabbitMQConnector = null; + } else { + LOGGER.warn("Got a request to close the RabbitMQ connector but it was already null."); + } } /** From 96ad3c8667a6c06646f132a07b66e6e9c31f03a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 5 Jan 2024 14:15:09 +0100 Subject: [PATCH 47/52] Attempt to fix the problem that RabbitMQ is not started properly in Github actions. --- docker-compose-dev.yml | 4 ++++ platform-controller/Makefile | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 61a26556..888d1661 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -144,6 +144,10 @@ services: networks: hobbit: + name: hobbit external: true + driver: overlay hobbit-core: + name: hobbit-core external: true + driver: overlay diff --git a/platform-controller/Makefile b/platform-controller/Makefile index 4a32db0d..486875fb 100644 --- a/platform-controller/Makefile +++ b/platform-controller/Makefile @@ -17,7 +17,7 @@ run: test: # Without HOBBIT_RABBIT_EXPERIMENTS_HOST we won't be able to connect to RabbitMQ - docker-compose --file=../docker-compose-dev.yml up -d cadvisor node-exporter prometheus rabbit redis + docker compose --file=../docker-compose-dev.yml up -d cadvisor node-exporter prometheus rabbit redis HOBBIT_RABBIT_IMAGE=rabbitmq:management \ HOBBIT_RABBIT_HOST=localhost \ HOBBIT_RABBIT_EXPERIMENTS_HOST=localhost \ From 4199dd0cc39ff58eca9555fabee7c748970141a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 5 Jan 2024 16:33:16 +0100 Subject: [PATCH 48/52] Added the usage of the HobbitConfiguraiton class instead of accessing system properties. --- .../main/java/org/hobbit/controller/ExperimentManager.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java index 4e795edf..79dfba01 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java @@ -143,10 +143,7 @@ protected ExperimentManager(PlatformController controller, HobbitConfiguration h this.hobbitConfig = hobbitConfig; try { - // TODO environment variable should have been used there - // TODO global static method in hobbit core for retrieving values like this - defaultMaxExecutionTime = Long - .parseLong(System.getProperty("MAX_EXECUTION_TIME", Long.toString(DEFAULT_MAX_EXECUTION_TIME))); + defaultMaxExecutionTime = hobbitConfig.getLong("MAX_EXECUTION_TIME", 1200000L, LOGGER); } catch (Exception e) { LOGGER.debug("Could not get execution time from env, using default value.."); } @@ -329,7 +326,7 @@ protected static Map getHardwareConstraints(String serializedBen return Collections.emptyMap(); } - private void createRabbitMQ(ExperimentConfiguration config) throws Exception { + protected void createRabbitMQ(ExperimentConfiguration config) throws Exception { String rabbitMQAddress = hobbitConfig.getString(RABBIT_MQ_EXPERIMENTS_HOST_NAME_KEY, (String) null); if (rabbitMQAddress == null) { LOGGER.info("Starting new RabbitMQ for the experiment..."); From daa699c549052269a11f3c065e2257d418d2557b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 5 Jan 2024 16:33:31 +0100 Subject: [PATCH 49/52] Fixed the ExperimentTimeoutTest class. --- .../controller/ExperimentTimeoutTest.java | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/platform-controller/src/test/java/org/hobbit/controller/ExperimentTimeoutTest.java b/platform-controller/src/test/java/org/hobbit/controller/ExperimentTimeoutTest.java index 215ea534..d819eaf6 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/ExperimentTimeoutTest.java +++ b/platform-controller/src/test/java/org/hobbit/controller/ExperimentTimeoutTest.java @@ -1,13 +1,19 @@ package org.hobbit.controller; +import java.util.HashMap; import java.util.concurrent.Semaphore; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.EnvironmentConfiguration; +import org.apache.commons.configuration2.MapConfiguration; import org.apache.jena.rdf.model.Model; import org.hobbit.controller.data.ExperimentConfiguration; import org.hobbit.controller.mocks.DummyImageManager; import org.hobbit.controller.mocks.DummyPlatformController; import org.hobbit.controller.mocks.DummyStorageServiceClient; +import org.hobbit.controller.utils.RabbitMQConnector; +import org.hobbit.core.Constants; import org.hobbit.core.data.status.ControllerStatus; import org.hobbit.utils.config.HobbitConfiguration; import org.hobbit.vocab.HOBBIT; @@ -37,15 +43,31 @@ public class ExperimentTimeoutTest { @Before public void init() { // set max execution time to 1s - System.setProperty("MAX_EXECUTION_TIME", "1000"); - controller = new DummyPlatformController(benchmarkControllerTerminated); - controller.queue.add(new ExperimentConfiguration(EXPERIMENT_ID, DummyImageManager.BENCHMARK_NAME, "{}", DummyImageManager.SYSTEM_URI)); + Configuration config = new MapConfiguration(new HashMap<>()); + config.addProperty("MAX_EXECUTION_TIME", "1000"); + config.addProperty("HOBBIT_RABBIT_IMAGE", "rabbitmq:management"); HobbitConfiguration configuration = new HobbitConfiguration(); - manager = new ExperimentManager(controller, configuration, 1000, 1000); + configuration.addConfiguration(config); + configuration.addConfiguration(new EnvironmentConfiguration()); + + controller = new DummyPlatformController(benchmarkControllerTerminated); + controller.queue.add(new ExperimentConfiguration(EXPERIMENT_ID, DummyImageManager.BENCHMARK_NAME, "{}", + DummyImageManager.SYSTEM_URI)); + manager = new ExperimentManager(controller, configuration, 1000, 1000) { + // We have to override the creation of the RabbitMQ connector to the + // experiment's RabbitMQ broker. Instead, we connect to the already running + // RabbitMQ. + protected void createRabbitMQ(ExperimentConfiguration config) throws Exception { + RabbitMQConnector rabbitMQConnector = new RabbitMQConnector(controller, + this.hobbitConfig.getString(Constants.RABBIT_MQ_HOST_NAME_KEY)); + controller.setExpRabbitMQConnector(rabbitMQConnector); + rabbitMQConnector.init(); + }; + }; controller.expManager = manager; } - @Test (timeout = 10000) + @Test(timeout = 20000) public void test() throws Exception { benchmarkControllerTerminated.acquire(); // Give the system some time to tidy up @@ -58,8 +80,8 @@ public void test() throws Exception { Assert.assertNull("Status of the running experiment", status.experiment); Model resultModel = ((DummyStorageServiceClient) controller.storage).insertedModel; Assert.assertTrue("Result model contains the error information about the failed experiment.", - resultModel.contains(HobbitExperiments.getExperiment(EXPERIMENT_ID), - HOBBIT.terminatedWithError, HobbitErrors.ExperimentTookTooMuchTime)); + resultModel.contains(HobbitExperiments.getExperiment(EXPERIMENT_ID), HOBBIT.terminatedWithError, + HobbitErrors.ExperimentTookTooMuchTime)); } @After From 40cf2201e61b1e98c2416dfc4b2037cfd0df23c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Thu, 18 Jan 2024 17:33:50 +0100 Subject: [PATCH 50/52] Added timout for connectivity assumption class. --- .../java/org/hobbit/controller/ConnectivityAssumptionUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platform-controller/src/test/java/org/hobbit/controller/ConnectivityAssumptionUtils.java b/platform-controller/src/test/java/org/hobbit/controller/ConnectivityAssumptionUtils.java index e406b01d..87175120 100644 --- a/platform-controller/src/test/java/org/hobbit/controller/ConnectivityAssumptionUtils.java +++ b/platform-controller/src/test/java/org/hobbit/controller/ConnectivityAssumptionUtils.java @@ -18,6 +18,8 @@ public static void assumeConnectivity(String httpUrl) { try { URL pingUrl = new URL(httpUrl); HttpURLConnection connection = (HttpURLConnection) pingUrl.openConnection(); + // The service has 10 seconds to establish a connection + connection.setConnectTimeout(10000); Assume.assumeTrue( "Got a wrong status (" + connection.getResponseCode() From b8e433ae53adf1bff275b5e19da28f91de1c3c01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Thu, 18 Jan 2024 17:34:15 +0100 Subject: [PATCH 51/52] Added usage of error instance class of the core library instead of using a local namespace. --- .../java/org/hobbit/controller/ExperimentManager.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java index 79dfba01..d6adfe52 100644 --- a/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java +++ b/platform-controller/src/main/java/org/hobbit/controller/ExperimentManager.java @@ -59,8 +59,8 @@ import org.hobbit.core.rabbit.RabbitMQUtils; import org.hobbit.utils.config.HobbitConfiguration; import org.hobbit.utils.rdf.RdfHelper; -import org.hobbit.vocab.Algorithm; import org.hobbit.vocab.HOBBIT; +import org.hobbit.vocab.HobbitErrorInstances; import org.hobbit.vocab.HobbitErrors; import org.hobbit.vocab.HobbitExperiments; import org.hobbit.vocab.PROV; @@ -102,10 +102,6 @@ public class ExperimentManager implements Closeable { * to start. */ public static final long CHECK_FOR_NEW_EXPERIMENT = 10000; - /** - * IRI name space used for the generation of IRIs for error instances. - */ - public static final String ERROR_INSTANCE_NAMESPACE = "http://project-hobbit.org/error-instances/"; /** * Default time an experiment has to terminate after it has been started. */ @@ -816,8 +812,8 @@ public void handleErrorReport(String sessionId, ErrorData errorData, boolean isB // Transform the error data into RDF Model resultModel = ModelFactory.createDefaultModel(); // Generate IRI for this error - Resource error = ResourceFactory.createResource(new StringBuilder(ERROR_INSTANCE_NAMESPACE).append(sessionId) - .append('_').append(experimentStatus.getNextErrorReportId()).toString()); + Resource error = HobbitErrorInstances.getErrorInstance( + new StringBuilder(sessionId).append('_').append(experimentStatus.getNextErrorReportId()).toString()); // Add the error type Resource errorType = (errorData.getErrorType() == null) ? HobbitErrors.UnspecifiedError : ResourceFactory.createResource(errorData.getErrorType()); From 3dbf6713cd186753d947b4ae9a6ea444b9d8a2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=B6der?= Date: Fri, 19 Jan 2024 10:40:02 +0100 Subject: [PATCH 52/52] Added an explicit pull of necessary images to the github action file. --- .github/workflows/make.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index 71ef0059..5e4e53b0 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -14,4 +14,5 @@ jobs: node-version: 8 - run: | docker swarm init + - run: docker compose -f docker-compose-dev.yml pull cadvisor node-exporter prometheus rabbit redis - run: make test