Skip to content

Commit

Permalink
Add support to handle many connectors at once
Browse files Browse the repository at this point in the history
  • Loading branch information
@cmendesce
cmendesce committed Jun 14, 2024
1 parent 877609a commit aa8c8de
Show file tree
Hide file tree
Showing 26 changed files with 752 additions and 613 deletions.
8 changes: 4 additions & 4 deletions ...ench/operator/src/main/java/io/resiliencebench/execution/istio/IstioScenarioExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,9 @@ private Job startLoadGeneration(Scenario scenario, ExecutionQueue executionQueue
return k6LoadGeneratorStep.execute(scenario, executionQueue);
}

private void runScenario(String namespace, String name, ExecutionQueue executionQueue) {
logger.info("Running scenario: {}", name);
var scenario = scenarioRepository.find(namespace, name);
private void runScenario(String namespace, String scenarioName, ExecutionQueue executionQueue) {
logger.info("Running scenario: {}", scenarioName);
var scenario = scenarioRepository.find(namespace, scenarioName);
if (scenario.isPresent()) {
preparationSteps.forEach(step -> step.execute(scenario.get(), executionQueue));
var job = startLoadGeneration(scenario.get(), executionQueue);
Expand All @@ -106,7 +106,7 @@ private void runScenario(String namespace, String name, ExecutionQueue execution
jobsClient.resource(job).watch(this);
logger.info("Job created: {}", job.getMetadata().getName());
} else {
throw new RuntimeException(format("Scenario not found: %s.%s", namespace, name));
throw new RuntimeException(format("Scenario not found: %s.%s", namespace, scenarioName));
}
}

Expand Down
62 changes: 33 additions & 29 deletions ...bench/operator/src/main/java/io/resiliencebench/execution/istio/steps/IstioFaultStep.java
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,22 @@
package io.resiliencebench.execution.istio.steps;

import io.fabric8.istio.api.networking.v1beta1.HTTPFaultInjection;
import io.fabric8.istio.api.networking.v1beta1.VirtualService;
import io.fabric8.istio.client.IstioClient;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.resiliencebench.resources.queue.ExecutionQueue;
import io.resiliencebench.resources.scenario.Scenario;
import io.resiliencebench.resources.scenario.ScenarioFaultTemplate;
import io.resiliencebench.resources.scenario.Target;
import io.resiliencebench.resources.service.ResilientService;
import io.resiliencebench.support.CustomResourceRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import java.util.Optional;

@Service
public class IstioFaultStep extends IstioExecutorStep<VirtualService> {
public class IstioFaultStep extends IstioExecutorStep<Scenario> {

private final static Logger logger = LoggerFactory.getLogger(IstioFaultStep.class);

Expand All @@ -23,39 +25,41 @@ public IstioFaultStep(KubernetesClient kubernetesClient, IstioClient istioClient
}

@Override
public VirtualService execute(Scenario scenario, ExecutionQueue executionQueue) {
var targetService =
findVirtualService(
scenario.getMetadata().getNamespace(),
scenario.getSpec().getTargetServiceName()
);

var fault = configureFault(scenario.getSpec().getFault());
public Scenario execute(Scenario scenario, ExecutionQueue executionQueue) {
for (var connector : scenario.getSpec().getConnectors()) {
var target = connector.getTarget();
configureFaultOnTarget(scenario.getMetadata().getNamespace(), target);
}
return scenario;
}

// TODO Handler error
// TODO check if the virtual service already has a fault. if yes, update it
private void configureFaultOnTarget(String namespace, Target target) {
var fault = createFault(target.getFault());

var editedVirtualService = targetService
.edit()
.editSpec()
.editFirstHttp()
.withFault(fault)
.endHttp()
.endSpec()
.build();
var targetService = findVirtualService(namespace, target.getServiceName());
if (fault.isPresent()) {
var editedVirtualService = targetService
.edit()
.editSpec()
.editFirstHttp()
.withFault(fault.get())
.endHttp()
.endSpec()
.build();

return istioClient()
.v1beta1()
.virtualServices()
.inNamespace(targetService.getMetadata().getNamespace() )
.resource(editedVirtualService)
.update();
istioClient()
.v1beta1()
.virtualServices()
.inNamespace(targetService.getMetadata().getNamespace())
.resource(editedVirtualService)
.update();
}
}

public HTTPFaultInjection configureFault(ScenarioFaultTemplate faultTemplate) {
public Optional<HTTPFaultInjection> createFault(ScenarioFaultTemplate faultTemplate) {
if (faultTemplate == null || (faultTemplate.getAbort() == null && faultTemplate.getDelay() == null)) {
logger.error("Fault template is null. No fault was configured.");
return null;
return Optional.empty();
}

var builder = new HTTPFaultInjection().toBuilder();
Expand All @@ -70,6 +74,6 @@ public HTTPFaultInjection configureFault(ScenarioFaultTemplate faultTemplate) {
.withNewHTTPFaultInjectionAbortHttpStatusErrorType(faultTemplate.getAbort().httpStatus())
.endAbort();
}
return builder.build();
return Optional.of(builder.build());
}
}
40 changes: 17 additions & 23 deletions ...bench/operator/src/main/java/io/resiliencebench/execution/istio/steps/IstioRetryStep.java
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package io.resiliencebench.execution.istio.steps;

import io.fabric8.istio.api.networking.v1beta1.HTTPRetry;
import io.fabric8.istio.api.networking.v1beta1.VirtualService;
import io.fabric8.istio.client.IstioClient;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.resiliencebench.resources.queue.ExecutionQueue;
import io.resiliencebench.resources.scenario.Scenario;
import io.resiliencebench.resources.scenario.Source;
import io.resiliencebench.resources.service.ResilientService;
import io.resiliencebench.support.CustomResourceRepository;
import org.slf4j.Logger;
Expand All @@ -15,7 +15,7 @@
import java.util.Optional;

@Service
public class IstioRetryStep extends IstioExecutorStep<VirtualService> {
public class IstioRetryStep extends IstioExecutorStep<Scenario> {

private final static Logger log = org.slf4j.LoggerFactory.getLogger(IstioRetryStep.class);

Expand All @@ -24,43 +24,37 @@ public IstioRetryStep(KubernetesClient kubernetesClient, IstioClient istioClient
}

@Override
public VirtualService execute(Scenario scenario, ExecutionQueue executionQueue) {

// TODO verify if scenario has a retry pattern configured

var targetService =
findVirtualService(
scenario.getMetadata().getNamespace(),
scenario.getSpec().getSourceServiceName()
);

var retry = configureRetryPattern(scenario.getSpec().getPatternConfig());
public Scenario execute(Scenario scenario, ExecutionQueue executionQueue) {
for (var connector : scenario.getSpec().getConnectors()) {
var source = connector.getSource();
configureRetryOnSource(scenario.getMetadata().getNamespace(), source);
}
return scenario;
}

if (retry.isPresent()) {
var newVirtualService = targetService
private void configureRetryOnSource(String namespace, Source source) {
var sourceVirtualService = findVirtualService(namespace, source.getServiceName());
var retryPolicy = createRetryPolicy(source.getPatternConfig());
if (retryPolicy.isPresent()) {
var newVirtualService = sourceVirtualService
.edit()
.editSpec()
.editFirstHttp()
.withRetries(retry.get())
.withRetries(retryPolicy.get())
.endHttp()
.endSpec()
.build();

istioClient()
.v1beta1()
.virtualServices()
.inNamespace(targetService.getMetadata().getNamespace())
.inNamespace(sourceVirtualService.getMetadata().getNamespace())
.resource(newVirtualService)
.update();
return newVirtualService;
} else {
return targetService;
}
}



public Optional<HTTPRetry> configureRetryPattern(Map<String, Object> patternConfig) {
public Optional<HTTPRetry> createRetryPolicy(Map<String, Object> patternConfig) {
var builder = new HTTPRetry().toBuilder();
var attempts = (Integer) patternConfig.get("attempts");
var perTryTimeout = (Integer) patternConfig.get("perTryTimeout");
Expand Down
2 changes: 1 addition & 1 deletion ...nce-bench/operator/src/main/java/io/resiliencebench/execution/k6/K6LoadGeneratorStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public Job execute(Scenario scenario, ExecutionQueue executionQueue) {

public ObjectMeta createMeta(Scenario scenario, Workload workload) {
return new ObjectMetaBuilder()
.withName(workload.getMetadata().getName() + "-" + UUID.fromString(scenario.getMetadata().getUid()))
.withName(workload.getMetadata().getName() + "-" + scenario.getMetadata().getName())
.withNamespace(workload.getMetadata().getNamespace())
.withLabels(Map.of("app", "k6"))
.addToAnnotations(CREATED_BY, "resiliencebench-operator")
Expand Down
123 changes: 74 additions & 49 deletions resilience-bench/operator/src/main/java/io/resiliencebench/resources/ScenarioFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,12 @@
import io.fabric8.kubernetes.api.model.ObjectMeta;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.resiliencebench.resources.benchmark.Benchmark;
import io.resiliencebench.resources.benchmark.Source;
import io.resiliencebench.resources.benchmark.Target;
import io.resiliencebench.resources.scenario.Scenario;
import io.resiliencebench.resources.scenario.ScenarioFaultTemplate;
import io.resiliencebench.resources.scenario.ScenarioSpec;
import io.resiliencebench.resources.scenario.ScenarioWorkload;
import io.resiliencebench.resources.benchmark.ConnectorTemplate;
import io.resiliencebench.resources.benchmark.SourceTemplate;
import io.resiliencebench.resources.scenario.*;
import io.resiliencebench.resources.workload.Workload;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;

import static io.resiliencebench.support.Annotations.OWNED_BY;

Expand Down Expand Up @@ -59,59 +53,90 @@ public static Object convertJsonNode(JsonNode jsonNode) {
return jsonNode;
}

public static List<Map<String, Object>> expandServiceParameters(Source serviceSource) {
public static List<Map<String, Object>> expandServiceParameters(SourceTemplate serviceSource) {
return ListExpansion.expandConfigTemplate(serviceSource.getPatternConfig());
}

private static List<Connector> expandConnector(ConnectorTemplate connectorTemplate) {
List<Connector> expandedConnectors = new ArrayList<>();
for (var faultPercentage : connectorTemplate.getTarget().getFault().getPercentage()) {
var fault = ScenarioFaultTemplate.create(
faultPercentage,
connectorTemplate.getTarget().getFault().getDelay(),
connectorTemplate.getTarget().getFault().getAbort()
);
for (var sourcePattern : expandServiceParameters(connectorTemplate.getSource())) {
var source = new Source(connectorTemplate.getSource().getService(), sourcePattern);
var target = new Target(connectorTemplate.getTarget().getService(), fault);
expandedConnectors.add(new Connector(connectorTemplate.getName(), source, target));
}
}

return expandedConnectors;
}

public static <T> List<List<T>> generateCombinations(List<List<T>> listOfLists) {
List<List<T>> result = new ArrayList<>();
generateCombinationsRecursive(listOfLists, 0, new ArrayList<>(), result);
return result;
}

private static <T> void generateCombinationsRecursive(List<List<T>> listOfLists, int depth, List<T> currentCombination, List<List<T>> result) {
if (depth == listOfLists.size()) {
result.add(new ArrayList<>(currentCombination));
return;
}

for (T element : listOfLists.get(depth)) {
currentCombination.add(element);
generateCombinationsRecursive(listOfLists, depth + 1, currentCombination, result);
currentCombination.remove(currentCombination.size() - 1);
}
}

public static List<Scenario> create(Benchmark benchmark, Workload workload) {
List<Scenario> scenarios = new ArrayList<>();
var workloadUsers = workload.getSpec().getUsers();
var workloadName = workload.getMetadata().getName();

for (var connection : benchmark.getSpec().getConnections()) {
var target = connection.target();
var source = connection.source();

for (var faultPercentage : target.getFault().getPercentage()) {
var fault =
ScenarioFaultTemplate.create(faultPercentage, target.getFault().getDelay(), target.getFault().getAbort());

for (var sourcePatternsParameters : expandServiceParameters(source)) {
for (var workloadUser : workloadUsers) {
var scenario = createScenario(workloadName, target, source, fault, sourcePatternsParameters, workloadUser);
scenario.setMetadata(createMeta(scenario, benchmark));
scenarios.add(scenario);
}
List<Scenario> executions = new ArrayList<>();

for (var scenarioTemplate : benchmark.getSpec().getScenarios()) {
var expandedConnectors = new ArrayList<List<Connector>>();
for (var connectorTemplate : scenarioTemplate.getConnectors()) {
expandedConnectors.add(expandConnector(connectorTemplate));
}

var expandedConnectorsCombined = new ArrayList<List<Connector>>();
generateCombinationsRecursive(expandedConnectors, 0, new ArrayList<>(), expandedConnectorsCombined);

var workloadUsers = workload.getSpec().getUsers();
var workloadName = workload.getMetadata().getName();

for (var workloadUser : workloadUsers) {
for (int i = 0; i < expandedConnectorsCombined.size(); i++) {
var scenarioName = generateScenarioName(scenarioTemplate.getName(), i+1);
var connectors = expandedConnectorsCombined.get(i);
var spec = new ScenarioSpec(
scenarioName,
new ScenarioWorkload(workloadName, workloadUser),
connectors);
var scenario = new Scenario();
scenario.setSpec(spec);
scenario.setMetadata(createMeta(scenarioName, benchmark));
executions.add(scenario);
}
}
}
return scenarios;

return executions;
}

private static Scenario createScenario(
String workloadName,
Target target,
Source source,
ScenarioFaultTemplate fault,
Map<String, Object> sourcePatternsParameters, Integer workloadUser
) {
return new Scenario(
new ScenarioSpec(
target.getService(),
source.getService(),
sourcePatternsParameters,
new ScenarioWorkload(workloadName, workloadUser),
fault
)
);
private static String generateScenarioName(String scenarioName, int index) {
return scenarioName + "-" + "00000".substring((""+index).length()) + index;
}

private static ObjectMeta createMeta(Scenario scenario, Benchmark benchmark) {
private static ObjectMeta createMeta(String name, Benchmark benchmark) {
return new ObjectMetaBuilder()
.withName(scenario.toString())
.withName(name)
.withNamespace(benchmark.getMetadata().getNamespace())
.addToAnnotations(OWNED_BY, benchmark.getMetadata().getName())
.addToAnnotations("resiliencebench.io/scenario-uid", scenario.toString())
.build();
}
}
Loading

0 comments on commit aa8c8de

Please sign in to comment.