Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/staging/matching-stats-perf' int…
Browse files Browse the repository at this point in the history 
…o staging/matching-stats-perf
  • Loading branch information
@awildturtok
awildturtok committed Dec 4, 2024
2 parents 1441397 + b6345bd commit ef78ded
Show file tree
Hide file tree
Showing 87 changed files with 1,697 additions and 518 deletions.
73 changes: 54 additions & 19 deletions backend/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

Expand All @@ -15,6 +15,8 @@
<apache-poi.version>5.2.2</apache-poi.version>
<apache-arrow.version>16.0.0</apache-arrow.version>
<apache-parquet.version>1.12.3</apache-parquet.version>
<prometheus-metrics.version>1.2.1</prometheus-metrics.version>
<openapi-generator.version>7.9.0</openapi-generator.version>
</properties>

<build>
Expand Down Expand Up @@ -75,6 +77,36 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.openapitools</groupId>
<artifactId>openapi-generator-maven-plugin</artifactId>
<version>${openapi-generator.version}</version>
<executions>
<execution>
<id>openapi-admin</id>
<goals>
<goal>generate</goal>
</goals>
<configuration>
<inputSpec>${project.basedir}/src/main/resources/openapi/admin-openapi.json</inputSpec>
<!-- Don't use jaxrs-jersey here because it does not allow dependency injection -->
<generatorName>jaxrs-spec</generatorName>
<modelsToGenerate>ApiResponse</modelsToGenerate>
<packageName>com.bakdata.conquery</packageName>
<!-- This package name is used by EndpointTestHelper to filter out openapi specs -->
<apiPackage>com.bakdata.conquery.models.api.openapi</apiPackage>
<modelPackage>com.bakdata.conquery.models.config</modelPackage>
<configOptions>
<!-- The implementation of these interfaces actually allows us to use DI (not possible with jaxrs-jersey) -->
<interfaceOnly>true</interfaceOnly>
<sourceFolder>src/main/java</sourceFolder>
<generateSupportingFiles>false</generateSupportingFiles>
<useJakartaEe>true</useJakartaEe>
</configOptions>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

Expand Down Expand Up @@ -123,7 +155,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
<version>2.18.0</version>
</dependency>
<dependency>
<groupId>com.univocity</groupId>
Expand Down Expand Up @@ -193,13 +225,9 @@
<dependency>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-core</artifactId>
<version>1.13.0</version>
<version>2.0.2</version>
<exclusions>
<!-- All these transitive deps are already bundled in shiro core-->
<exclusion>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-cache</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-config-ogdl</artifactId>
Expand All @@ -220,14 +248,6 @@
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-config-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-event</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-lang</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down Expand Up @@ -383,17 +403,32 @@
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>prometheus-metrics-simpleclient-bridge</artifactId>
<version>1.2.1</version>
<version>${prometheus-metrics.version}</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>prometheus-metrics-exporter-servlet-jakarta</artifactId>
<version>1.2.1</version>
<version>${prometheus-metrics.version}</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>prometheus-metrics-core</artifactId>
<version>1.2.1</version>
<version>${prometheus-metrics.version}</version>
</dependency>
<dependency>
<groupId>org.openapitools</groupId>
<artifactId>openapi-generator-maven-plugin</artifactId>
<version>${openapi-generator.version}</version>
</dependency>
<dependency>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-integration-jakarta</artifactId>
<version>2.2.25</version>
</dependency>
<dependency>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-jaxrs2-jakarta</artifactId>
<version>2.2.25</version>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
Expand Down
4 changes: 2 additions & 2 deletions backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ public boolean isSearchDisabled() {
public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) throws IndexCreationException {

final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath());
log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI);
log.trace("Resolved filter template reference url for search '{}': {}", getId(), resolvedURI);

final FrontendValueIndex search = indexService.getIndex(new FrontendValueIndexKey(
resolvedURI,
Expand All @@ -101,7 +101,7 @@ public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) throws Ind
config.getSearchSplitChars()
));

return search;
return search.getDelegate();
}

@Override
Expand Down
65 changes: 29 additions & 36 deletions backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import com.bakdata.conquery.tasks.QueryCleanupTask;
import com.bakdata.conquery.tasks.ReloadMetaStorageTask;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Throwables;
import io.dropwizard.core.setup.Environment;
import io.dropwizard.lifecycle.Managed;
import lombok.Getter;
Expand All @@ -53,12 +52,11 @@ public class ManagerNode implements Managed {
public static final String DEFAULT_NAME = "manager";

private final String name;

private final List<ResourcesProvider> providers = new ArrayList<>();
private Validator validator;
private AdminServlet admin;
private AuthorizationController authController;
private ScheduledExecutorService maintenanceService;
private final List<ResourcesProvider> providers = new ArrayList<>();
@Delegate(excludes = Managed.class)
private Manager manager;

Expand Down Expand Up @@ -88,7 +86,8 @@ public void run(Manager manager) throws InterruptedException {
formScanner = new FormScanner(config);


config.initialize(this);
// Init all plugins
config.getPlugins().forEach(pluginConfig -> pluginConfig.initialize(this));


// Initialization of internationalization
Expand Down Expand Up @@ -127,34 +126,11 @@ public void run(Manager manager) throws InterruptedException {
}
}

try {
formScanner.execute(null, null);
}
catch (Exception e) {
Throwables.throwIfUnchecked(e);
throw new RuntimeException(e);
}
formScanner.execute(null, null);

registerTasks(manager, environment, config);
}

private void registerTasks(Manager manager, Environment environment, ConqueryConfig config) {
environment.admin().addTask(formScanner);
environment.admin().addTask(
new QueryCleanupTask(getMetaStorage(), Duration.of(
config.getQueries().getOldQueriesTime().getQuantity(),
config.getQueries().getOldQueriesTime().getUnit().toChronoUnit()
)));

environment.admin().addTask(new PermissionCleanupTask(getMetaStorage()));
manager.getAdminTasks().forEach(environment.admin()::addTask);
environment.admin().addTask(new ReloadMetaStorageTask(getMetaStorage()));

final ShutdownTask shutdown = new ShutdownTask();
environment.admin().addTask(shutdown);
environment.lifecycle().addServerLifecycleListener(shutdown);
}

private void configureApiServlet(ConqueryConfig config, Environment environment) {
ResourceConfig jerseyConfig = environment.jersey().getResourceConfig();
RESTServer.configure(config, jerseyConfig);
Expand All @@ -171,14 +147,6 @@ protected void configure() {
jerseyConfig.register(PathParamInjector.class);
}

private void loadMetaStorage() {
log.info("Opening MetaStorage");
getMetaStorage().openStores(getInternalMapperFactory().createManagerPersistenceMapper(getDatasetRegistry(), getMetaStorage()), getEnvironment().metrics());
log.info("Loading MetaStorage");
getMetaStorage().loadData();
log.info("MetaStorage loaded {}", getMetaStorage());
}

@SneakyThrows(InterruptedException.class)
public void loadNamespaces() {

Expand All @@ -203,6 +171,31 @@ public void loadNamespaces() {
}
}

private void loadMetaStorage() {
log.info("Opening MetaStorage");
getMetaStorage().openStores(getInternalMapperFactory().createManagerPersistenceMapper(getDatasetRegistry(), getMetaStorage()), getEnvironment().metrics());
log.info("Loading MetaStorage");
getMetaStorage().loadData();
log.info("MetaStorage loaded {}", getMetaStorage());
}

private void registerTasks(Manager manager, Environment environment, ConqueryConfig config) {
environment.admin().addTask(formScanner);
environment.admin().addTask(
new QueryCleanupTask(getMetaStorage(), Duration.of(
config.getQueries().getOldQueriesTime().getQuantity(),
config.getQueries().getOldQueriesTime().getUnit().toChronoUnit()
)));

environment.admin().addTask(new PermissionCleanupTask(getMetaStorage()));
manager.getAdminTasks().forEach(environment.admin()::addTask);
environment.admin().addTask(new ReloadMetaStorageTask(getMetaStorage()));

final ShutdownTask shutdown = new ShutdownTask();
environment.admin().addTask(shutdown);
environment.lifecycle().addServerLifecycleListener(shutdown);
}

@Override
public void start() throws Exception {
manager.start();
Expand Down
16 changes: 9 additions & 7 deletions backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@
import java.util.Map;
import java.util.UUID;
import java.util.function.Function;
import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.Entity;
import jakarta.ws.rs.client.Invocation;
import jakarta.ws.rs.client.WebTarget;
import jakarta.ws.rs.core.GenericType;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;

import com.bakdata.conquery.apiv1.forms.ExternalForm;
import com.bakdata.conquery.models.auth.entities.User;
Expand All @@ -16,16 +23,11 @@
import com.codahale.metrics.health.HealthCheck;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.dropwizard.health.check.http.HttpHealthCheck;
import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.Entity;
import jakarta.ws.rs.client.Invocation;
import jakarta.ws.rs.client.WebTarget;
import jakarta.ws.rs.core.GenericType;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;

@Slf4j
@Getter
public class ExternalFormBackendApi {

public final static String TASK_ID = "task-id";
Expand Down
64 changes: 31 additions & 33 deletions backend/src/main/java/com/bakdata/conquery/io/jackson/Jackson.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,48 +33,46 @@ public class Jackson {

/**
* Helper method that also creates a copy of the injected values to reduce side effects.
*
* @param om the {@link ObjectMapper} which is copied. Its {@link com.fasterxml.jackson.databind.InjectableValues} must be {@link MutableInjectableValues}
* @return A copy of the {@link ObjectMapper} along with a copy of its {@link MutableInjectableValues}.
*/
public static ObjectMapper copyMapperAndInjectables(ObjectMapper om) {
final ObjectMapper copy = om.copy();
copy.setInjectableValues(((MutableInjectableValues)copy.getInjectableValues()).copy());
copy.setInjectableValues(((MutableInjectableValues) copy.getInjectableValues()).copy());
return copy;
}

public static <T extends ObjectMapper> T configure(T objectMapper){
public static <T extends ObjectMapper> T configure(T objectMapper) {

objectMapper
.enable(MapperFeature.PROPAGATE_TRANSIENT_MARKER)
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
.enable(Feature.ALLOW_UNQUOTED_FIELD_NAMES)
.enable(Feature.ALLOW_COMMENTS)
.enable(Feature.ALLOW_UNQUOTED_CONTROL_CHARS)
//TODO this is just a hotfix to avoid reimports
// .enable(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES)
.enable(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE)
.enable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES)
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS)
.enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY)
.enable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.enable(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS)
.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS)
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
.setLocale(Locale.ROOT)
.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET)
.enable(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS)
.enable(SerializationFeature.WRITE_NULL_MAP_VALUES)
.registerModule(new JavaTimeModule())
.registerModule(new ParameterNamesModule())
.registerModule(new GuavaModule())
.registerModule(new BlackbirdModule())
.registerModule(ConquerySerializersModule.INSTANCE)
.setSerializationInclusion(Include.ALWAYS)
.setDefaultPropertyInclusion(Include.ALWAYS)
//.setAnnotationIntrospector(new RestrictingAnnotationIntrospector())
.setInjectableValues(new MutableInjectableValues())
.addMixIn(Permission.class, ConqueryPermission.class)
.addMixIn(Object2IntMap.class, Object2IntMapMixin.class);
objectMapper.enable(MapperFeature.PROPAGATE_TRANSIENT_MARKER)
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
.enable(Feature.ALLOW_UNQUOTED_FIELD_NAMES)
.enable(Feature.ALLOW_COMMENTS)
.enable(Feature.ALLOW_UNQUOTED_CONTROL_CHARS)
.enable(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE)
.enable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES)
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS)
.enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY)
.enable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.enable(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS)
.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS)
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
.setLocale(Locale.ROOT)
.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET)
.enable(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS)
.enable(SerializationFeature.WRITE_NULL_MAP_VALUES)
.registerModule(new JavaTimeModule())
.registerModule(new ParameterNamesModule())
.registerModule(new GuavaModule())
.registerModule(new BlackbirdModule())
.registerModule(ConquerySerializersModule.INSTANCE)
.setSerializationInclusion(Include.ALWAYS)
.setDefaultPropertyInclusion(Include.ALWAYS)
//.setAnnotationIntrospector(new RestrictingAnnotationIntrospector())
.setInjectableValues(new MutableInjectableValues())
.addMixIn(Permission.class, ConqueryPermission.class)
.addMixIn(Object2IntMap.class, Object2IntMapMixin.class);

return objectMapper;
}
Expand Down
Loading

0 comments on commit ef78ded

Please sign in to comment.