Skip to content

Commit

Permalink
Capture graph project metrics
Browse files Browse the repository at this point in the history 
Co-authored-by: Ioannis Panagiotas <ioannis.panagiotas@neotechnology.com>
Co-Authored-By: Lasse Westh-Nielsen <lassewesth@gmail.com>
  • Loading branch information
@IoannisPanagiotas @lassewesth @vnickolov
2 people authored and vnickolov committed Nov 16, 2023
1 parent 33dbf48 commit 5c56a05
Show file tree
Hide file tree
Showing 26 changed files with 195 additions and 88 deletions.
1 change: 1 addition & 0 deletions applications/graph-store-catalog/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ dependencies {
implementation project(':graph-schema-api')
implementation project(':logging')
implementation project(':memory-usage')
implementation project(':metrics-api')
implementation project(':native-projection')
implementation project(':neo4j-api')
implementation project(':progress-tracking')
Expand Down
60 changes: 39 additions & 21 deletions .../main/java/org/neo4j/gds/applications/graphstorecatalog/DefaultCatalogBusinessFacade.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.neo4j.gds.graphsampling.RandomWalkBasedNodesSampler;
import org.neo4j.gds.graphsampling.config.RandomWalkWithRestartsConfig;
import org.neo4j.gds.logging.Log;
import org.neo4j.gds.metrics.projections.ProjectionMetricsService;
import org.neo4j.gds.projection.GraphProjectNativeResult;
import org.neo4j.gds.results.MemoryEstimateResult;
import org.neo4j.gds.transaction.TransactionContext;
Expand Down Expand Up @@ -107,6 +108,7 @@ public class DefaultCatalogBusinessFacade implements CatalogBusinessFacade {
private final GraphSamplingApplication graphSamplingApplication;
private final EstimateCommonNeighbourAwareRandomWalkApplication estimateCommonNeighbourAwareRandomWalkApplication;
private final GenerateGraphApplication generateGraphApplication;
private final ProjectionMetricsService projectionMetricsService;

public DefaultCatalogBusinessFacade(
Log log,
Expand All @@ -132,7 +134,8 @@ public DefaultCatalogBusinessFacade(
WriteNodePropertiesApplication writeNodePropertiesApplication,
WriteRelationshipPropertiesApplication writeRelationshipPropertiesApplication,
WriteNodeLabelApplication writeNodeLabelApplication,
WriteRelationshipsApplication writeRelationshipsApplication
WriteRelationshipsApplication writeRelationshipsApplication,
ProjectionMetricsService projectionMetricsService
) {
this.log = log;

Expand Down Expand Up @@ -160,6 +163,7 @@ public DefaultCatalogBusinessFacade(
this.graphSamplingApplication = graphSamplingApplication;
this.estimateCommonNeighbourAwareRandomWalkApplication = estimateCommonNeighbourAwareRandomWalkApplication;
this.generateGraphApplication = generateGraphApplication;
this.projectionMetricsService = projectionMetricsService;
}

@Override
Expand Down Expand Up @@ -230,16 +234,23 @@ public GraphProjectNativeResult nativeProject(
rawConfiguration
);

return nativeProjectApplication.project(
databaseId,
graphDatabaseService,
graphProjectMemoryUsageService,
taskRegistryFactory,
terminationFlag,
transactionContext,
userLogRegistryFactory,
configuration
);
var projectMetric = projectionMetricsService.createNative();
try(projectMetric) {
projectMetric.start();
return nativeProjectApplication.project(
databaseId,
graphDatabaseService,
graphProjectMemoryUsageService,
taskRegistryFactory,
terminationFlag,
transactionContext,
userLogRegistryFactory,
configuration
);
} catch (Exception e) {
projectMetric.failed();
throw e;
}
}

@Override
Expand Down Expand Up @@ -296,16 +307,23 @@ public GraphProjectCypherResult cypherProject(
rawConfiguration
);

return cypherProjectApplication.project(
databaseId,
graphDatabaseService,
graphProjectMemoryUsageService,
taskRegistryFactory,
terminationFlag,
transactionContext,
userLogRegistryFactory,
configuration
);
var projectMetric = projectionMetricsService.createCypher();
try(projectMetric) {
projectMetric.start();
return cypherProjectApplication.project(
databaseId,
graphDatabaseService,
graphProjectMemoryUsageService,
taskRegistryFactory,
terminationFlag,
transactionContext,
userLogRegistryFactory,
configuration
);
} catch (Exception e) {
projectMetric.failed();
throw e;
}
}

@Override
Expand Down
8 changes: 8 additions & 0 deletions ...t/java/org/neo4j/gds/applications/graphstorecatalog/DefaultCatalogBusinessFacadeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ void shouldDetermineGraphExists() {
null,
null,
null,
null,
null
);

Expand Down Expand Up @@ -101,6 +102,7 @@ void shouldDetermineGraphDoesNotExist() {
null,
null,
null,
null,
null
);

Expand Down Expand Up @@ -145,6 +147,7 @@ void shouldValidateInputGraphName() {
null,
null,
null,
null,
null
);

Expand Down Expand Up @@ -180,6 +183,7 @@ void shouldUseStrictValidationWhenProjecting() {
null,
null,
null,
null,
null
);

Expand Down Expand Up @@ -250,6 +254,7 @@ void shouldHandleNullsInNativeProjectParameters() {
null,
null,
null,
null,
null
);

Expand Down Expand Up @@ -345,6 +350,7 @@ void shouldHandleNullsInCypherProjectParameters() {
null,
null,
null,
null,
null
);

Expand Down Expand Up @@ -438,6 +444,7 @@ void shouldDoExistenceCheckWhenProjecting() {
null,
null,
null,
null,
null
);

Expand Down Expand Up @@ -519,6 +526,7 @@ void shouldDoPositiveExistenceCheckWhenProjectingSubGraph() {
null,
null,
null,
null,
null
);

Expand Down
5 changes: 4 additions & 1 deletion cypher-aggregation/src/main/java/org/neo4j/gds/projection/AlphaCypherAggregation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.neo4j.gds.compat.Neo4jProxy;
import org.neo4j.gds.core.Username;
import org.neo4j.gds.core.loading.Capabilities.WriteMode;
import org.neo4j.gds.metrics.MetricsFacade;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Transaction;
import org.neo4j.internal.kernel.api.exceptions.ProcedureException;
Expand Down Expand Up @@ -104,6 +105,7 @@ public static CompatUserAggregationFunction newInstance() {
@Override
public CompatUserAggregator create(Context ctx) throws ProcedureException {
var databaseService = Neo4jProxy.lookupComponentProvider(ctx, GraphDatabaseService.class, true);
var metricsFacade = Neo4jProxy.lookupComponentProvider(ctx, MetricsFacade.class, true);
var username = Neo4jProxy.lookupComponentProvider(ctx, Username.class, true);
var transaction = Neo4jProxy.lookupComponentProvider(ctx, Transaction.class, true);
var queryProvider = ExecutingQueryProvider.fromTransaction(transaction);
Expand All @@ -117,7 +119,8 @@ public CompatUserAggregator create(Context ctx) throws ProcedureException {
DatabaseId.of(databaseService.databaseName()),
username.username(),
writeMode,
queryProvider
queryProvider,
metricsFacade.projectionMetrics()
);
}
}
6 changes: 4 additions & 2 deletions cypher-aggregation/src/main/java/org/neo4j/gds/projection/AlphaGraphAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.neo4j.gds.api.DatabaseId;
import org.neo4j.gds.core.loading.Capabilities.WriteMode;
import org.neo4j.gds.metrics.projections.ProjectionMetricsService;
import org.neo4j.internal.kernel.api.exceptions.ProcedureException;
import org.neo4j.kernel.api.exceptions.Status;
import org.neo4j.values.AnyValue;
Expand All @@ -36,9 +37,10 @@ public class AlphaGraphAggregator extends GraphAggregator {
DatabaseId databaseId,
String username,
WriteMode writeMode,
ExecutingQueryProvider queryProvider
ExecutingQueryProvider queryProvider,
ProjectionMetricsService projectionMetricsService
) {
super(databaseId, username, writeMode, queryProvider);
super(databaseId, username, writeMode, queryProvider, projectionMetricsService);
}

@Override
Expand Down
5 changes: 4 additions & 1 deletion cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.neo4j.gds.compat.Neo4jProxy;
import org.neo4j.gds.core.Username;
import org.neo4j.gds.core.loading.Capabilities.WriteMode;
import org.neo4j.gds.metrics.MetricsFacade;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Transaction;
import org.neo4j.internal.kernel.api.exceptions.ProcedureException;
Expand Down Expand Up @@ -103,6 +104,7 @@ public static CompatUserAggregationFunction newInstance() {
@Override
public CompatUserAggregator create(Context ctx) throws ProcedureException {
var databaseService = Neo4jProxy.lookupComponentProvider(ctx, GraphDatabaseService.class, true);
var metricsFacade = Neo4jProxy.lookupComponentProvider(ctx, MetricsFacade.class, true);
var username = Neo4jProxy.lookupComponentProvider(ctx, Username.class, true);
var transaction = Neo4jProxy.lookupComponentProvider(ctx, Transaction.class, true);
var queryProvider = ExecutingQueryProvider.fromTransaction(transaction);
Expand All @@ -116,7 +118,8 @@ public CompatUserAggregator create(Context ctx) throws ProcedureException {
DatabaseId.of(databaseService.databaseName()),
username.username(),
writeMode,
queryProvider
queryProvider,
metricsFacade.projectionMetrics()
);
}
}
11 changes: 9 additions & 2 deletions cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.neo4j.gds.core.loading.construction.NodeLabelTokens;
import org.neo4j.gds.core.loading.construction.PropertyValues;
import org.neo4j.gds.core.utils.ProgressTimer;
import org.neo4j.gds.metrics.projections.ProjectionMetricsService;
import org.neo4j.internal.kernel.api.exceptions.ProcedureException;
import org.neo4j.kernel.api.exceptions.Status;
import org.neo4j.kernel.impl.util.ValueUtils;
Expand Down Expand Up @@ -67,6 +68,7 @@ abstract class GraphAggregator implements CompatUserAggregator {
private final String username;
private final WriteMode writeMode;
private final ExecutingQueryProvider queryProvider;
private final ProjectionMetricsService projectionMetricsService;

private final ProgressTimer progressTimer;
private final ConfigValidator configValidator;
Expand All @@ -83,12 +85,14 @@ abstract class GraphAggregator implements CompatUserAggregator {
DatabaseId databaseId,
String username,
WriteMode writeMode,
ExecutingQueryProvider queryProvider
ExecutingQueryProvider queryProvider,
ProjectionMetricsService projectionMetricsService
) {
this.databaseId = databaseId;
this.username = username;
this.writeMode = writeMode;
this.queryProvider = queryProvider;
this.projectionMetricsService = projectionMetricsService;
this.progressTimer = ProgressTimer.start();
this.lock = new ReentrantLock();
this.configValidator = new ConfigValidator();
Expand Down Expand Up @@ -189,10 +193,13 @@ private static NodeLabelToken tryLabelsConfig(AnyValue nodeLabels, String nodeLa

@Override
public AnyValue result() throws ProcedureException {
var projectionMetric = projectionMetricsService.createCypherV2();
AggregationResult result;
try {
try(projectionMetric) {
projectionMetric.start();
result = buildGraph();
} catch (Exception e) {
projectionMetric.failed();
throw new ProcedureException(
Status.Procedure.ProcedureCallFailed,
e,
Expand Down
6 changes: 4 additions & 2 deletions cypher-aggregation/src/main/java/org/neo4j/gds/projection/ProductGraphAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.neo4j.gds.api.DatabaseId;
import org.neo4j.gds.core.loading.Capabilities.WriteMode;
import org.neo4j.gds.metrics.projections.ProjectionMetricsService;
import org.neo4j.internal.kernel.api.exceptions.ProcedureException;
import org.neo4j.kernel.api.exceptions.Status;
import org.neo4j.values.AnyValue;
Expand All @@ -34,9 +35,10 @@ public class ProductGraphAggregator extends GraphAggregator {
DatabaseId databaseId,
String username,
WriteMode writeMode,
ExecutingQueryProvider queryProvider
ExecutingQueryProvider queryProvider,
ProjectionMetricsService projectionMetricsService
) {
super(databaseId, username, writeMode, queryProvider);
super(databaseId, username, writeMode, queryProvider, projectionMetricsService);
}

@Override
Expand Down
4 changes: 3 additions & 1 deletion cypher-aggregation/src/test/java/org/neo4j/gds/projection/ProductGraphAggregatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.neo4j.gds.api.DatabaseId;
import org.neo4j.gds.core.loading.Capabilities.WriteMode;
import org.neo4j.gds.core.loading.GraphStoreCatalog;
import org.neo4j.gds.metrics.MetricsFacade;
import org.neo4j.values.storable.NoValue;
import org.neo4j.values.storable.Values;
import org.neo4j.values.virtual.MapValue;
Expand All @@ -41,7 +42,8 @@ void shouldImportHighNodeIds() {
databaseId,
userName,
WriteMode.LOCAL,
ExecutingQueryProvider.empty()
ExecutingQueryProvider.empty(),
MetricsFacade.PASSTHROUGH_METRICS_FACADE.projectionMetrics()
);

long source = 1L << 50;
Expand Down
10 changes: 5 additions & 5 deletions executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,7 @@
import org.neo4j.gds.core.write.NodePropertyExporterBuilder;
import org.neo4j.gds.core.write.RelationshipExporterBuilder;
import org.neo4j.gds.core.write.RelationshipStreamExporterBuilder;
import org.neo4j.gds.metrics.algorithms.AlgorithmMetricsService;
import org.neo4j.gds.metrics.PassthroughExecutionMetricRegistrar;
import org.neo4j.gds.metrics.MetricsFacade;
import org.neo4j.logging.Log;
import org.neo4j.logging.NullLog;

Expand Down Expand Up @@ -72,7 +71,7 @@ public interface ExecutionContext {

boolean isGdsAdmin();

AlgorithmMetricsService algorithmMetricsService();
MetricsFacade metricsFacade();

@Nullable
RelationshipStreamExporterBuilder relationshipStreamExporterBuilder();
Expand Down Expand Up @@ -178,8 +177,9 @@ public boolean isGdsAdmin() {
}

@Override
public AlgorithmMetricsService algorithmMetricsService() {
return new AlgorithmMetricsService(new PassthroughExecutionMetricRegistrar());
public MetricsFacade metricsFacade() {
return MetricsFacade.PASSTHROUGH_METRICS_FACADE;

}

@Override
Expand Down
2 changes: 1 addition & 1 deletion executor/src/main/java/org/neo4j/gds/executor/ProcedureExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public RESULT compute(
algo.getProgressTracker().setEstimatedResourceFootprint(memoryEstimationInBytes, config.concurrency());


ALGO_RESULT result = executeAlgorithm(builder, algo, executionContext.algorithmMetricsService());
ALGO_RESULT result = executeAlgorithm(builder, algo, executionContext.metricsFacade().algorithmMetrics());

var computationResult = builder
.graph(graph)
Expand Down
5 changes: 2 additions & 3 deletions executor/src/test/java/org/neo4j/gds/executor/MemoryEstimationExecutorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,7 @@
import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory;
import org.neo4j.gds.core.utils.warnings.EmptyUserLogRegistryFactory;
import org.neo4j.gds.gdl.GdlGraphs;
import org.neo4j.gds.metrics.algorithms.AlgorithmMetricsService;
import org.neo4j.gds.metrics.PassthroughExecutionMetricRegistrar;
import org.neo4j.gds.metrics.MetricsFacade;
import org.neo4j.gds.projection.GraphProjectFromStoreConfig;
import org.neo4j.gds.test.TestAlgorithm;
import org.neo4j.gds.test.TestAlgorithmResult;
Expand Down Expand Up @@ -87,7 +86,7 @@ void setup() throws Exception {
.nodeLookup(NodeLookup.EMPTY)
.modelCatalog(ModelCatalog.EMPTY)
.isGdsAdmin(false)
.algorithmMetricsService(new AlgorithmMetricsService(new PassthroughExecutionMetricRegistrar()))
.metricsFacade(MetricsFacade.PASSTHROUGH_METRICS_FACADE)
.build();

memoryEstimationExecutor = new MemoryEstimationExecutor<>(
Expand Down
Loading

0 comments on commit 5c56a05

Please sign in to comment.