Skip to content

Commit

Permalink
migrate export to database
Browse files Browse the repository at this point in the history
  • Loading branch information
lassewesth committed Aug 15, 2024
1 parent 403e95a commit 95c8786
Show file tree
Hide file tree
Showing 23 changed files with 527 additions and 248 deletions.
6 changes: 6 additions & 0 deletions applications/facade/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ description = 'Neo4j Graph Data Science :: Application Facade'
group = 'org.neo4j.gds'

dependencies {
neodeps().each {
compileOnly(group: 'org.neo4j', name: it, version: ver.'neo4j') {
transitive = false
}
}

implementation project(":algo")
implementation project(":algorithms-machinery")
implementation project(":centrality-algorithms")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.neo4j.gds.algorithms.similarity.WriteRelationshipService;
import org.neo4j.gds.applications.algorithms.centrality.CentralityApplications;
import org.neo4j.gds.applications.algorithms.community.CommunityApplications;
import org.neo4j.gds.applications.modelcatalog.ModelRepository;
import org.neo4j.gds.applications.algorithms.embeddings.NodeEmbeddingApplications;
import org.neo4j.gds.applications.algorithms.machinery.AlgorithmEstimationTemplate;
import org.neo4j.gds.applications.algorithms.machinery.AlgorithmProcessingTemplate;
Expand All @@ -40,13 +39,16 @@
import org.neo4j.gds.applications.graphstorecatalog.GraphCatalogApplications;
import org.neo4j.gds.applications.modelcatalog.DefaultModelCatalogApplications;
import org.neo4j.gds.applications.modelcatalog.ModelCatalogApplications;
import org.neo4j.gds.applications.modelcatalog.ModelRepository;
import org.neo4j.gds.applications.operations.OperationsApplications;
import org.neo4j.gds.core.loading.GraphStoreCatalogService;
import org.neo4j.gds.core.model.ModelCatalog;
import org.neo4j.gds.logging.Log;
import org.neo4j.gds.memest.DatabaseGraphStoreEstimationService;
import org.neo4j.gds.metrics.algorithms.AlgorithmMetricsService;
import org.neo4j.gds.metrics.projections.ProjectionMetricsService;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Transaction;

import java.util.Optional;
import java.util.function.Function;
Expand Down Expand Up @@ -106,7 +108,9 @@ public static ApplicationsFacade create(
RequestScopedDependencies requestScopedDependencies,
WriteContext writeContext,
ModelCatalog modelCatalog,
ModelRepository modelRepository
ModelRepository modelRepository,
GraphDatabaseService graphDatabaseService,
Transaction procedureTransaction
) {
var databaseGraphStoreEstimationService = new DatabaseGraphStoreEstimationService(
requestScopedDependencies.getGraphLoaderContext(),
Expand Down Expand Up @@ -154,9 +158,12 @@ public static ApplicationsFacade create(

var graphCatalogApplications = createGraphCatalogApplications(
log,
graphCatalogApplicationsDecorator,
graphStoreCatalogService,
projectionMetricsService
projectionMetricsService,
requestScopedDependencies,
graphDatabaseService,
procedureTransaction,
graphCatalogApplicationsDecorator
);

var miscellaneousApplications = MiscellaneousApplications.create(
Expand Down Expand Up @@ -245,14 +252,20 @@ private static AlgorithmProcessingTemplate createAlgorithmProcessingTemplate(

private static GraphCatalogApplications createGraphCatalogApplications(
Log log,
Optional<Function<GraphCatalogApplications, GraphCatalogApplications>> graphCatalogApplicationsDecorator,
GraphStoreCatalogService graphStoreCatalogService,
ProjectionMetricsService projectionMetricsService
ProjectionMetricsService projectionMetricsService,
RequestScopedDependencies requestScopedDependencies,
GraphDatabaseService graphDatabaseService,
Transaction procedureTransaction,
Optional<Function<GraphCatalogApplications, GraphCatalogApplications>> graphCatalogApplicationsDecorator
) {
var graphCatalogApplications = DefaultGraphCatalogApplications.create(
log,
graphStoreCatalogService,
projectionMetricsService
projectionMetricsService,
requestScopedDependencies,
graphDatabaseService,
procedureTransaction
);

if (graphCatalogApplicationsDecorator.isEmpty()) return graphCatalogApplications;
Expand Down
1 change: 1 addition & 0 deletions applications/graph-store-catalog/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ dependencies {
implementation project(':graph-projection-api')
implementation project(':graph-sampling')
implementation project(':graph-schema-api')
implementation project(':io-core')
implementation project(':legacy-cypher-projection')
implementation project(':logging')
implementation project(':memory-usage')
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright (c) "Neo4j"
* Neo4j Sweden AB [http://neo4j.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.gds.applications.graphstorecatalog;

public class DatabaseExportResult extends GraphStoreExportResult {
public final String dbName;

public DatabaseExportResult(
String graphName,
String dbName,
long nodeCount,
long relationshipCount,
long relationshipTypeCount,
long nodePropertyCount,
long relationshipPropertyCount,
long writeMillis
) {
super(
graphName,
nodeCount,
relationshipCount,
relationshipTypeCount,
nodePropertyCount,
relationshipPropertyCount,
writeMillis
);
this.dbName = dbName;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@
import org.neo4j.gds.api.GraphName;
import org.neo4j.gds.api.User;
import org.neo4j.gds.applications.algorithms.machinery.MemoryEstimateResult;
import org.neo4j.gds.applications.algorithms.machinery.RequestScopedDependencies;
import org.neo4j.gds.beta.filter.GraphFilterResult;
import org.neo4j.gds.core.io.db.GraphStoreToDatabaseExporterConfig;
import org.neo4j.gds.core.loading.CatalogRequest;
import org.neo4j.gds.core.loading.GraphDropNodePropertiesResult;
import org.neo4j.gds.core.loading.GraphDropRelationshipResult;
Expand All @@ -44,6 +46,7 @@
import org.neo4j.gds.termination.TerminationFlag;
import org.neo4j.gds.transaction.TransactionContext;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Transaction;

import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -104,6 +107,7 @@ public class DefaultGraphCatalogApplications implements GraphCatalogApplications
private final GraphSamplingApplication graphSamplingApplication;
private final EstimateCommonNeighbourAwareRandomWalkApplication estimateCommonNeighbourAwareRandomWalkApplication;
private final GenerateGraphApplication generateGraphApplication;
private final ExportToDatabaseApplication exportToDatabaseApplication;

DefaultGraphCatalogApplications(
Log log,
Expand All @@ -128,7 +132,8 @@ public class DefaultGraphCatalogApplications implements GraphCatalogApplications
WriteNodeLabelApplication writeNodeLabelApplication,
WriteNodePropertiesApplication writeNodePropertiesApplication,
WriteRelationshipPropertiesApplication writeRelationshipPropertiesApplication,
WriteRelationshipsApplication writeRelationshipsApplication
WriteRelationshipsApplication writeRelationshipsApplication,
ExportToDatabaseApplication exportToDatabaseApplication
) {
this.log = log;
this.graphStoreCatalogService = graphStoreCatalogService;
Expand All @@ -155,12 +160,16 @@ public class DefaultGraphCatalogApplications implements GraphCatalogApplications
this.graphSamplingApplication = graphSamplingApplication;
this.estimateCommonNeighbourAwareRandomWalkApplication = estimateCommonNeighbourAwareRandomWalkApplication;
this.generateGraphApplication = generateGraphApplication;
this.exportToDatabaseApplication = exportToDatabaseApplication;
}

public static GraphCatalogApplications create(
Log log,
GraphStoreCatalogService graphStoreCatalogService,
ProjectionMetricsService projectionMetricsService
ProjectionMetricsService projectionMetricsService,
RequestScopedDependencies requestScopedDependencies,
GraphDatabaseService graphDatabaseService,
Transaction procedureTransaction
) {
var graphNameValidationService = new GraphNameValidationService();

Expand All @@ -175,6 +184,7 @@ public static GraphCatalogApplications create(
var dropNodePropertiesApplication = new DropNodePropertiesApplication(log);
var dropRelationshipsApplication = new DropRelationshipsApplication(log);
var estimateCommonNeighbourAwareRandomWalkApplication = new EstimateCommonNeighbourAwareRandomWalkApplication();
var exportToDatabaseApplication = new ExportToDatabaseApplication(log, graphStoreCatalogService, graphDatabaseService, procedureTransaction, requestScopedDependencies.getDatabaseId(), requestScopedDependencies.getTaskRegistryFactory(), requestScopedDependencies.getUser(), requestScopedDependencies.getUserLogRegistryFactory());
var generateGraphApplication = new GenerateGraphApplication(log, graphStoreCatalogService);
var graphMemoryUsageApplication = new GraphMemoryUsageApplication(graphStoreCatalogService);
var graphSamplingApplication = new GraphSamplingApplication(log, graphStoreCatalogService);
Expand Down Expand Up @@ -209,6 +219,7 @@ public static GraphCatalogApplications create(
.withDropNodePropertiesApplication(dropNodePropertiesApplication)
.withDropRelationshipsApplication(dropRelationshipsApplication)
.withEstimateCommonNeighbourAwareRandomWalkApplication(estimateCommonNeighbourAwareRandomWalkApplication)
.withExportToDatabaseApplication(exportToDatabaseApplication)
.withGenerateGraphApplication(generateGraphApplication)
.withGraphMemoryUsageApplication(graphMemoryUsageApplication)
.withGraphSamplingApplication(graphSamplingApplication)
Expand All @@ -234,9 +245,9 @@ public boolean graphExists(User user, DatabaseId databaseId, String graphNameAsS
}

/**
* @param failIfMissing enable validation that graphs exist before dropping them
* @param databaseNameOverride optional override
* @param usernameOverride optional override
* @param failIfMissing enable validation that graphs exist before dropping them
* @param databaseNameOverride optional override
* @param usernameOverride optional override
* @throws IllegalArgumentException if a database name was null or blank or not a String
*/
@Override
Expand All @@ -254,7 +265,13 @@ public List<GraphStoreCatalogEntry> dropGraph(
var databaseId = currentDatabase.orOverride(databaseNameOverride);
var parsedUsernameOverride = User.parseUsernameOverride(usernameOverride);

return dropGraphApplication.compute(validatedGraphNames, failIfMissing, databaseId, operator, parsedUsernameOverride);
return dropGraphApplication.compute(
validatedGraphNames,
failIfMissing,
databaseId,
operator,
parsedUsernameOverride
);
}

@Override
Expand Down Expand Up @@ -968,6 +985,14 @@ public GraphGenerationStats generateGraph(
return generateGraphApplication.compute(databaseId, averageDegree, configuration);
}

@Override
public DatabaseExportResult exportToDatabase(
GraphName graphName,
GraphStoreToDatabaseExporterConfig configuration
) {
return exportToDatabaseApplication.run(graphName, configuration);
}

private RandomWalkSamplingResult sampleRandomWalk(
User user,
DatabaseId databaseId,
Expand Down
Loading

0 comments on commit 95c8786

Please sign in to comment.