Skip to content

Commit

Permalink
Integration with Neptune Analytics (#31)
Browse files Browse the repository at this point in the history 
Added support for integration with Neptune Analytics via SDK or http:

    added new dependency on client-neptune-graph
    added logic to fall back to neptune graph SDK if Axios request fails during pipeline creation (previous logic threw Error as the analytics SDK was not yet available)
    created new lambda template which is used if the user specifies --output-resolver-query-sdk option
    set additional lambda environment variable for neptune db name which is required to execute queries using the neptune graph SDK
    fixed function which retrieves graph summary to use neptune graph SDK if the neptune-type is neptune-graph (the summary endpoint path for neptune-db is not the same for neptune-graph)
    fixed CDK pipeline to only fetch cluster info if the type is neptune-db as it is not required for neptune-graph (analytics)
    set isNeptuneIAMAuth to true if the neptune type is detected as neptune-graph
    introduced util.js for parsing functions that are used across multiple modules
    refactored function which had many params to use an object param instead for better readability
    introduced new test case 7 which sets --output-resolver-query-sdk option
  • Loading branch information
@andreachild
andreachild authored Oct 15, 2024
1 parent 5380603 commit e0656ae
Show file tree
Hide file tree
Showing 24 changed files with 5,698 additions and 278 deletions.
14 changes: 2 additions & 12 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,6 @@
output/**
package-lock.json
.vscode/launch.json
node_modules/**
templates/Lambda4AppSyncHTTP/node_modules/**
templates/Lambda4AppSyncSDK/node_modules/**
**/node_modules/
coverage/**
test/node_modules/**
test/TestCases/Case01/output/**
test/TestCases/Case01/output/**
test/TestCases/Case02/output/**
test/TestCases/Case03/output/**
test/TestCases/Case04/output/**
test/TestCases/Case05/output/**
test/TestCases/Case06/output/**
**/output/
*.iml
8 changes: 6 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@aws/neptune-for-graphql",
"version": "1.0.0",
"version": "1.1.0",
"description": "CLI utility to create and maintain a GraphQL API for Amazon Neptune",
"keywords": [
"Amazon Neptune",
Expand All @@ -19,9 +19,10 @@
"test": "test"
},
"scripts": {
"postinstall": "cd templates/Lambda4AppSyncHTTP && npm install && cd ../Lambda4AppSyncSDK && npm install",
"postinstall": "cd templates/Lambda4AppSyncHTTP && npm install && cd ../Lambda4AppSyncSDK && npm install && cd ../Lambda4AppSyncGraphSDK && npm install",
"lint": "eslint neptune-for-graphql.mjs ./src",
"test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --runInBand --detectOpenHandles --config .jest.js",
"test:sdk": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --runInBand --detectOpenHandles --config .jest.js --testPathPattern=test/TestCases/Case07",
"test:resolver": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --runInBand --detectOpenHandles --config .jest.js --testPathPattern=test/TestCases/Case01"
},
"jest": {
Expand All @@ -42,6 +43,8 @@
"./templates/Lambda4AppSyncHTTP/package.json",
"./templates/Lambda4AppSyncSDK/index.mjs",
"./templates/Lambda4AppSyncSDK/package.json",
"./templates/Lambda4AppSyncGraphSDK/index.mjs",
"./templates/Lambda4AppSyncGraphSDK/package.json",
"./src/**"
],
"author": "AWS",
Expand All @@ -53,6 +56,7 @@
"@aws-sdk/client-lambda": "3.387.0",
"@aws-sdk/client-neptune": "3.387.0",
"@aws-sdk/client-neptunedata": "3.403.0",
"@aws-sdk/client-neptune-graph": "3.662.0",
"@aws-sdk/credential-providers": "3.414.0",
"archiver": "5.3.1",
"aws4-axios": "3.3.0",
Expand Down
105 changes: 63 additions & 42 deletions src/CDKPipelineApp.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ express or implied. See the License for the specific language governing
permissions and limitations under the License.
*/

import { getNeptuneClusterinfoBy } from './pipelineResources.js'
import { getNeptuneClusterDbInfoBy } from './pipelineResources.js'
import { readFile, writeFile } from 'fs/promises';
//import semver from 'semver';
import fs from 'fs';
Expand All @@ -23,6 +23,7 @@ let REGION = '';
let NEPTUNE_DB_NAME = '';
let NEPTUNE_HOST = null;
let NEPTUNE_PORT = null;
let NEPTUNE_TYPE = null;
let NEPTUNE_DBSubnetGroup = null;
let NEPTUNE_IAM_POLICY_RESOURCE = '*';
let LAMBDA_ZIP_FILE = '';
Expand Down Expand Up @@ -73,11 +74,27 @@ async function createDeploymentFile(folderPath, zipFilePath) {
}


async function createAWSpipelineCDK (pipelineName, neptuneDBName, neptuneDBregion, appSyncSchema, schemaModel, lambdaFilesPath, outputFile, __dirname, quiet, isNeptuneIAMAuth, neptuneHost, neptunePort, outputFolderPath ) {
async function createAWSpipelineCDK({
pipelineName,
neptuneDBName,
neptuneDBregion,
appSyncSchema,
schemaModel,
lambdaFilesPath,
outputFile,
__dirname,
quiet,
isNeptuneIAMAuth,
neptuneHost,
neptunePort,
outputFolderPath,
neptuneType
}) {

NAME = pipelineName;
REGION = neptuneDBregion;
NEPTUNE_DB_NAME = neptuneDBName;
NEPTUNE_TYPE = neptuneType;
APPSYNC_SCHEMA = appSyncSchema;
SCHEMA_MODEL = schemaModel;
NEPTUNE_HOST = neptuneHost;
Expand All @@ -88,50 +105,52 @@ async function createAWSpipelineCDK (pipelineName, neptuneDBName, neptuneDBregio
let spinner = null;
let neptuneClusterInfo = null;

try {
if (!quiet) console.log('Get Neptune Cluster Info');
if (!quiet) spinner = ora('Getting ...').start();
neptuneClusterInfo = await getNeptuneClusterinfoBy(NEPTUNE_DB_NAME, REGION);
if (!quiet) spinner.succeed('Got Neptune Cluster Info');
if (isNeptuneIAMAuth) {
if (!neptuneClusterInfo.isIAMauth) {
console.error("The Neptune database authentication is set to VPC.");
console.error("Remove the --output-aws-pipeline-cdk-neptune-IAM option.");
process.exit(1);
}
} else {
if (neptuneClusterInfo.isIAMauth) {
console.error("The Neptune database authentication is set to IAM.");
console.error("Add the --output-aws-pipeline-cdk-neptune-IAM option.");
process.exit(1);
if (neptuneType === 'neptune-db') {
try {
if (!quiet) console.log('Get Neptune Cluster Info');
if (!quiet) spinner = ora('Getting ...').start();
neptuneClusterInfo = await getNeptuneClusterDbInfoBy(NEPTUNE_DB_NAME, REGION);
if (!quiet) spinner.succeed('Got Neptune Cluster Info');
if (isNeptuneIAMAuth) {
if (!neptuneClusterInfo.isIAMauth) {
console.error("The Neptune database authentication is set to VPC.");
console.error("Remove the --output-aws-pipeline-cdk-neptune-IAM option.");
process.exit(1);
}
} else {
if (!quiet) console.log(`Subnet Group: ` + yellow(neptuneClusterInfo.dbSubnetGroup));
if (neptuneClusterInfo.isIAMauth) {
console.error("The Neptune database authentication is set to IAM.");
console.error("Add the --output-aws-pipeline-cdk-neptune-IAM option.");
process.exit(1);
} else {
if (!quiet) console.log(`Subnet Group: ` + yellow(neptuneClusterInfo.dbSubnetGroup));
}
}
}

if (neptuneClusterInfo.version != '') {
const v = neptuneClusterInfo.version;
if (lambdaFilesPath.includes('SDK') == true && //semver.satisfies(v, '>=1.2.1.0') ) {
(v == '1.2.1.0' || v == '1.2.0.2' || v == '1.2.0.1' || v == '1.2.0.0' || v == '1.1.1.0' || v == '1.1.0.0')) {
console.error("Neptune SDK query is supported starting with Neptune versions 1.2.1.0.R5");
console.error("Switch to Neptune HTTPS query with option --output-resolver-query-https");
process.exit(1);
if (neptuneClusterInfo.version != '') {
const v = neptuneClusterInfo.version;
if (lambdaFilesPath.includes('SDK') == true && //semver.satisfies(v, '>=1.2.1.0') ) {
(v == '1.2.1.0' || v == '1.2.0.2' || v == '1.2.0.1' || v == '1.2.0.0' || v == '1.1.1.0' || v == '1.1.0.0')) {
console.error("Neptune SDK query is supported starting with Neptune versions 1.2.1.0.R5");
console.error("Switch to Neptune HTTPS query with option --output-resolver-query-https");
process.exit(1);
}
}
}

NEPTUNE_HOST = neptuneClusterInfo.host;
NEPTUNE_PORT = neptuneClusterInfo.port;
NEPTUNE_DBSubnetGroup = neptuneClusterInfo.dbSubnetGroup.replace('default-', '');
NEPTUNE_IAM_POLICY_RESOURCE = neptuneClusterInfo.iamPolicyResource;

} catch (error) {
if (!quiet) spinner.fail("Error getting Neptune Cluster Info.");
if (!isNeptuneIAMAuth) {
spinner.clear();
console.error("VPC data is not available to proceed.");
process.exit(1);
} else {
if (!quiet) console.log("Proceeding without getting Neptune Cluster info.");
NEPTUNE_HOST = neptuneClusterInfo.host;
NEPTUNE_PORT = neptuneClusterInfo.port;
NEPTUNE_DBSubnetGroup = neptuneClusterInfo.dbSubnetGroup.replace('default-', '');
NEPTUNE_IAM_POLICY_RESOURCE = neptuneClusterInfo.iamPolicyResource;

} catch (error) {
if (!quiet) spinner.fail("Error getting Neptune Cluster Info.");
if (!isNeptuneIAMAuth) {
spinner.clear();
console.error("VPC data is not available to proceed.");
process.exit(1);
} else {
if (!quiet) console.log("Proceeding without getting Neptune Cluster info.");
}
}
}

Expand All @@ -147,7 +166,9 @@ async function createAWSpipelineCDK (pipelineName, neptuneDBName, neptuneDBregio
CDKFile = CDKFile.replace( "const NAME = '';", `const NAME = '${NAME}';` );
CDKFile = CDKFile.replace( "const REGION = '';", `const REGION = '${REGION}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_HOST = '';", `const NEPTUNE_HOST = '${NEPTUNE_HOST}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_PORT = '';", `const NEPTUNE_PORT = '${NEPTUNE_PORT}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_PORT = '';", `const NEPTUNE_PORT = '${NEPTUNE_PORT}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_DB_NAME = '';", `const NEPTUNE_DB_NAME = '${NEPTUNE_DB_NAME}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_TYPE = '';", `const NEPTUNE_TYPE = '${NEPTUNE_TYPE}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_DBSubnetGroup = null;", `const NEPTUNE_DBSubnetGroup = '${NEPTUNE_DBSubnetGroup}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_IAM_AUTH = false;", `const NEPTUNE_IAM_AUTH = ${isNeptuneIAMAuth};` );
CDKFile = CDKFile.replace( "const NEPTUNE_IAM_POLICY_RESOURCE = '*';", `const NEPTUNE_IAM_POLICY_RESOURCE = '${NEPTUNE_IAM_POLICY_RESOURCE}';` );
Expand Down
Loading

0 comments on commit e0656ae

Please sign in to comment.