Merge remote-tracking branch 'upstream/main' into main

forward_engineering/configs/templates.js

@@ -11,6 +11,11 @@ module.exports = {
 		'${columnDefinitions}${keyConstraints}${checkConstraints}${foreignKeyConstraints}\n' +
 		')${options};\n\n${comment}${columnDescriptions}',
 
+	createTablePartitionOf:
+		'CREATE${temporary} TABLE${ifNotExist} ${name}\n' +
+		'${partitionOf} ${openParenthesis}${keyConstraints}${checkConstraints}${foreignKeyConstraints}\n' +
+		'${closeParenthesis}${options};\n\n${comment}${columnDescriptions}',
+
 	columnDefinition: '${name} ${type}${collation}${primaryKey}${uniqueKey}${defaultValue}${notNull}',
 
 	checkConstraint: '${name} CHECK (${expression})${noInherit}',

forward_engineering/ddlProvider.js

@@ -161,7 +161,8 @@ module.exports = (baseProvider, options, app) => {
 				unlogged,
 				selectStatement,
 				triggers,
-				relatedSchemas,
+				partitionOf,
+				partitionBounds,
 			},
 			isActivated,
 		) {
@@ -178,19 +179,31 @@ module.exports = (baseProvider, options, app) => {
 				key => key.statement,
 			);
 			const keyConstraintsString = generateConstraintsString(dividedKeysConstraints, isActivated);
+			const keyConstraintsValue = partitionOf ? keyConstraintsString?.slice(1) : keyConstraintsString;
 
 			const dividedForeignKeys = divideIntoActivatedAndDeactivated(foreignKeyConstraints, key => key.statement);
 			const foreignKeyConstraintsString = generateConstraintsString(dividedForeignKeys, isActivated);
 
 			const columnDescriptions = '\n' + getColumnComments(tableName, columnDefinitions);
+			const template = partitionOf ? templates.createTablePartitionOf : templates.createTable;
 
-			const tableStatement = assignTemplates(templates.createTable, {
+			const checkConstraintPrefix = partitionOf && !keyConstraintsString ? '\n\t' : ',\n\t';
+			const checkConstraintsValue = !_.isEmpty(checkConstraints)
+				? wrap(_.join(checkConstraints, ',\n\t'), checkConstraintPrefix, '')
+				: '';
+
+			const isEmptyPartitionBody =
+				partitionOf && !keyConstraintsValue && !checkConstraintsValue && !foreignKeyConstraintsString;
+			const openParenthesis = isEmptyPartitionBody ? '' : '(';
+			const closeParenthesis = isEmptyPartitionBody ? '' : ')';
+
+			const tableStatement = assignTemplates(template, {
 				temporary: getTableTemporaryValue(temporary, unlogged),
 				ifNotExist: ifNotExistStr,
 				name: tableName,
-				columnDefinitions: '\t' + _.join(columns, ',\n\t'),
-				keyConstraints: keyConstraintsString,
-				checkConstraints: !_.isEmpty(checkConstraints) ? ',\n\t' + _.join(checkConstraints, ',\n\t') : '',
+				columnDefinitions: !partitionOf ? '\t' + _.join(columns, ',\n\t') : '',
+				keyConstraints: keyConstraintsValue,
+				checkConstraints: checkConstraintsValue,
 				foreignKeyConstraints: foreignKeyConstraintsString,
 				options: getTableOptions({
 					inherits,
@@ -200,9 +213,13 @@ module.exports = (baseProvider, options, app) => {
 					storage_parameter,
 					table_tablespace_name,
 					selectStatement,
+					partitionBounds,
 				}),
 				comment: description ? comment : '',
+				partitionOf: partitionOf ? ` PARTITION OF ${partitionOf} ` : '',
 				columnDescriptions,
+				openParenthesis,
+				closeParenthesis,
 			});
 
 			const createTriggerStatements = getTriggersScript({
@@ -549,11 +566,16 @@ module.exports = (baseProvider, options, app) => {
 
 			const partitioning = _.first(detailsTab.partitioning) || {};
 			const compositePartitionKey = keyHelper.getKeys(partitioning.compositePartitionKey, jsonSchema);
+			const partitionParent = _.get(tableData, `relatedSchemas[${detailsTab.partitionOf}]`);
+			const partitionOf = partitionParent
+				? getNamePrefixedWithSchemaName(partitionParent.collectionName, partitionParent.bucketName)
+				: '';
 			const triggers = hydrateTriggers(entityData, tableData.relatedSchemas);
 
 			return {
 				...tableData,
 				triggers,
+				partitionOf,
 				keyConstraints: keyHelper.getTableKeyConstraints(jsonSchema),
 				inherits: parentTables,
 				selectStatement: _.trim(detailsTab.selectStatement),
@@ -568,6 +590,7 @@ module.exports = (baseProvider, options, app) => {
 					'on_commit',
 					'storage_parameter',
 					'table_tablespace_name',
+					'partitionBounds',
 				),
 			};
 		},

forward_engineering/helpers/tableHelper.js

@@ -16,6 +16,7 @@ module.exports = ({ _, getColumnsList, checkAllKeysDeactivated }) => {
 
 		const statements = [
 			{ key: 'inherits', getValue: getBasicValue('INHERITS') },
+			{ key: 'partitionBounds', getValue: getBasicValue('') },
 			{ key: 'partitioning', getValue: getPartitioning },
 			{ key: 'usingMethod', getValue: getBasicValue('USING') },
 			{ key: 'storage_parameter', getValue: getStorageParameters },

package.json

@@ -1,7 +1,7 @@
 {
     "name": "PostgreSQL",
-    "version": "0.1.10",
-    "versionDate": "2022-01-31",
+    "version": "0.1.13",
+    "versionDate": "2022-02-09",
     "author": "hackolade",
     "engines": {
         "hackolade": "5.4.4",
@@ -39,7 +39,8 @@
             "relationships": {
                 "compositeRelationships": true
             },
-            "FEScriptCommentsSupported": true
+            "FEScriptCommentsSupported": true,
+            "enableFetchSystemEntitiesCheckbox": true
         }
     },
     "description": "Hackolade plugin for PostgreSQL",

properties_pane/entity_level/entityLevelConfig.json

@@ -159,6 +159,22 @@ making sure that you maintain a proper JSON format.
 					}
 				]
 			},
+			{
+				"propertyName": "Partition of",
+				"propertyKeyword": "partitionOf",
+				"propertyType": "selecthashed",
+				"template": "entities",
+				"withEmptyOption": true,
+				"excludeCurrent": true
+			},
+			{
+				"propertyName": "Partition bounds",
+				"propertyKeyword": "partitionBounds",
+				"propertyTooltip": "Partition bounds",
+				"propertyType": "details",
+				"template": "textarea",
+				"markdown": false
+			},
 			{
 				"propertyName": "Partitioning",
 				"propertyKeyword": "partitioning",

reverse_engineering/api.js

@@ -129,6 +129,7 @@ module.exports = {
 						schemaName,
 						collections[schemaName],
 						data.recordSamplingSettings,
+						data.includePartitions,
 					);
 					const { functions, procedures, triggers } = await postgresService.retrieveSchemaLevelData(
 						schemaName,

reverse_engineering/config.json

@@ -14,5 +14,12 @@
 		"port",
 		"userName"
 	],
-	"helpUrl": "https://hackolade.com/help/ConnecttoaPostgreSQLinstance.html"
+	"helpUrl": "https://hackolade.com/help/ConnecttoaPostgreSQLinstance.html",
+	"options": [
+		{
+			"inputLabel": "Include partitions",
+			"inputTooltip": "Reverse-engineer all partitions as tables",
+			"inputKeyword": "includePartitions"
+		}
+	]
 }

reverse_engineering/helpers/connectionHelper.js

@@ -121,6 +121,7 @@ const createClient = async (connectionInfo, logger) => {
 		statement_timeout: Number(connectionInfo.queryRequestTimeout) || 60000,
 		database: connectionInfo.database || connectionInfo.maintenanceDatabase,
 		application_name: 'Hackolade',
+		idleTimeoutMillis: Number(connectionInfo.queryRequestTimeout) || 10000,
 	};
 
 	const client = await connectClient(config).catch(retryOnSslError(connectionInfo, config, logger));
@@ -153,8 +154,7 @@ const retryOnSslError = (connectionInfo, config, logger) => async error => {
 };
 
 const connectClient = async config => {
-	const client = new pg.Client(config);
-	await client.connect();
+	const client = new pg.Pool(config);
 
 	return client;
 };

reverse_engineering/helpers/postgresHelpers/tableHelper.js

@@ -290,12 +290,14 @@ const prepareTableLevelData = (tableLevelData, tableToastOptions) => {
 	const unlogged = tableLevelData?.relpersistence === 'u';
 	const storage_parameter = prepareStorageParameters(tableLevelData?.reloptions, tableToastOptions);
 	const table_tablespace_name = tableLevelData?.spcname;
+	const partitionBounds = tableLevelData.partition_expr;
 
 	return {
 		temporary,
 		unlogged,
 		storage_parameter,
 		table_tablespace_name,
+		partitionBounds,
 	};
 };
 

reverse_engineering/helpers/postgresService.js

@@ -142,15 +142,24 @@ module.exports = {
 		});
 	},
 
-	async retrieveEntitiesData(schemaName, entitiesNames, recordSamplingSettings) {
+	async retrieveEntitiesData(schemaName, entitiesNames, recordSamplingSettings, includePartitions = false) {
 		const userDefinedTypes = await this._retrieveUserDefinedTypes(schemaName);
 		const schemaOidResult = await db.queryTolerant(queryConstants.GET_NAMESPACE_OID, [schemaName], true);
 		const schemaOid = schemaOidResult?.oid;
+		const partitions = includePartitions ? await db.queryTolerant(queryConstants.GET_PARTITIONS, [schemaOid]) : [];
 
 		const [viewsNames, tablesNames] = _.partition(entitiesNames, isViewByName);
 
+		const allTablesList = tablesNames.flatMap(tableName => [
+			{ tableName },
+			..._.filter(partitions, { parent_name: tableName }).map(({ child_name, is_parent_partitioned }) => ({
+				isParentPartitioned: is_parent_partitioned,
+				tableName: child_name,
+			})),
+		]);
+
 		const tables = await mapPromises(
-			tablesNames,
+			_.uniq(allTablesList),
 			_.bind(
 				this._retrieveSingleTableData,
 				this,
@@ -230,7 +239,13 @@ module.exports = {
 		return getUserDefinedTypes(udtsWithColumns, domainTypesWithConstraints);
 	},
 
-	async _retrieveSingleTableData(recordSamplingSettings, schemaOid, schemaName, userDefinedTypes, tableName) {
+	async _retrieveSingleTableData(
+		recordSamplingSettings,
+		schemaOid,
+		schemaName,
+		userDefinedTypes,
+		{ tableName, isParentPartitioned },
+	) {
 		logger.progress('Get table data', schemaName, tableName);
 
 		const tableLevelData = await db.queryTolerant(
@@ -270,11 +285,11 @@ module.exports = {
 		const tableData = {
 			partitioning,
 			description,
-			inherits,
 			triggers,
 			Indxs: tableIndexes,
 			...tableLevelProperties,
 			...tableConstraint,
+			...(isParentPartitioned ? { partitionOf: _.first(inherits)?.parentTable } : { inherits }),
 		};
 
 		const entityLevel = clearEmptyPropertiesInObject(tableData);

reverse_engineering/helpers/queryConstants.js

@@ -97,13 +97,26 @@ const queryConstants = {
 	GET_VERSION_AS_NUM: 'SHOW server_version_num;',
 	GET_SCHEMA_NAMES: 'SELECT schema_name FROM information_schema.schemata;',
 	GET_TABLE_NAMES: `
-        SELECT table_name, table_type
-            FROM information_schema.tables
- 	        WHERE table_schema = $1
-	        ORDER BY table_name;`,
+        SELECT tables.table_name, tables.table_type FROM information_schema.tables AS tables
+        INNER JOIN 
+        (SELECT
+                pg_class.relname AS table_name,
+                 pg_namespace.nspname AS table_schema
+        FROM pg_catalog.pg_class AS pg_class
+        INNER JOIN pg_catalog.pg_namespace AS pg_namespace 
+                ON (pg_namespace.oid = pg_class.relnamespace)
+        WHERE pg_class.relispartition = false
+                AND pg_class.relkind = ANY('{"r","v","t","m","p"}'))
+        AS catalog_table_data
+        ON (catalog_table_data.table_name = tables.table_name AND catalog_table_data.table_schema = tables.table_schema)
+	LEFT JOIN (SELECT relname AS child_name FROM pg_catalog.pg_inherits AS inherit
+		LEFT JOIN pg_catalog.pg_class AS child ON (child.oid = inherit.inhrelid)) AS inherited_tables
+	ON (inherited_tables.child_name = tables.table_name)
+	WHERE inherited_tables.child_name IS NULL
+        AND tables.table_schema = $1;`,
 	GET_NAMESPACE_OID: 'SELECT oid FROM pg_catalog.pg_namespace WHERE nspname = $1',
 	GET_TABLE_LEVEL_DATA: `
-        SELECT pc.oid, pc.relpersistence, pc.reloptions, pt.spcname
+        SELECT pc.oid, pc.relpersistence, pc.reloptions, pt.spcname, pg_get_expr(pc.relpartbound, pc.oid) AS partition_expr
             FROM pg_catalog.pg_class AS pc 
             LEFT JOIN pg_catalog.pg_tablespace AS pt 
             ON pc.reltablespace = pt.oid
@@ -327,6 +340,15 @@ const queryConstants = {
                 description,
                 referenced_table_name,
                 referenced_table_schema;`,
+	GET_PARTITIONS: `
+        SELECT
+        	inher_child.relname AS child_name,
+        	inher_parent.relname AS parent_name,
+                CASE WHEN inher_parent.relkind = 'p' THEN TRUE ELSE FALSE END AS is_parent_partitioned
+        FROM pg_inherits
+        LEFT JOIN pg_class AS inher_child ON (inher_child.oid = pg_inherits.inhrelid)
+        LEFT JOIN pg_class AS inher_parent ON (inher_parent.oid = pg_inherits.inhparent)
+        WHERE inher_parent.relnamespace = $1;`,
 };
 
 const getQueryName = query => {