diff --git a/v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java b/v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java index 99c8e8c7f..62e6eda4c 100644 --- a/v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java +++ b/v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java @@ -31,6 +31,7 @@ import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_ENTITY_PROPERTY_GRAPH; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_FOREIGN_KEY; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_INDEX; +import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_INTERLEAVE_TYPE; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_LABEL; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_NAMED_SCHEMA; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_NODE_TABLE; @@ -68,6 +69,7 @@ import com.google.cloud.teleport.spanner.ddl.PropertyGraph.PropertyDeclaration; import com.google.cloud.teleport.spanner.ddl.Sequence; import com.google.cloud.teleport.spanner.ddl.Table; +import com.google.cloud.teleport.spanner.ddl.Table.InterleaveType; import com.google.cloud.teleport.spanner.ddl.View; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; @@ -613,6 +615,16 @@ public Table toTable(String tableName, Schema schema) { if (!Strings.isNullOrEmpty(spannerParent)) { table.interleaveInParent(spannerParent); + // Process the interleave type. + String spannerInterleaveType = schema.getProp(SPANNER_INTERLEAVE_TYPE); + if (!Strings.isNullOrEmpty(spannerInterleaveType)) { + table.interleaveType( + spannerInterleaveType.equals("IN") ? InterleaveType.IN : InterleaveType.IN_PARENT); + } else { + // Default to IN_PARENT for backwards compatibility with older exports. + table.interleaveType(InterleaveType.IN_PARENT); + } + // Process the on delete action. String onDeleteAction = schema.getProp(SPANNER_ON_DELETE_ACTION); if (onDeleteAction == null) { diff --git a/v1/src/main/java/com/google/cloud/teleport/spanner/AvroUtil.java b/v1/src/main/java/com/google/cloud/teleport/spanner/AvroUtil.java index 8e3782f86..5f01634a1 100644 --- a/v1/src/main/java/com/google/cloud/teleport/spanner/AvroUtil.java +++ b/v1/src/main/java/com/google/cloud/teleport/spanner/AvroUtil.java @@ -43,6 +43,7 @@ private AvroUtil() {} public static final String SPANNER_ON_DELETE_ACTION = "spannerOnDeleteAction"; public static final String SPANNER_OPTION = "spannerOption_"; public static final String SPANNER_PARENT = "spannerParent"; + public static final String SPANNER_INTERLEAVE_TYPE = "spannerInterleaveType"; public static final String SPANNER_PRIMARY_KEY = "spannerPrimaryKey"; public static final String SPANNER_REMOTE = "spannerRemote"; public static final String SPANNER_SEQUENCE_OPTION = "sequenceOption_"; diff --git a/v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java b/v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java index 15e0a1be0..2bbfeb2c7 100644 --- a/v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java +++ b/v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java @@ -33,6 +33,7 @@ import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_ENTITY_PROPERTY_GRAPH; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_FOREIGN_KEY; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_INDEX; +import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_INTERLEAVE_TYPE; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_LABEL; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_NAME; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_NAMED_SCHEMA; @@ -69,6 +70,7 @@ import com.google.cloud.teleport.spanner.ddl.PropertyGraph; import com.google.cloud.teleport.spanner.ddl.Sequence; import com.google.cloud.teleport.spanner.ddl.Table; +import com.google.cloud.teleport.spanner.ddl.Table.InterleaveType; import com.google.cloud.teleport.spanner.ddl.View; import com.google.common.collect.ImmutableList; import java.util.ArrayList; @@ -119,6 +121,9 @@ public Collection convert(Ddl ddl) { recordBuilder.prop(GOOGLE_STORAGE, "CloudSpanner"); if (table.interleaveInParent() != null) { recordBuilder.prop(SPANNER_PARENT, table.interleaveInParent()); + recordBuilder.prop( + SPANNER_INTERLEAVE_TYPE, + table.interleaveType() == InterleaveType.IN ? "IN" : "IN PARENT"); recordBuilder.prop( SPANNER_ON_DELETE_ACTION, table.onDeleteCascade() ? "cascade" : "no action"); } diff --git a/v1/src/main/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScanner.java b/v1/src/main/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScanner.java index 3fa757a27..c4224e31c 100644 --- a/v1/src/main/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScanner.java +++ b/v1/src/main/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScanner.java @@ -28,6 +28,7 @@ import com.google.cloud.spanner.ResultSet; import com.google.cloud.spanner.Statement; import com.google.cloud.teleport.spanner.ddl.ForeignKey.ReferentialAction; +import com.google.cloud.teleport.spanner.ddl.Table.InterleaveType; import com.google.cloud.teleport.spanner.proto.ExportProtos.Export; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; @@ -228,7 +229,7 @@ private void listTables(Ddl.Builder builder) { case GOOGLE_STANDARD_SQL: queryBuilder = Statement.newBuilder( - "SELECT t.table_schema, t.table_name, t.parent_table_name, t.on_delete_action FROM" + "SELECT t.table_schema, t.table_name, t.parent_table_name, t.interleave_type, t.on_delete_action FROM" + " information_schema.tables AS t" + " WHERE t.table_schema NOT IN" + " ('INFORMATION_SCHEMA', 'SPANNER_SYS')"); @@ -241,7 +242,7 @@ private void listTables(Ddl.Builder builder) { case POSTGRESQL: queryBuilder = Statement.newBuilder( - "SELECT t.table_schema, t.table_name, t.parent_table_name, t.on_delete_action FROM" + "SELECT t.table_schema, t.table_name, t.parent_table_name, t.interleave_type, t.on_delete_action FROM" + " information_schema.tables AS t" + " WHERE t.table_schema NOT IN " + "('information_schema', 'spanner_sys', 'pg_catalog')"); @@ -272,18 +273,39 @@ private void listTables(Ddl.Builder builder) { // Parent table and child table has to be in same schema. String parentTableName = resultSet.isNull(2) ? null : getQualifiedName(tableSchema, resultSet.getString(2)); - String onDeleteAction = resultSet.isNull(3) ? null : resultSet.getString(3); + String interleaveTypeStr = resultSet.isNull(3) ? null : resultSet.getString(3); + Table.InterleaveType interleaveType = null; + if (!Strings.isNullOrEmpty(interleaveTypeStr)) { + interleaveType = + interleaveTypeStr.equals("IN PARENT") ? InterleaveType.IN_PARENT : InterleaveType.IN; + } + String onDeleteAction = resultSet.isNull(4) ? null : resultSet.getString(4); + + boolean hasParentTable = !Strings.isNullOrEmpty(parentTableName); + boolean hasInterleaveType = !Strings.isNullOrEmpty(interleaveTypeStr); + boolean hasOnDeleteAction = !Strings.isNullOrEmpty(onDeleteAction); + + // If parent_table_name is set, then it is required that there also be an interleave_type. + // Conversely, if there is no parent, then there should also be no interleave_type. + if (hasParentTable != hasInterleaveType) { + throw new IllegalStateException( + String.format( + "Invalid combination of parentTableName %s and interleaveType %s", + parentTableName, interleaveTypeStr)); + } - // Error out when the parent table or on delete action are set incorrectly. - if (Strings.isNullOrEmpty(parentTableName) != Strings.isNullOrEmpty(onDeleteAction)) { + // If this table is interleaved with IN PARENT semantics, then an ON DELETE action is + // required. Conversely, if this table is interleaved with IN semantics or is not interleaved + // at all, then it is required that there not be an ON DELETE action. + if ((interleaveType == InterleaveType.IN_PARENT) == hasOnDeleteAction) { throw new IllegalStateException( String.format( - "Invalid combination of parentTableName %s and onDeleteAction %s", + "Invalid combination of IN PARENT %s and onDeleteAction %s", parentTableName, onDeleteAction)); } boolean onDeleteCascade = false; - if (onDeleteAction != null) { + if (hasOnDeleteAction) { if (onDeleteAction.equals("CASCADE")) { onDeleteCascade = true; } else if (!onDeleteAction.equals("NO ACTION")) { @@ -296,6 +318,7 @@ private void listTables(Ddl.Builder builder) { builder .createTable(tableName) .interleaveInParent(parentTableName) + .interleaveType(interleaveType) .onDeleteCascade(onDeleteCascade) .endTable(); } diff --git a/v1/src/main/java/com/google/cloud/teleport/spanner/ddl/Table.java b/v1/src/main/java/com/google/cloud/teleport/spanner/ddl/Table.java index 098c62246..d111c3f9b 100644 --- a/v1/src/main/java/com/google/cloud/teleport/spanner/ddl/Table.java +++ b/v1/src/main/java/com/google/cloud/teleport/spanner/ddl/Table.java @@ -39,6 +39,14 @@ public abstract class Table implements Serializable { @Nullable public abstract String interleaveInParent(); + public enum InterleaveType { + IN, + IN_PARENT + }; + + @Nullable + public abstract InterleaveType interleaveType(); + public abstract ImmutableList primaryKeys(); public abstract boolean onDeleteCascade(); @@ -117,7 +125,8 @@ private void prettyPrintPg( appendable.append("\n)"); if (interleaveInParent() != null) { appendable - .append(" \nINTERLEAVE IN PARENT ") + .append(" \nINTERLEAVE IN ") + .append(interleaveType() == InterleaveType.IN ? "" : "PARENT ") .append(quoteIdentifier(interleaveInParent(), dialect())); if (onDeleteCascade()) { appendable.append(" ON DELETE CASCADE"); @@ -156,7 +165,8 @@ private void prettyPrintGsql( appendable.append(")"); if (interleaveInParent() != null) { appendable - .append(",\nINTERLEAVE IN PARENT ") + .append(",\nINTERLEAVE IN ") + .append(interleaveType() == InterleaveType.IN ? " " : "PARENT ") .append(quoteIdentifier(interleaveInParent(), dialect())); if (onDeleteCascade()) { appendable.append(" ON DELETE CASCADE"); @@ -206,6 +216,8 @@ Builder ddlBuilder(Ddl.Builder ddlBuilder) { public abstract Builder interleaveInParent(String parent); + public abstract Builder interleaveType(Table.InterleaveType type); + abstract Builder primaryKeys(ImmutableList value); abstract Builder onDeleteCascade(boolean onDeleteCascade); diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java b/v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java index 2ebd794ce..de20aadf7 100644 --- a/v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java +++ b/v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java @@ -1311,4 +1311,256 @@ public void pgPlacementTable() { + "\"location\" character varying NOT NULL PLACEMENT KEY,\n\t" + "PRIMARY KEY (\"id\")\n)\n\n")); } + + @Test + public void interleaveTable() { + String parentTable = + "{" + + " \"type\" : \"record\"," + + " \"name\" : \"ParentTable\"," + + " \"namespace\" : \"spannertest\"," + + " \"fields\" : [ {" + + " \"name\" : \"k1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"STRING(MAX)\"," + + " \"notNull\" : \"true\"" + + " }, {" + + " \"name\" : \"v1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"STRING(MAX)\"," + + " \"notNull\" : \"true\"" + + " }]," + + " \"googleStorage\" : \"CloudSpanner\"," + + " \"googleFormatVersion\" : \"booleans\"," + + " \"spannerPrimaryKey_0\" : \"`k1` ASC\"" + + "}"; + + // Confirming backwards compatibility, so that an avro schema with no interleave type is treated + // as IN PARENT by default. + String interleaveInParentTableNoInterleaveType = + "{" + + " \"type\" : \"record\"," + + " \"name\" : \"interleaveInParentTableNoInterleaveType\"," + + " \"namespace\" : \"spannertest\"," + + " \"fields\" : [ {" + + " \"name\" : \"k1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"STRING(MAX)\"," + + " \"notNull\" : \"true\"" + + " }, {" + + " \"name\" : \"v1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"STRING(MAX)\"," + + " \"notNull\" : \"true\"" + + " }]," + + " \"googleStorage\" : \"CloudSpanner\"," + + " \"spannerParent\" : \"ParentTable\"," + + " \"googleFormatVersion\" : \"booleans\"," + + " \"spannerPrimaryKey_0\" : \"`k1` ASC\"" + + "}"; + + String interleaveInParentTable = + "{" + + " \"type\" : \"record\"," + + " \"name\" : \"interleaveInParentTable\"," + + " \"namespace\" : \"spannertest\"," + + " \"fields\" : [ {" + + " \"name\" : \"k1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"STRING(MAX)\"," + + " \"notNull\" : \"true\"" + + " }, {" + + " \"name\" : \"v1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"STRING(MAX)\"," + + " \"notNull\" : \"true\"" + + " }]," + + " \"googleStorage\" : \"CloudSpanner\"," + + " \"spannerParent\" : \"ParentTable\"," + + " \"spannerInterleaveType\" : \"IN PARENT\"," + + " \"googleFormatVersion\" : \"booleans\"," + + " \"spannerPrimaryKey_0\" : \"`k1` ASC\"" + + "}"; + + String interleaveInTable = + "{" + + " \"type\" : \"record\"," + + " \"name\" : \"interleaveInTable\"," + + " \"namespace\" : \"spannertest\"," + + " \"fields\" : [ {" + + " \"name\" : \"k1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"STRING(MAX)\"," + + " \"notNull\" : \"true\"" + + " }, {" + + " \"name\" : \"v1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"STRING(MAX)\"," + + " \"notNull\" : \"true\"" + + " }]," + + " \"googleStorage\" : \"CloudSpanner\"," + + " \"spannerParent\" : \"ParentTable\"," + + " \"spannerInterleaveType\" : \"IN\"," + + " \"googleFormatVersion\" : \"booleans\"," + + " \"spannerPrimaryKey_0\" : \"`k1` ASC\"" + + "}"; + + Collection schemas = new ArrayList<>(); + Schema.Parser parser = new Schema.Parser(); + schemas.add(parser.parse(parentTable)); + schemas.add(parser.parse(interleaveInParentTableNoInterleaveType)); + schemas.add(parser.parse(interleaveInParentTable)); + schemas.add(parser.parse(interleaveInTable)); + + AvroSchemaToDdlConverter converter = new AvroSchemaToDdlConverter(); + Ddl ddl = converter.toDdl(schemas); + assertThat(ddl.allTables(), hasSize(4)); + assertThat( + ddl.prettyPrint(), + equalToCompressingWhiteSpace( + "CREATE TABLE `ParentTable` (\n\t" + + "`k1` STRING(MAX) NOT NULL,\n\t" + + "`v1` STRING(MAX) NOT NULL,\n" + + ") PRIMARY KEY (`k1` ASC)\n\n\n" + + "CREATE TABLE `interleaveInParentTable` (\n\t" + + "`k1` STRING(MAX) NOT NULL,\n\t" + + "`v1` STRING(MAX) NOT NULL,\n" + + ") PRIMARY KEY (`k1` ASC),\nINTERLEAVE IN PARENT `ParentTable`\n\n" + + "CREATE TABLE `interleaveInParentTableNoInterleaveType` (\n\t" + + "`k1` STRING(MAX) NOT NULL,\n\t" + + "`v1` STRING(MAX) NOT NULL,\n" + + ") PRIMARY KEY (`k1` ASC),\nINTERLEAVE IN PARENT `ParentTable`\n\n" + + "CREATE TABLE `interleaveInTable` (\n\t" + + "`k1` STRING(MAX) NOT NULL,\n\t" + + "`v1` STRING(MAX) NOT NULL,\n" + + ") PRIMARY KEY (`k1` ASC),\nINTERLEAVE IN `ParentTable`\n\n")); + } + + @Test + public void pgInterleaveTable() { + String parentTable = + "{" + + " \"type\" : \"record\"," + + " \"name\" : \"ParentTable\"," + + " \"namespace\" : \"spannertest\"," + + " \"fields\" : [ {" + + " \"name\" : \"k1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"character varying\"," + + " \"notNull\" : \"true\"" + + " }, {" + + " \"name\" : \"v1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"character varying\"," + + " \"notNull\" : \"true\"" + + " }]," + + " \"googleStorage\" : \"CloudSpanner\"," + + " \"googleFormatVersion\" : \"booleans\"," + + " \"spannerPrimaryKey_0\" : \"k1 ASC\"" + + "}"; + + // Confirming backwards compatibility, so that an avro schema with no interleave type is treated + // as IN PARENT by default. + String interleaveInParentTableNoInterleaveType = + "{" + + " \"type\" : \"record\"," + + " \"name\" : \"interleaveInParentTableNoInterleaveType\"," + + " \"namespace\" : \"spannertest\"," + + " \"fields\" : [ {" + + " \"name\" : \"k1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"character varying\"," + + " \"notNull\" : \"true\"" + + " }, {" + + " \"name\" : \"v1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"character varying\"," + + " \"notNull\" : \"true\"" + + " }]," + + " \"googleStorage\" : \"CloudSpanner\"," + + " \"spannerParent\" : \"ParentTable\"," + + " \"googleFormatVersion\" : \"booleans\"," + + " \"spannerPrimaryKey_0\" : \"k1 ASC\"" + + "}"; + + String interleaveInParentTable = + "{" + + " \"type\" : \"record\"," + + " \"name\" : \"interleaveInParentTable\"," + + " \"namespace\" : \"spannertest\"," + + " \"fields\" : [ {" + + " \"name\" : \"k1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"character varying\"," + + " \"notNull\" : \"true\"" + + " }, {" + + " \"name\" : \"v1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"character varying\"," + + " \"notNull\" : \"true\"" + + " }]," + + " \"googleStorage\" : \"CloudSpanner\"," + + " \"spannerParent\" : \"ParentTable\"," + + " \"spannerInterleaveType\" : \"IN PARENT\"," + + " \"googleFormatVersion\" : \"booleans\"," + + " \"spannerPrimaryKey_0\" : \"k1 ASC\"" + + "}"; + + String interleaveInTable = + "{" + + " \"type\" : \"record\"," + + " \"name\" : \"interleaveInTable\"," + + " \"namespace\" : \"spannertest\"," + + " \"fields\" : [ {" + + " \"name\" : \"k1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"character varying\"," + + " \"notNull\" : \"true\"" + + " }, {" + + " \"name\" : \"v1\"," + + " \"type\" : \"string\"," + + " \"sqlType\" : \"character varying\"," + + " \"notNull\" : \"true\"" + + " }]," + + " \"googleStorage\" : \"CloudSpanner\"," + + " \"spannerParent\" : \"ParentTable\"," + + " \"spannerInterleaveType\" : \"IN\"," + + " \"googleFormatVersion\" : \"booleans\"," + + " \"spannerPrimaryKey_0\" : \"k1 ASC\"" + + "}"; + + Collection schemas = new ArrayList<>(); + Schema.Parser parser = new Schema.Parser(); + schemas.add(parser.parse(parentTable)); + schemas.add(parser.parse(interleaveInParentTableNoInterleaveType)); + schemas.add(parser.parse(interleaveInParentTable)); + schemas.add(parser.parse(interleaveInTable)); + + AvroSchemaToDdlConverter converter = new AvroSchemaToDdlConverter(Dialect.POSTGRESQL); + Ddl ddl = converter.toDdl(schemas); + assertThat(ddl.allTables(), hasSize(4)); + assertThat( + ddl.prettyPrint(), + equalToCompressingWhiteSpace( + "CREATE TABLE \"ParentTable\" (\n\t" + + "\"k1\" character varying NOT NULL,\n\t" + + "\"v1\" character varying NOT NULL,\n\t" + + "PRIMARY KEY (\"k1\")\n" + + ")\n\n\n" + + "CREATE TABLE \"interleaveInParentTable\" (\n\t" + + "\"k1\" character varying NOT NULL,\n\t" + + "\"v1\" character varying NOT NULL,\n\t" + + "PRIMARY KEY (\"k1\")\n" + + ") \nINTERLEAVE IN PARENT \"ParentTable\"\n\n" + + "CREATE TABLE \"interleaveInParentTableNoInterleaveType\" (\n\t" + + "\"k1\" character varying NOT NULL,\n\t" + + "\"v1\" character varying NOT NULL,\n\t" + + "PRIMARY KEY (\"k1\")\n" + + ") \nINTERLEAVE IN PARENT \"ParentTable\"\n\n" + + "CREATE TABLE \"interleaveInTable\" (\n\t" + + "\"k1\" character varying NOT NULL,\n\t" + + "\"v1\" character varying NOT NULL,\n\t" + + "PRIMARY KEY (\"k1\")\n" + + ") \nINTERLEAVE IN \"ParentTable\"\n\n")); + } } diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverterTest.java b/v1/src/test/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverterTest.java index a421555a5..213ad1900 100644 --- a/v1/src/test/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverterTest.java +++ b/v1/src/test/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverterTest.java @@ -33,6 +33,7 @@ import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_ENTITY_PROPERTY_GRAPH; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_FOREIGN_KEY; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_INDEX; +import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_INTERLEAVE_TYPE; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_LABEL; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_NODE_TABLE; import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_ON_DELETE_ACTION; @@ -68,6 +69,7 @@ import com.google.cloud.teleport.spanner.ddl.GraphElementTable.LabelToPropertyDefinitions; import com.google.cloud.teleport.spanner.ddl.GraphElementTable.PropertyDefinition; import com.google.cloud.teleport.spanner.ddl.PropertyGraph; +import com.google.cloud.teleport.spanner.ddl.Table.InterleaveType; import com.google.cloud.teleport.spanner.ddl.View; import com.google.common.collect.ImmutableList; import java.util.Arrays; @@ -891,6 +893,7 @@ public void allTypes() { assertThat(avroSchema.getProp(SPANNER_PRIMARY_KEY + "_0"), equalTo("`bool_field` ASC")); assertThat(avroSchema.getProp(SPANNER_PARENT), equalTo("ParentTable")); assertThat(avroSchema.getProp(SPANNER_ON_DELETE_ACTION), equalTo("cascade")); + assertThat(avroSchema.getProp(SPANNER_INTERLEAVE_TYPE), equalTo("IN PARENT")); System.out.println(avroSchema.toString(true)); } @@ -1060,6 +1063,7 @@ public void pgAllTypes() { assertThat(avroSchema.getProp(SPANNER_PRIMARY_KEY + "_0"), equalTo("\"bool_field\" ASC")); assertThat(avroSchema.getProp(SPANNER_PARENT), equalTo("ParentTable")); assertThat(avroSchema.getProp(SPANNER_ON_DELETE_ACTION), equalTo("cascade")); + assertThat(avroSchema.getProp(SPANNER_INTERLEAVE_TYPE), equalTo("IN PARENT")); } @Test @@ -1884,6 +1888,106 @@ public void pgPlacementTable() { assertThat(fields.get(1).getProp(SPANNER_PLACEMENT_KEY), equalTo(null)); } + @Test + public void interleaveInTable() { + DdlToAvroSchemaConverter converter = + new DdlToAvroSchemaConverter("spannertest", "booleans", false); + Ddl ddl = + Ddl.builder() + .createTable("InterleaveInTable") + .column("k1") + .type(Type.string()) + .max() + .notNull() + .endColumn() + .column("v1") + .type(Type.string()) + .size(10) + .endColumn() + .interleaveInParent("ParentTable") + .interleaveType(InterleaveType.IN) + .endTable() + .build(); + + Collection result = converter.convert(ddl); + assertThat(result, hasSize(1)); + Schema avroSchema = result.iterator().next(); + + assertThat(avroSchema.getNamespace(), equalTo("spannertest")); + assertThat(avroSchema.getProp(GOOGLE_FORMAT_VERSION), equalTo("booleans")); + assertThat(avroSchema.getProp(GOOGLE_STORAGE), equalTo("CloudSpanner")); + + assertThat(avroSchema.getName(), equalTo("InterleaveInTable")); + + List fields = avroSchema.getFields(); + + assertThat(fields, hasSize(2)); + + // k1 + assertThat(fields.get(0).name(), equalTo("k1")); + assertThat(fields.get(0).schema().getType(), equalTo(Schema.Type.STRING)); + assertThat(fields.get(0).getProp(SQL_TYPE), equalTo("STRING(MAX)")); + + // v1 + assertThat(fields.get(1).name(), equalTo("v1")); + assertThat(fields.get(1).schema(), equalTo(nullableUnion(Schema.Type.STRING))); + assertThat(fields.get(1).getProp(SQL_TYPE), equalTo("STRING(10)")); + + assertThat(avroSchema.getProp(SPANNER_PARENT), equalTo("ParentTable")); + assertThat(avroSchema.getProp(SPANNER_ON_DELETE_ACTION), equalTo("no action")); + assertThat(avroSchema.getProp(SPANNER_INTERLEAVE_TYPE), equalTo("IN")); + } + + @Test + public void pgInterleaveInTable() { + DdlToAvroSchemaConverter converter = + new DdlToAvroSchemaConverter("spannertest", "booleans", false); + Ddl ddl = + Ddl.builder(Dialect.POSTGRESQL) + .createTable("InterleaveInTable") + .column("k1") + .type(Type.string()) + .max() + .notNull() + .endColumn() + .column("v1") + .type(Type.string()) + .size(10) + .endColumn() + .interleaveInParent("ParentTable") + .interleaveType(InterleaveType.IN) + .endTable() + .build(); + + Collection result = converter.convert(ddl); + assertThat(result, hasSize(1)); + Schema avroSchema = result.iterator().next(); + + assertThat(avroSchema.getNamespace(), equalTo("spannertest")); + assertThat(avroSchema.getProp(GOOGLE_FORMAT_VERSION), equalTo("booleans")); + assertThat(avroSchema.getProp(GOOGLE_STORAGE), equalTo("CloudSpanner")); + + assertThat(avroSchema.getName(), equalTo("InterleaveInTable")); + + List fields = avroSchema.getFields(); + + assertThat(fields, hasSize(2)); + + // k1 + assertThat(fields.get(0).name(), equalTo("k1")); + assertThat(fields.get(0).schema().getType(), equalTo(Schema.Type.STRING)); + assertThat(fields.get(0).getProp(SQL_TYPE), equalTo("STRING(MAX)")); + + // v1 + assertThat(fields.get(1).name(), equalTo("v1")); + assertThat(fields.get(1).schema(), equalTo(nullableUnion(Schema.Type.STRING))); + assertThat(fields.get(1).getProp(SQL_TYPE), equalTo("STRING(10)")); + + assertThat(avroSchema.getProp(SPANNER_PARENT), equalTo("ParentTable")); + assertThat(avroSchema.getProp(SPANNER_ON_DELETE_ACTION), equalTo("no action")); + assertThat(avroSchema.getProp(SPANNER_INTERLEAVE_TYPE), equalTo("IN")); + } + private Schema nullableUnion(Schema.Type s) { return Schema.createUnion(Schema.create(Schema.Type.NULL), Schema.create(s)); } diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/RandomDdlGenerator.java b/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/RandomDdlGenerator.java index 0f4de77bc..547236d9b 100644 --- a/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/RandomDdlGenerator.java +++ b/v1/src/test/java/com/google/cloud/teleport/spanner/ddl/RandomDdlGenerator.java @@ -19,6 +19,7 @@ import com.google.cloud.spanner.Dialect; import com.google.cloud.teleport.spanner.common.Type; import com.google.cloud.teleport.spanner.ddl.ForeignKey.ReferentialAction; +import com.google.cloud.teleport.spanner.ddl.Table.InterleaveType; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; @@ -319,9 +320,14 @@ private void generateTable(Ddl.Builder builder, Table parent, int level) { String name = generateIdentifier(getMaxIdLength()); Table.Builder tableBuilder = builder.createTable(name); + Random rnd = getRandom(); int pkSize = 0; if (parent != null) { tableBuilder.interleaveInParent(parent.name()); + tableBuilder.interleaveType( + getDialect() == Dialect.GOOGLE_STANDARD_SQL && rnd.nextBoolean() + ? InterleaveType.IN + : InterleaveType.IN_PARENT); for (IndexColumn pk : parent.primaryKeys()) { Column pkColumn = parent.column(pk.name()); tableBuilder.addColumn(pkColumn); @@ -330,7 +336,6 @@ private void generateTable(Ddl.Builder builder, Table parent, int level) { } } - Random rnd = getRandom(); int numPks = Math.min(1 + rnd.nextInt(getMaxPkComponents()), MAX_PKS - pkSize); for (int i = 0; i < numPks; i++) { Column pkColumn =