Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Spanner Import/Export INTERLEAVE IN #2128

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Spanner Import/Export INTERLEAVE IN #2128

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b67b6e1
Changes to write read interleave type from information schema, and wr…
jjfox15 Jan 12, 2025
59b51c3
Fix warnings, bug in InformationSchemaScanner, and bug in AvroSchemaT…
jjfox15 Jan 13, 2025
1ca7db0
Set interleaveType in InfoSchemaScanner
jjfox15 Jan 15, 2025
e0e4c5c
Style fixes, and only generate INTERLEAVE IN ddl for gsql.
jjfox15 Jan 22, 2025
0558709
another style fix - remove unused import
jjfox15 Jan 23, 2025
7ced952
Make conditions more readable and add comments
jjfox15 Jan 23, 2025
c7e3e23
Tests
jjfox15 Jan 28, 2025
13fc5d4
Merge branch 'GoogleCloudPlatform:main' into main
jjfox15 Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_ENTITY_PROPERTY_GRAPH;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_FOREIGN_KEY;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_INDEX;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_INTERLEAVE_TYPE;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_LABEL;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_NAMED_SCHEMA;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_NODE_TABLE;
Expand Down Expand Up @@ -68,6 +69,7 @@
import com.google.cloud.teleport.spanner.ddl.PropertyGraph.PropertyDeclaration;
import com.google.cloud.teleport.spanner.ddl.Sequence;
import com.google.cloud.teleport.spanner.ddl.Table;
import com.google.cloud.teleport.spanner.ddl.Table.InterleaveType;
import com.google.cloud.teleport.spanner.ddl.View;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
Expand Down Expand Up @@ -613,6 +615,16 @@ public Table toTable(String tableName, Schema schema) {
if (!Strings.isNullOrEmpty(spannerParent)) {
table.interleaveInParent(spannerParent);

// Process the interleave type.
String spannerInterleaveType = schema.getProp(SPANNER_INTERLEAVE_TYPE);
if (!Strings.isNullOrEmpty(spannerInterleaveType)) {
table.interleaveType(
spannerInterleaveType.equals("IN") ? InterleaveType.IN : InterleaveType.IN_PARENT);
} else {
// Default to IN_PARENT for backwards compatibility with older exports.
table.interleaveType(InterleaveType.IN_PARENT);
}

// Process the on delete action.
String onDeleteAction = schema.getProp(SPANNER_ON_DELETE_ACTION);
if (onDeleteAction == null) {
Expand Down
1 change: 1 addition & 0 deletions v1/src/main/java/com/google/cloud/teleport/spanner/AvroUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ private AvroUtil() {}
public static final String SPANNER_ON_DELETE_ACTION = "spannerOnDeleteAction";
public static final String SPANNER_OPTION = "spannerOption_";
public static final String SPANNER_PARENT = "spannerParent";
public static final String SPANNER_INTERLEAVE_TYPE = "spannerInterleaveType";
public static final String SPANNER_PRIMARY_KEY = "spannerPrimaryKey";
public static final String SPANNER_REMOTE = "spannerRemote";
public static final String SPANNER_SEQUENCE_OPTION = "sequenceOption_";
Expand Down
5 changes: 5 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_ENTITY_PROPERTY_GRAPH;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_FOREIGN_KEY;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_INDEX;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_INTERLEAVE_TYPE;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_LABEL;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_NAME;
import static com.google.cloud.teleport.spanner.AvroUtil.SPANNER_NAMED_SCHEMA;
Expand Down Expand Up @@ -69,6 +70,7 @@
import com.google.cloud.teleport.spanner.ddl.PropertyGraph;
import com.google.cloud.teleport.spanner.ddl.Sequence;
import com.google.cloud.teleport.spanner.ddl.Table;
import com.google.cloud.teleport.spanner.ddl.Table.InterleaveType;
import com.google.cloud.teleport.spanner.ddl.View;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
Expand Down Expand Up @@ -119,6 +121,9 @@ public Collection<Schema> convert(Ddl ddl) {
recordBuilder.prop(GOOGLE_STORAGE, "CloudSpanner");
if (table.interleaveInParent() != null) {
recordBuilder.prop(SPANNER_PARENT, table.interleaveInParent());
recordBuilder.prop(
SPANNER_INTERLEAVE_TYPE,
table.interleaveType() == InterleaveType.IN ? "IN" : "IN PARENT");
recordBuilder.prop(
SPANNER_ON_DELETE_ACTION, table.onDeleteCascade() ? "cascade" : "no action");
darshan-sj marked this conversation as resolved.
Show resolved Hide resolved
}
Expand Down
25 changes: 20 additions & 5 deletions v1/src/main/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.google.cloud.spanner.ResultSet;
import com.google.cloud.spanner.Statement;
import com.google.cloud.teleport.spanner.ddl.ForeignKey.ReferentialAction;
import com.google.cloud.teleport.spanner.ddl.Table.InterleaveType;
import com.google.cloud.teleport.spanner.proto.ExportProtos.Export;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
Expand Down Expand Up @@ -228,7 +229,7 @@ private void listTables(Ddl.Builder builder) {
case GOOGLE_STANDARD_SQL:
queryBuilder =
Statement.newBuilder(
"SELECT t.table_schema, t.table_name, t.parent_table_name, t.on_delete_action FROM"
"SELECT t.table_schema, t.table_name, t.parent_table_name, t.interleave_type, t.on_delete_action FROM"
+ " information_schema.tables AS t"
+ " WHERE t.table_schema NOT IN"
+ " ('INFORMATION_SCHEMA', 'SPANNER_SYS')");
Expand All @@ -241,7 +242,7 @@ private void listTables(Ddl.Builder builder) {
case POSTGRESQL:
queryBuilder =
Statement.newBuilder(
"SELECT t.table_schema, t.table_name, t.parent_table_name, t.on_delete_action FROM"
"SELECT t.table_schema, t.table_name, t.parent_table_name, t.interleave_type, t.on_delete_action FROM"
darshan-sj marked this conversation as resolved.
Show resolved Hide resolved
+ " information_schema.tables AS t"
+ " WHERE t.table_schema NOT IN "
+ "('information_schema', 'spanner_sys', 'pg_catalog')");
Expand Down Expand Up @@ -272,13 +273,26 @@ private void listTables(Ddl.Builder builder) {
// Parent table and child table has to be in same schema.
String parentTableName =
resultSet.isNull(2) ? null : getQualifiedName(tableSchema, resultSet.getString(2));
String onDeleteAction = resultSet.isNull(3) ? null : resultSet.getString(3);
String interleaveTypeStr = resultSet.isNull(3) ? null : resultSet.getString(3);
Table.InterleaveType interleaveType = null;
if (!Strings.isNullOrEmpty(interleaveTypeStr)) {
interleaveType =
interleaveTypeStr.equals("IN PARENT") ? InterleaveType.IN_PARENT : InterleaveType.IN;
darshan-sj marked this conversation as resolved.
Show resolved Hide resolved
}
String onDeleteAction = resultSet.isNull(4) ? null : resultSet.getString(4);

// Error out when the parent table or on delete action are set incorrectly.
if (Strings.isNullOrEmpty(parentTableName) != Strings.isNullOrEmpty(onDeleteAction)) {
if (Strings.isNullOrEmpty(parentTableName) != Strings.isNullOrEmpty(interleaveTypeStr)) {
darshan-sj marked this conversation as resolved.
Show resolved Hide resolved
throw new IllegalStateException(
String.format(
"Invalid combination of parentTableName %s and interleaveType %s",
parentTableName, interleaveType));
}

if ((interleaveType == InterleaveType.IN_PARENT) == Strings.isNullOrEmpty(onDeleteAction)) {
darshan-sj marked this conversation as resolved.
Show resolved Hide resolved
throw new IllegalStateException(
String.format(
"Invalid combination of parentTableName %s and onDeleteAction %s",
"Invalid combination of IN PARENT %s and onDeleteAction %s",
parentTableName, onDeleteAction));
}

Expand All @@ -296,6 +310,7 @@ private void listTables(Ddl.Builder builder) {
builder
.createTable(tableName)
.interleaveInParent(parentTableName)
.interleaveType(interleaveType)
.onDeleteCascade(onDeleteCascade)
.endTable();
}
Expand Down
17 changes: 15 additions & 2 deletions v1/src/main/java/com/google/cloud/teleport/spanner/ddl/Table.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import com.google.auto.value.AutoValue;
import com.google.cloud.spanner.Dialect;
import com.google.cloud.teleport.spanner.ddl.Table.InterleaveType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import java.io.IOException;
Expand All @@ -39,6 +40,14 @@ public abstract class Table implements Serializable {
@Nullable
public abstract String interleaveInParent();

public enum InterleaveType {
IN,
IN_PARENT
};

@Nullable
public abstract InterleaveType interleaveType();

public abstract ImmutableList<IndexColumn> primaryKeys();

public abstract boolean onDeleteCascade();
Expand Down Expand Up @@ -117,7 +126,8 @@ private void prettyPrintPg(
appendable.append("\n)");
if (interleaveInParent() != null) {
appendable
.append(" \nINTERLEAVE IN PARENT ")
.append(" \nINTERLEAVE IN ")
.append(interleaveType() == InterleaveType.IN ? "" : "PARENT")
.append(quoteIdentifier(interleaveInParent(), dialect()));
if (onDeleteCascade()) {
appendable.append(" ON DELETE CASCADE");
Expand Down Expand Up @@ -156,7 +166,8 @@ private void prettyPrintGsql(
appendable.append(")");
if (interleaveInParent() != null) {
appendable
.append(",\nINTERLEAVE IN PARENT ")
.append(",\nINTERLEAVE IN ")
.append(interleaveType() == InterleaveType.IN ? " " : "PARENT")
.append(quoteIdentifier(interleaveInParent(), dialect()));
if (onDeleteCascade()) {
appendable.append(" ON DELETE CASCADE");
Expand Down Expand Up @@ -206,6 +217,8 @@ Builder ddlBuilder(Ddl.Builder ddlBuilder) {

public abstract Builder interleaveInParent(String parent);

public abstract Builder interleaveType(Table.InterleaveType type);

abstract Builder primaryKeys(ImmutableList<IndexColumn> value);

abstract Builder onDeleteCascade(boolean onDeleteCascade);
Expand Down
7 changes: 6 additions & 1 deletion v1/src/test/java/com/google/cloud/teleport/spanner/ddl/RandomDdlGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.google.cloud.spanner.Dialect;
import com.google.cloud.teleport.spanner.common.Type;
import com.google.cloud.teleport.spanner.ddl.ForeignKey.ReferentialAction;
import com.google.cloud.teleport.spanner.ddl.Table.InterleaveType;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
Expand Down Expand Up @@ -319,9 +320,14 @@ private void generateTable(Ddl.Builder builder, Table parent, int level) {
String name = generateIdentifier(getMaxIdLength());
Table.Builder tableBuilder = builder.createTable(name);

Random rnd = getRandom();
int pkSize = 0;
if (parent != null) {
tableBuilder.interleaveInParent(parent.name());
tableBuilder.interleaveType(
getDialect() == Dialect.GOOGLE_STANDARD_SQL && rnd.nextBoolean()
? InterleaveType.IN
: InterleaveType.IN_PARENT);
for (IndexColumn pk : parent.primaryKeys()) {
Column pkColumn = parent.column(pk.name());
tableBuilder.addColumn(pkColumn);
Expand All @@ -330,7 +336,6 @@ private void generateTable(Ddl.Builder builder, Table parent, int level) {
}
}

Random rnd = getRandom();
int numPks = Math.min(1 + rnd.nextInt(getMaxPkComponents()), MAX_PKS - pkSize);
for (int i = 0; i < numPks; i++) {
Column pkColumn =
Expand Down
Loading