[Feature][Server] Add metadata lineage api (#423)

datavines-common/src/main/java/io/datavines/common/config/enums/SourceType.java

@@ -23,9 +23,9 @@
 public enum SourceType {
 
     /**
-     * 0 normal
-     * 1 invalidate items
-     * 2 actual value
+     * 0 source
+     * 1 target
+     * 2 metadata
      **/
     SOURCE(0, "source"),
     TARGET(1, "target"),

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorFactory.java

@@ -38,4 +38,8 @@ public interface ConnectorFactory {
     ConfigBuilder getConfigBuilder();
 
     DataSourceClient getDataSourceClient();
+
+    StatementSplitter getStatementSplitter();
+
+    StatementParser getStatementParser();
 }

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/LineageParser.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.connector.api;
+
+import io.datavines.common.utils.StringUtils;
+import io.datavines.connector.api.entity.ScriptMetadata;
+import io.datavines.connector.api.entity.StatementMetadata;
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.time.LocalDateTime;
+import java.util.List;
+
+public class LineageParser {
+
+    public static ScriptMetadata parseScript(String script, StatementSplitter statementSplitter, StatementParser   statementParser) {
+        if (StringUtils.isEmpty(script)) {
+            return null;
+        }
+
+        ScriptMetadata scriptMetadata = new ScriptMetadata();
+        scriptMetadata.setScript(script);
+        List<String> statements = statementSplitter.splitStatements(script);
+
+        if (CollectionUtils.isEmpty(statements)) {
+            return null;
+        }
+
+        for (int i=0; i<statements.size(); i++) {
+            StatementMetadata statementMetadata = new StatementMetadata();
+            statementMetadata.setStatementIndex(i);
+            statementMetadata.setStatementText(statements.get(i));
+            statementMetadata.setStatementParseStartTime(LocalDateTime.now());
+            statementMetadata.setStatementMetadataFragment(statementParser.parseStatement(statements.get(i)));
+            statementMetadata.setStatementParseEndTime(LocalDateTime.now());
+            scriptMetadata.addStatementMetadata(statementMetadata);
+        }
+
+        return scriptMetadata;
+    }
+
+}

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/StatementParser.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.connector.api;
+
+import io.datavines.connector.api.entity.StatementMetadataFragment;
+
+public interface StatementParser {
+
+    StatementMetadataFragment parseStatement(String statement);
+}

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/StatementSplitter.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.connector.api;
+
+import java.util.List;
+
+public interface StatementSplitter {
+
+    List<String> splitStatements(String body) ;
+}

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/ColumnLineage.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.connector.api.entity;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+public class ColumnLineage {
+
+    private List<String> inputColumns;
+
+    private List<String> outputColumns;
+}

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/MetaDataConstants.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.connector.api.entity;
+
+public class MetaDataConstants {
+
+    public final static String UNKNOWN_STATEMENT_TYPE = "UNKNOWN";
+}

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/ScriptMetadata.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.connector.api.entity;
+
+import lombok.Data;
+import org.apache.commons.collections4.CollectionUtils;
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+public class ScriptMetadata {
+
+    private String script;
+
+    private List<StatementMetadata> statementMetadataList;
+
+    public void addStatementMetadata(StatementMetadata statementMetadata) {
+        if (CollectionUtils.isEmpty(statementMetadataList)) {
+            statementMetadataList = new ArrayList<>();
+        }
+        statementMetadataList.add(statementMetadata);
+    }
+}

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/StatementMetadata.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.connector.api.entity;
+
+import lombok.Data;
+
+import java.time.LocalDateTime;
+
+@Data
+public class StatementMetadata {
+
+    private int statementIndex;
+
+    private String statementText;
+
+    private StatementMetadataFragment statementMetadataFragment;
+
+    private LocalDateTime statementParseStartTime;
+
+    private LocalDateTime statementParseEndTime;
+}

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/StatementMetadataFragment.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.connector.api.entity;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+public class StatementMetadataFragment {
+
+    private List<String> inputTables;
+
+    private List<String> outputTables;
+
+    private List<ColumnLineage> columnLineageList;
+}

datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java

@@ -64,4 +64,14 @@ public ConfigBuilder getConfigBuilder() {
     public DataSourceClient getDataSourceClient() {
         return null;
     }
+
+    @Override
+    public StatementSplitter getStatementSplitter() {
+        return null;
+    }
+
+    @Override
+    public StatementParser getStatementParser() {
+        return null;
+    }
 }

datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/AbstractJdbcConnectorFactory.java

@@ -44,4 +44,14 @@ public ConfigBuilder getConfigBuilder() {
     public DataSourceClient getDataSourceClient() {
         return new JdbcDataSourceClient();
     }
+
+    @Override
+    public StatementSplitter getStatementSplitter() {
+        return new DefaultStatementSplitter();
+    }
+
+    @Override
+    public StatementParser getStatementParser() {
+        return new DefaultStatementParser();
+    }
 }

datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementParser.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.connector.plugin;
+
+import io.datavines.connector.api.StatementParser;
+import io.datavines.connector.api.entity.StatementMetadataFragment;
+
+public class DefaultStatementParser implements StatementParser {
+
+    @Override
+    public StatementMetadataFragment parseStatement(String statement) {
+        return null;
+    }
+}

datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementSplitter.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.connector.plugin;
+
+import io.datavines.common.utils.StringUtils;
+import io.datavines.connector.api.StatementSplitter;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class DefaultStatementSplitter implements StatementSplitter {
+
+    @Override
+    public List<String> splitStatements(String body) {
+
+        String DELIMITER = ";";
+        return StringUtils.isEmpty(body) ? null : Arrays.asList(body.split(DELIMITER));
+    }
+}

datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnector.java

@@ -210,7 +210,7 @@ public ConnectorResponse testConnect(TestConnectionRequestParam param) {
 
     @Override
     public List<String> keyProperties() {
-        return Arrays.asList(HOST, PORT, DATABASE);
+        return Arrays.asList(HOST, PORT);
     }
 
     @Override