[Feature][Metadata] Add get column lineage api (#439)

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/LineageParser.java

@@ -26,7 +26,7 @@
 
 public class LineageParser {
 
-    public static ScriptMetadata parseScript(String script, StatementSplitter statementSplitter, StatementParser   statementParser) {
+    public static ScriptMetadata parseScript(String script, StatementSplitter statementSplitter, StatementParser  statementParser) {
         if (StringUtils.isEmpty(script)) {
             return null;
         }

datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/StatementSplitter.java

@@ -20,5 +20,5 @@
 
 public interface StatementSplitter {
 
-    List<String> splitStatements(String body) ;
+    List<String> splitStatements(String body);
 }

datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/java/io/datavines/metric/result/formula/DiffPercentage.java

@@ -38,8 +38,8 @@ public String getZhName() {
     @Override
     public BigDecimal getResult(BigDecimal actualValue, BigDecimal expectedValue) {
         BigDecimal result = BigDecimal.valueOf(0);
-        if (expectedValue != null && expectedValue.compareTo(BigDecimal.ZERO) != 0) {
-            BigDecimal quotient = actualValue.divide(expectedValue, 2, RoundingMode.HALF_UP);
+        if (expectedValue != null && expectedValue.compareTo(BigDecimal.ZERO) != 0 && actualValue != null) {
+            BigDecimal quotient = actualValue.subtract(expectedValue).abs();
             result = quotient.divide(expectedValue, 2, RoundingMode.HALF_UP).multiply(new BigDecimal(100));
         }
 

datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/java/io/datavines/notification/plugin/dingtalk/DingTalkSender.java

@@ -108,7 +108,7 @@ public SlaNotificationResultRecord sendCardMsg(Set<ReceiverConfig> receiverSet,
     private String generateMsgJson(String title, String content,ReceiverConfig receiverConfig) {
 
         final String atMobiles = receiverConfig.getAtMobiles();
-        final String atUserIds = receiverConfig.getAtUserIds();
+        final String atDingtalkIds = receiverConfig.getAtDingtalkIds();
         final Boolean isAtAll = receiverConfig.getIsAtAll();
 
         if (org.apache.commons.lang3.StringUtils.isBlank(msgType)) {
@@ -133,8 +133,8 @@ private String generateMsgJson(String title, String content,ReceiverConfig recei
                     builder.append(" ");
                 });
             }
-            if (org.apache.commons.lang3.StringUtils.isNotBlank(atUserIds)) {
-                Arrays.stream(atUserIds.split(",")).forEach(value -> {
+            if (org.apache.commons.lang3.StringUtils.isNotBlank(atDingtalkIds)) {
+                Arrays.stream(atDingtalkIds.split(",")).forEach(value -> {
                     builder.append("@");
                     builder.append(value);
                     builder.append(" ");
@@ -163,10 +163,10 @@ private String generateMsgJson(String title, String content,ReceiverConfig recei
                 org.apache.commons.lang3.StringUtils.isNotBlank(atMobiles) ? atMobiles.split(",")
                         : new String[0];
         String[] atUserArray =
-                org.apache.commons.lang3.StringUtils.isNotBlank(atUserIds) ? atUserIds.split(",")
+                org.apache.commons.lang3.StringUtils.isNotBlank(atDingtalkIds) ? atDingtalkIds.split(",")
                         : new String[0];
         at.put("atMobiles", atMobileArray);
-        at.put("atUserIds", atUserArray);
+        at.put("atDingtalkIds", atUserArray);
         at.put("isAtAll", isAtAll);
 
         items.put("at", at);

datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/java/io/datavines/notification/plugin/dingtalk/DingTalkSlasHandlerPlugin.java

@@ -107,10 +107,10 @@ public String getConfigJson() {
         List<PluginParams> paramsList = new ArrayList<>();
 
         InputParam atMobiles = InputParam.newBuilder("atMobiles", "atMobiles")
-                .addValidate(Validate.newBuilder().setRequired(true).build())
+                .addValidate(Validate.newBuilder().setRequired(false).build())
                 .build();
-        InputParam atUserIds = InputParam.newBuilder("atUserIds", "atUserIds")
-                .addValidate(Validate.newBuilder().setRequired(true).build())
+        InputParam atDingtalkIds = InputParam.newBuilder("atDingtalkIds", "atDingtalkIds")
+                .addValidate(Validate.newBuilder().setRequired(false).build())
                 .build();
         RadioParam isAtAll = RadioParam.newBuilder("isAtAll", "isAtAll")
                 .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false))
@@ -120,7 +120,7 @@ public String getConfigJson() {
                 .build();
 
         paramsList.add(atMobiles);
-        paramsList.add(atUserIds);
+        paramsList.add(atDingtalkIds);
         paramsList.add(isAtAll);
 
         ObjectMapper mapper = new ObjectMapper();
@@ -135,4 +135,4 @@ public String getConfigJson() {
 
         return result;
     }
-}
+}

datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/java/io/datavines/notification/plugin/dingtalk/entity/ReceiverConfig.java

@@ -29,7 +29,7 @@ public class ReceiverConfig {
 
     private String atMobiles;
 
-    private String atUserIds;
+    private String atDingtalkIds;
 
     private Boolean isAtAll;
 
@@ -47,6 +47,6 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(atMobiles, atUserIds);
+        return Objects.hash(atMobiles, atDingtalkIds);
     }
-}
+}

datavines-server/src/main/java/io/datavines/server/api/controller/CatalogLineageController.java

@@ -18,7 +18,7 @@
 
 import io.datavines.core.aop.RefreshToken;
 import io.datavines.core.constant.DataVinesConstants;
-import io.datavines.server.api.dto.bo.catalog.lineage.EntityEdgeInfo;
+import io.datavines.server.api.dto.bo.catalog.lineage.LineageEntityEdgeInfo;
 import io.datavines.server.api.dto.bo.catalog.lineage.SqlWithDataSourceKeyProperties;
 import io.datavines.server.api.dto.bo.catalog.lineage.SqlWithDataSourceList;
 import io.datavines.server.repository.entity.catalog.CatalogTagCategory;
@@ -42,7 +42,7 @@ public class CatalogLineageController {
 
     @ApiOperation(value = "add lineage", response = Long.class)
     @PostMapping(value = "/add", consumes = MediaType.APPLICATION_JSON_VALUE)
-    public Object addLineage(@Valid @RequestBody EntityEdgeInfo entityEdgeInfo) {
+    public Object addLineage(@Valid @RequestBody LineageEntityEdgeInfo entityEdgeInfo) {
         return catalogEntityRelService.addLineage(entityEdgeInfo);
     }
 

datavines-server/src/main/java/io/datavines/server/api/controller/SlaController.java

@@ -100,6 +100,16 @@ public Object test(@PathVariable("slaId") Long slaId) {
         return client.notify(message, configuration);
     }
 
+    @ApiOperation(value = "test sla sender")
+    @PostMapping(value = "/sender/test")
+    public Object testSend(@Valid @RequestBody SlaNotificationCreate slaNotificationSendTest) {
+        SlaNotificationMessage message = new SlaNotificationMessage();
+        message.setMessage("[\"test sla sender\"]");
+        message.setSubject("Test sender");
+        Map<SlaSenderMessage, Set<SlaConfigMessage>> configuration = slaNotificationService.getSlasNotificationConfigurationBySlasIdAndSenderId(slaNotificationSendTest.getSlaId(), slaNotificationSendTest.getSenderId(), slaNotificationSendTest.getConfig());
+        return client.notify(message, configuration);
+    }
+
     @ApiOperation(value = "page list sla")
     @GetMapping(value = "/page")
     public Object listSlas(@RequestParam("workspaceId") Long workspaceId,
@@ -219,4 +229,4 @@ public Object pageListNotification(@RequestParam("workspaceId") Long workspaceId
                                    @RequestParam("pageSize") Integer pageSize) {
         return slaNotificationService.pageListNotification(workspaceId, slaId, searchVal, pageNumber, pageSize);
     }
-}
+}

datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/lineage/CatalogEntityColumnLineageDetail.java

@@ -16,6 +16,7 @@
  */
 package io.datavines.server.api.dto.bo.catalog.lineage;
 
+import io.datavines.server.api.dto.bo.catalog.CatalogEntityInstanceInfo;
 import lombok.Data;
 
 import java.io.Serializable;
@@ -26,9 +27,9 @@ public class CatalogEntityColumnLineageDetail implements Serializable {
 
     private static final long serialVersionUID = -1L;
 
-    private List<String> fromChildren;
+    private List<CatalogEntityInstanceInfo> fromChildren;
 
     private String function;
 
-    private String toChild;
+    private CatalogEntityInstanceInfo toChild;
 }

datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/lineage/CatalogEntityLineageDetail.java

@@ -20,13 +20,14 @@
 import lombok.Data;
 
 import java.io.Serializable;
+import java.util.List;
 
 @Data
 public class CatalogEntityLineageDetail implements Serializable {
 
     private static final long serialVersionUID = -1L;
 
-    private CatalogEntityColumnLineageDetail childRelDetail;
+    private List<CatalogEntityColumnLineageDetail> childRelDetailList;
 
     private String description;
 

datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/lineage/LineageEntityEdgeInfo.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.server.api.dto.bo.catalog.lineage;
+
+import io.datavines.server.api.dto.bo.catalog.CatalogEntityInstanceInfo;
+import lombok.Data;
+
+import java.io.Serializable;
+
+@Data
+public class LineageEntityEdgeInfo implements Serializable {
+
+    private static final long serialVersionUID = -1L;
+
+    private String uuid;
+
+    private CatalogEntityInstanceInfo fromEntity;
+
+    private String description;
+
+    private CatalogEntityLineageDetail lineageDetail;
+
+    private CatalogEntityInstanceInfo toEntity;
+
+}

datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/lineage/LineageEntityNodeInfo.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.server.api.dto.bo.catalog.lineage;
+
+import io.datavines.server.api.dto.bo.catalog.CatalogEntityInstanceInfo;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.List;
+
+@Data
+public class LineageEntityNodeInfo implements Serializable {
+
+    private static final long serialVersionUID = -1L;
+
+    private Long id;
+
+    private String uuid;
+
+    private Long datasourceId;
+
+    private String type;
+
+    private String fullyQualifiedName;
+
+    private String displayName;
+
+    private String description;
+
+    private CatalogEntityInstanceInfo datasource;
+
+    private CatalogEntityInstanceInfo database;
+
+    private CatalogEntityInstanceInfo schema;
+
+    private CatalogEntityInstanceInfo catalog;
+
+    private List<CatalogEntityInstanceInfo> columns;
+
+}

datavines-server/src/main/java/io/datavines/server/api/dto/vo/catalog/lineage/CatalogEntityLineageVO.java

@@ -16,8 +16,8 @@
  */
 package io.datavines.server.api.dto.vo.catalog.lineage;
 
-import io.datavines.server.api.dto.bo.catalog.CatalogEntityInstanceInfo;
-import io.datavines.server.api.dto.bo.catalog.lineage.EntityEdgeInfo;
+import io.datavines.server.api.dto.bo.catalog.lineage.LineageEntityEdgeInfo;
+import io.datavines.server.api.dto.bo.catalog.lineage.LineageEntityNodeInfo;
 import lombok.Data;
 
 import java.io.Serializable;
@@ -26,9 +26,9 @@
 @Data
 public class CatalogEntityLineageVO implements Serializable {
 
-    public List<EntityEdgeInfo> upstreamEdges;
+    public LineageEntityNodeInfo currentNode;
 
-    public List<EntityEdgeInfo> downstreamEdges;
+    public List<LineageEntityNodeInfo> nodes;
 
-    public CatalogEntityInstanceInfo currentEntity;
-}
+    public List<LineageEntityEdgeInfo> edges;
+ }

datavines-server/src/main/java/io/datavines/server/enums/LineageSourceType.java

@@ -42,19 +42,29 @@ public enum LineageSourceType {
 
     private final String description;
 
-    private static final Map<Integer, LineageSourceType> COMMAND_CATEGORY_MAP = new HashMap<>();
+    private static final Map<Integer, LineageSourceType> CODE_2_MAP = new HashMap<>();
+
+    private static final Map<String, LineageSourceType> DESC_2_MAP = new HashMap<>();
 
     static {
         for (LineageSourceType commandCategory : LineageSourceType.values()) {
-            COMMAND_CATEGORY_MAP.put(commandCategory.code,commandCategory);
+            CODE_2_MAP.put(commandCategory.code,commandCategory);
+            DESC_2_MAP.put(commandCategory.description,commandCategory);
         }
     }
 
     public static LineageSourceType of(Integer status) {
-        if (COMMAND_CATEGORY_MAP.containsKey(status)) {
-            return COMMAND_CATEGORY_MAP.get(status);
+        if (CODE_2_MAP.containsKey(status)) {
+            return CODE_2_MAP.get(status);
+        }
+        throw new IllegalArgumentException("invalid lineage source type : " + status);
+    }
+
+    public static LineageSourceType descOf(String description) {
+        if (DESC_2_MAP.containsKey(description)) {
+            return DESC_2_MAP.get(description);
         }
-        throw new IllegalArgumentException("invalid command category : " + status);
+        throw new IllegalArgumentException("invalid lineage source type : " + description);
     }
 
     public int getCode() {

datavines-server/src/main/java/io/datavines/server/repository/service/CatalogEntityInstanceService.java

@@ -34,6 +34,10 @@ public interface CatalogEntityInstanceService extends IService<CatalogEntityInst
 
     CatalogEntityInstance getByTypeAndFQN(String type, String fqn);
 
+    CatalogEntityInstance getParent(String uuid);
+
+    List<CatalogEntityInstance> getChildren(String uuid);
+
     CatalogEntityInstance getByUUID(String uuid);
 
     CatalogEntityInstance getByDataSourceAndFQN(Long dataSourceId, String fqn);

datavines-server/src/main/java/io/datavines/server/repository/service/CatalogEntityRelService.java

@@ -17,15 +17,15 @@
 package io.datavines.server.repository.service;
 
 import com.baomidou.mybatisplus.extension.service.IService;
-import io.datavines.server.api.dto.bo.catalog.lineage.EntityEdgeInfo;
+import io.datavines.server.api.dto.bo.catalog.lineage.LineageEntityEdgeInfo;
 import io.datavines.server.api.dto.bo.catalog.lineage.SqlWithDataSourceKeyProperties;
 import io.datavines.server.api.dto.bo.catalog.lineage.SqlWithDataSourceList;
 import io.datavines.server.api.dto.vo.catalog.lineage.CatalogEntityLineageVO;
 import io.datavines.server.repository.entity.catalog.CatalogEntityRel;
 
 public interface CatalogEntityRelService extends IService<CatalogEntityRel> {
 
-    public boolean addLineage(EntityEdgeInfo entityEdgeInfo);
+    public boolean addLineage(LineageEntityEdgeInfo entityEdgeInfo);
 
     public boolean addLineageByParseSql(SqlWithDataSourceList sqlWithDataSourceList);
 

datavines-server/src/main/java/io/datavines/server/repository/service/SlaNotificationService.java

@@ -31,6 +31,8 @@ public interface SlaNotificationService extends IService<SlaNotification>{
 
     Map<SlaSenderMessage, Set<SlaConfigMessage>> getSlasNotificationConfigurationBySlasId(Long slaId);
 
+    Map<SlaSenderMessage, Set<SlaConfigMessage>> getSlasNotificationConfigurationBySlasIdAndSenderId(Long slaId, Long senderId, String config);
+
     Map<SlaSenderMessage, Set<SlaConfigMessage>> getSlasNotificationConfigurationByJobId(Long job);
 
     String getConfigJson(String type);