Skip to content

Commit

Permalink
[Fix][Server] Fix data quality report generate error (datavane#434)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zixi0825
zixi0825 committed Sep 1, 2024
1 parent ec7f1c2 commit 27c74b3
Show file tree
Hide file tree
Showing 4 changed files with 29 additions and 18 deletions.
2 changes: 1 addition & 1 deletion datavines-common/src/main/java/io/datavines/common/ConfigConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@ public class ConfigConstants {
public static final String VALIDATE_RESULT_DATA_DIR = "validate_result_data_dir";
public static final String INVALIDATE_ITEM_CAN_OUTPUT = "invalidate_item_can_output";
public static final String ERROR_DATA_OUTPUT_TO_DATASOURCE_DATABASE = "error_data_output_to_datasource_database";
public static final String DATASOURCE = "datasource";
public static final String DATABASE = "database";

public static final String DATABASE_NAME = "database_name";
public static final String TABLE_NAME = "table_name";
public static final String COLUMN_NAME = "column_name";
Expand Down
2 changes: 1 addition & 1 deletion ...mula-diff-percentage/src/main/java/io/datavines/metric/result/formula/DiffPercentage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public BigDecimal getResult(BigDecimal actualValue, BigDecimal expectedValue) {
BigDecimal result = BigDecimal.valueOf(0);
if (expectedValue != null && expectedValue.compareTo(BigDecimal.ZERO) != 0 && actualValue != null) {
BigDecimal quotient = actualValue.subtract(expectedValue).abs();
result = quotient.divide(expectedValue, 2, RoundingMode.HALF_UP).multiply(new BigDecimal(100));
result = quotient.divide(expectedValue, 4, RoundingMode.HALF_UP).multiply(new BigDecimal(100));
}

return result;
Expand Down
5 changes: 3 additions & 2 deletions ...esult-formula-percentage/src/main/java/io/datavines/metric/result/formula/Percentage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ public BigDecimal getScore(BigDecimal actualValue, BigDecimal expectedValue, boo
BigDecimal result = BigDecimal.valueOf(0);
if (expectedValue != null && expectedValue.compareTo(BigDecimal.ZERO) != 0) {
if (MetricDirectionType.NEGATIVE == direction) {
result = new BigDecimal(100).subtract(actualValue.divide(expectedValue, 2, RoundingMode.HALF_UP).multiply(new BigDecimal(100)));
result = new BigDecimal(100).subtract(actualValue.divide(expectedValue, 4, RoundingMode.HALF_UP).multiply(new BigDecimal(100)));
} else if (MetricDirectionType.POSITIVE == direction) {
result = actualValue.divide(expectedValue, 2, RoundingMode.HALF_UP).multiply(new BigDecimal(100));
result = actualValue.divide(expectedValue, 4, RoundingMode.HALF_UP).multiply(new BigDecimal(100));
}
}

Expand All @@ -74,4 +74,5 @@ public String getSymbol() {
public ResultFormulaType getType() {
return ResultFormulaType.PERCENTAGE;
}

}
38 changes: 24 additions & 14 deletions ...rc/main/java/io/datavines/server/repository/service/impl/JobQualityReportServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,12 @@
package io.datavines.server.repository.service.impl;

import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import io.datavines.common.entity.job.BaseJobParameter;
import io.datavines.common.enums.OperatorType;
import io.datavines.common.exception.DataVinesException;
import io.datavines.common.utils.DateUtils;
import io.datavines.common.utils.JSONUtils;
import io.datavines.common.utils.ParameterUtils;
import io.datavines.common.utils.StringUtils;
import io.datavines.core.utils.LanguageUtils;
Expand All @@ -46,6 +43,7 @@
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import java.math.BigDecimal;
import java.math.RoundingMode;
Expand Down Expand Up @@ -74,6 +72,7 @@ public class JobQualityReportServiceImpl extends ServiceImpl<JobQualityReportMap
@Autowired
private JobExecutionService jobExecutionService;

@Transactional(rollbackFor = Exception.class)
@Override
public boolean generateQualityReport(Long datasourceId) {
String yesterday = DateUtils.format(DateUtils.addDays(DateUtils.getCurrentDate(),-1),DateUtils.YYYY_MM_DD);
Expand Down Expand Up @@ -181,28 +180,39 @@ public boolean generateQualityReport(Long datasourceId) {
}

if (CollectionUtils.isNotEmpty(relList)) {
jobExecutionResultReportRelService.remove(new QueryWrapper<JobExecutionResultReportRel>().eq("quality_report_id",qualityReportId));
jobExecutionResultReportRelService.remove(new LambdaQueryWrapper<JobExecutionResultReportRel>()
.eq(JobExecutionResultReportRel::getQualityReportId,qualityReportId));
jobExecutionResultReportRelService.saveBatch(relList);
}
}
}

// 删除datasource_id 下所有表级别的评分
jobQualityReportMapper.delete(new QueryWrapper<JobQualityReport>().eq("entity_level","table").eq("datasource_id",datasourceId));
jobQualityReportMapper.delete(new LambdaQueryWrapper<JobQualityReport>()
.eq(JobQualityReport::getEntityLevel, TABLE)
.eq(JobQualityReport::getDatasourceId, datasourceId)
.eq(JobQualityReport::getReportDate, yesterday));
// 根据datasource_id,database_name,table_name 进行group by,计算得到表的分数
List<JobQualityReport> tableScoreList = jobQualityReportMapper.listTableScoreGroupByDatasource(datasourceId, yesterday);
if (CollectionUtils.isNotEmpty(tableScoreList)) {
saveBatch(tableScoreList);
}

jobQualityReportMapper.delete(new QueryWrapper<JobQualityReport>().eq("entity_level","database").eq("datasource_id",datasourceId));
jobQualityReportMapper.delete(new LambdaQueryWrapper<JobQualityReport>()
.eq(JobQualityReport::getEntityLevel, DATABASE)
.eq(JobQualityReport::getDatasourceId, datasourceId)
.eq(JobQualityReport::getReportDate, yesterday));
// 根据datasource_id,database_name 进行 group by,计算得到数据库的分数
List<JobQualityReport> databaseScoreList = jobQualityReportMapper.listDbScoreGroupByDatasource(datasourceId, yesterday);
if (CollectionUtils.isNotEmpty(databaseScoreList)) {
saveBatch(databaseScoreList);
}

jobQualityReportMapper.delete(new QueryWrapper<JobQualityReport>().eq("entity_level","datasource").eq("datasource_id", datasourceId));
jobQualityReportMapper.delete(new LambdaQueryWrapper<JobQualityReport>()
.eq(JobQualityReport::getEntityLevel, DATASOURCE)
.eq(JobQualityReport::getDatasourceId, datasourceId)
.eq(JobQualityReport::getReportDate, yesterday));;

// 根据datasource_id,database_name 进行 group by,计算得到数据库的分数
List<JobQualityReport> datasourceScoreList = jobQualityReportMapper.listDatasourceScoreGroupByDatasource(datasourceId, yesterday);
if (CollectionUtils.isNotEmpty(datasourceScoreList)) {
Expand Down Expand Up @@ -259,16 +269,16 @@ public JobQualityReportScoreTrend getScoreTrendByCondition(JobQualityReportDashb
JobQualityReportScoreTrend scoreTrend = new JobQualityReportScoreTrend();
String startDateStr = "";
String endDateStr = "";
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(YYYY_MM_DD);
if (StringUtils.isEmpty(dashboardParam.getReportDate())) {
startDateStr = DateUtils.format(DateUtils.addDays(new Date(), -7),"yyyy-MM-dd");
endDateStr = DateUtils.format(DateUtils.addDays(new Date(), -1),"yyyy-MM-dd");
startDateStr = DateUtils.format(DateUtils.addDays(new Date(), -7),YYYY_MM_DD);
endDateStr = DateUtils.format(DateUtils.addDays(new Date(), -1),YYYY_MM_DD);
} else {
endDateStr = dashboardParam.getReportDate();
LocalDate endDate = LocalDate.parse(endDateStr, formatter);
ZonedDateTime zonedDateTime = endDate.atStartOfDay(ZoneId.systemDefault());
Date date = Date.from(zonedDateTime.toInstant());
startDateStr = DateUtils.format(DateUtils.addDays(date,-6),"yyyy-MM-dd");
startDateStr = DateUtils.format(DateUtils.addDays(date,-6),YYYY_MM_DD);
}

LocalDate startDate = LocalDate.parse(startDateStr, formatter);
Expand All @@ -278,7 +288,7 @@ public JobQualityReportScoreTrend getScoreTrendByCondition(JobQualityReportDashb
LocalDate currentDate = startDate;

while (!currentDate.isAfter(endDate)) {
dateList.add(currentDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")));
dateList.add(currentDate.format(DateTimeFormatter.ofPattern(YYYY_MM_DD)));
currentDate = currentDate.plusDays(1);
}

Expand Down Expand Up @@ -342,9 +352,9 @@ public IPage<JobQualityReportVO> getQualityReportPage(JobQualityReportDashboardP
}

if (StringUtils.isNotEmpty(dashboardParam.getTableName())) {
queryWrapper.eq(JobQualityReport::getEntityLevel, "column");
queryWrapper.eq(JobQualityReport::getEntityLevel, COLUMN);
} else {
queryWrapper.eq(JobQualityReport::getEntityLevel, "table");
queryWrapper.eq(JobQualityReport::getEntityLevel, TABLE);
}

return page(page, queryWrapper).convert(jobQualityReport -> {
Expand Down

0 comments on commit 27c74b3

Please sign in to comment.