hanjian/data-center
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
gaochang 2023-12-27 19:23:12 +08:00
parent c674440585
commit fa88f9a88a
5 changed files with 47 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
studio/modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/dto/CheckRuleDto.java
View File

@ -9,6 +9,7 @@ import javax.validation.Valid;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import java.io.Serializable;
import java.util.List;
/**
* <p>
@ -61,4 +62,8 @@ public class CheckRuleDto implements Serializable {
private String status;
@ApiModelProperty(value = "备注")
private String remark;
private List<String> transferValue;
private String qualityType;
}

2
studio/modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/query/CheckRuleQuery.java
View File

@ -23,4 +23,6 @@ public class CheckRuleQuery extends BaseQueryParams {
private String ruleSource;
private String ruleTable;
private String ruleColumn;
private String ruleType;
}

18
studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/controller/CheckRuleController.java
View File

@ -85,6 +85,13 @@ public class CheckRuleController extends BaseController {
queryWrapper.like(StrUtil.isNotBlank(checkRuleQuery.getRuleSource()), "r.rule_source", checkRuleQuery.getRuleSource());
queryWrapper.like(StrUtil.isNotBlank(checkRuleQuery.getRuleTable()), "r.rule_table", checkRuleQuery.getRuleTable());
queryWrapper.like(StrUtil.isNotBlank(checkRuleQuery.getRuleColumn()), "r.rule_column", checkRuleQuery.getRuleColumn());
//结构符合校验过滤
if("jj".equals(checkRuleQuery.getRuleType())){
queryWrapper.in("t.code","table_jc","length_jc","null_jc","pk_jc","fk_jc");
} else {
queryWrapper.in("t.code","unique","integrity","accuracy","consistent","relevance","regular","timeliness");
}
// queryWrapper.in("t.code","")
IPage<CheckRuleEntity> page = checkRuleService.page(new Page<>(checkRuleQuery.getPageNum(), checkRuleQuery.getPageSize()), queryWrapper);
List<CheckRuleVo> collect = page.getRecords().stream().map(checkRuleMapper::toVO).collect(Collectors.toList());
JsonPage<CheckRuleVo> jsonPage = new JsonPage<>(page.getCurrent(), page.getSize(), page.getTotal(), collect);
@ -104,6 +111,7 @@ public class CheckRuleController extends BaseController {
return R.ok().setData(checkRuleMapper.toVO(checkRuleEntity));
}
/**
* 修改
* @param checkRule
@ -145,4 +153,14 @@ public class CheckRuleController extends BaseController {
checkRuleService.deleteCheckRuleBatch(ids);
return R.ok();
}
@GetMapping("/listSourceIdBytable/{id}")
public R listSourceIdBytable(@PathVariable String id) {
QueryWrapper<CheckRuleEntity> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("rule_source_id",id);
List<CheckRuleEntity> checkRuleList = checkRuleService.list(queryWrapper);
return R.ok().setData(checkRuleMapper.toVO(checkRuleList));
}
}

7
studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/rules/RegularRule.java
View File

@ -17,11 +17,10 @@ public class RegularRule implements RuleItem{
switch (dbType) {
case ORACLE:
case ORACLE_12C:
builder.append("SELECT SUM(CASE WHEN ROUND(TO_NUMBER(SYSDATE - ").append(column).append(")) >= ").append(map.get(REGULAR)).append(" THEN 1 ELSE 0 END), COUNT(*) FROM ").append(table);
break;
case MYSQL:
builder.append("select sum(case when ").append(column).append(" REGEXP ").append("'"+map.get(REGULAR)+"'").append(" != '' THEN 0 ELSE 1 END),")
.append(" COUNT(*) from ").append(table);
// mysql 执行正则
builder.append("SELECT SUM(CASE WHEN ").append(column).append(" REGEXP ").append("'"+map.get(REGULAR)+"'").append(" != '' THEN 0 ELSE 1 END),").append(" COUNT(*) FROM ").append(table);
break;
case MARIADB:
case SQL_SERVER:
case SQL_SERVER2008:

14
studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/CheckRuleServiceImpl.java
View File

@ -50,12 +50,25 @@ public class CheckRuleServiceImpl extends BaseServiceImpl<CheckRuleDao, CheckRul
@Override
@Transactional(rollbackFor = Exception.class)
public CheckRuleEntity saveCheckRule(CheckRuleDto checkRuleDto) {
// 结构性检测
if("jg".equals(checkRuleDto.getQualityType())){
CheckRuleEntity checkRule = null;
for (int i = 0; i < checkRuleDto.getTransferValue().size(); i++) {
String currentString = checkRuleDto.getTransferValue().get(i);
checkRule = checkRuleMapper.toEntity(checkRuleDto);
checkRule.setRuleTable(currentString);
checkRuleDao.insert(checkRule);
}
return checkRule;
}else {
CheckRuleEntity checkRule = checkRuleMapper.toEntity(checkRuleDto);
String sql = parseSql(checkRule);
checkRule.setRuleSql(sql);
checkRuleDao.insert(checkRule);
return checkRule;
}
}
@Override
@Transactional(rollbackFor = Exception.class)
@ -90,6 +103,7 @@ public class CheckRuleServiceImpl extends BaseServiceImpl<CheckRuleDao, CheckRul
return checkRuleDao.selectOne(new QueryWrapper<CheckRuleEntity>().eq("rule_source_id", sourceId).last("limit 1"));
}
private String parseSql(CheckRuleEntity checkRule) {
RuleConfig ruleConfig = checkRule.getRuleConfig();
Map<String, Object> map = new HashMap<>();