From 58df49f91cceda525f90111d37bb7c660017c8fd Mon Sep 17 00:00:00 2001 From: Jane <272005125@qq.com> Date: Tue, 2 Jan 2024 19:38:11 +0800 Subject: [PATCH] update --- .../config/service-data-metadata-dev.yml | 4 + .../api/entity/OfflineDataFileEntity.java | 87 +++++++++++++++++++ .../api/query/OfflineDataFileQuery.java | 22 +++++ .../controller/MetadataSourceController.java | 7 ++ .../controller/OfflineDataFileController.java | 60 +++++++++++++ .../data/metadata/dao/OfflineDataFileDao.java | 24 +++++ .../service/MetadataSourceService.java | 6 ++ .../service/OfflineDataFileService.java | 23 +++++ .../impl/MetadataSourceServiceImpl.java | 48 ++++++++++ .../impl/OfflineDataFileServiceImpl.java | 50 +++++++++++ .../mapper/OfflineDataFileMapper.xml | 72 +++++++++++++++ .../quality/api/entity/CheckReportEntity.java | 4 + .../data/quality/api/vo/CheckReportVo.java | 2 + .../data-quality-service/pom.xml | 1 - .../data/quality/config/StartedUpRunner.java | 2 +- .../controller/CheckReportController.java | 2 - .../quality/schedule/SchedulingRunnable.java | 6 +- .../quality/schedule/task/QualityTask.java | 7 ++ .../service/impl/CheckRuleServiceImpl.java | 9 +- .../service/impl/ScheduleJobServiceImpl.java | 6 +- .../resources/mapper/CheckReportMapper.xml | 9 +- .../rest/AuthorizationController.java | 4 +- 22 files changed, 438 insertions(+), 17 deletions(-) create mode 100644 studio/modules/data-metadata-service-parent/data-metadata-service-api/src/main/java/cn/datax/service/data/metadata/api/entity/OfflineDataFileEntity.java create mode 100644 studio/modules/data-metadata-service-parent/data-metadata-service-api/src/main/java/cn/datax/service/data/metadata/api/query/OfflineDataFileQuery.java create mode 100644 studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/controller/OfflineDataFileController.java create mode 100644 studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/dao/OfflineDataFileDao.java create mode 100644 studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/OfflineDataFileService.java create mode 100644 studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/impl/OfflineDataFileServiceImpl.java create mode 100644 studio/modules/data-metadata-service-parent/data-metadata-service/src/main/resources/mapper/OfflineDataFileMapper.xml diff --git a/studio/config/src/main/resources/config/service-data-metadata-dev.yml b/studio/config/src/main/resources/config/service-data-metadata-dev.yml index 91d93e8..7ecf96d 100644 --- a/studio/config/src/main/resources/config/service-data-metadata-dev.yml +++ b/studio/config/src/main/resources/config/service-data-metadata-dev.yml @@ -24,6 +24,10 @@ spring: url: ${common.mysql.master.url} username: ${common.mysql.master.username} password: ${common.mysql.master.password} + servlet: + multipart: + max-file-size: 50MB + mybatis-plus: mapper-locations: classpath*:mapper/*Mapper.xml diff --git a/studio/modules/data-metadata-service-parent/data-metadata-service-api/src/main/java/cn/datax/service/data/metadata/api/entity/OfflineDataFileEntity.java b/studio/modules/data-metadata-service-parent/data-metadata-service-api/src/main/java/cn/datax/service/data/metadata/api/entity/OfflineDataFileEntity.java new file mode 100644 index 0000000..fe38b5a --- /dev/null +++ b/studio/modules/data-metadata-service-parent/data-metadata-service-api/src/main/java/cn/datax/service/data/metadata/api/entity/OfflineDataFileEntity.java @@ -0,0 +1,87 @@ +package cn.datax.service.data.metadata.api.entity; + +import com.baomidou.mybatisplus.annotation.*; +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.time.LocalDateTime; + +/** + *

+ * 数据源信息表 + *

+ * + * @author AllDataDC + * @date 2022-11-14 + */ +@Data +@Accessors(chain = true) +@TableName(value = "offline_data_file", autoResultMap = true) +public class OfflineDataFileEntity { + + private static final long serialVersionUID=1L; + + @TableId(value = "id", type = IdType.ASSIGN_ID) + private String id; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @TableField(value = "create_time", fill = FieldFill.INSERT) + private LocalDateTime createTime; + + /** + * 创建人 + */ + @TableField(value = "create_by", fill = FieldFill.INSERT) + private String createBy; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @TableField(value = "switch_time") + private LocalDateTime switchTime; + + @TableField(value = "remark") + private String remark; + + @TableField(value = "offline_type") + private String offlineType; + + @TableField(value = "is_switch") + private String isSwitch; + + /** + * 文件原始名称 + */ + @TableField(value = "original_file_name") + private String originalFileName; + + /** + * 文件名称 + */ + @TableField(value = "file_name") + private String fileName; + + /** + * 文件大小 + */ + @TableField(value = "file_size") + private Long fileSize; + + /** + * 访问路径 + */ + @TableField(value = "file_path") + private String filePath; + + /** + * 文件类型 + */ + @TableField(value = "content_type") + private String contentType; + + /** + * 文件来源 + */ + @TableField(value = "file_type") + private String fileType; + +} diff --git a/studio/modules/data-metadata-service-parent/data-metadata-service-api/src/main/java/cn/datax/service/data/metadata/api/query/OfflineDataFileQuery.java b/studio/modules/data-metadata-service-parent/data-metadata-service-api/src/main/java/cn/datax/service/data/metadata/api/query/OfflineDataFileQuery.java new file mode 100644 index 0000000..332f24a --- /dev/null +++ b/studio/modules/data-metadata-service-parent/data-metadata-service-api/src/main/java/cn/datax/service/data/metadata/api/query/OfflineDataFileQuery.java @@ -0,0 +1,22 @@ +package cn.datax.service.data.metadata.api.query; + +import cn.datax.common.base.BaseQueryParams; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + *

+ * 数据源信息表 查询实体 + *

+ * + * @author AllDataDC + * @date 2022-11-14 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class OfflineDataFileQuery extends BaseQueryParams { + + private static final long serialVersionUID=1L; + + private String fileName; +} diff --git a/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/controller/MetadataSourceController.java b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/controller/MetadataSourceController.java index 14a386e..41f4c80 100644 --- a/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/controller/MetadataSourceController.java +++ b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/controller/MetadataSourceController.java @@ -29,6 +29,7 @@ import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletResponse; import java.io.OutputStream; @@ -261,4 +262,10 @@ public class MetadataSourceController extends BaseController { List> list = metadataSourceService.getMetadatablood(params.get("sourceId"), params.get("tableId")); return R.ok().setData(list); } + + @PostMapping("/upload/{type}") + public R upload(@RequestParam("file") MultipartFile file, @PathVariable String type) { + metadataSourceService.uploadFile(file, type); + return R.ok(); + } } diff --git a/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/controller/OfflineDataFileController.java b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/controller/OfflineDataFileController.java new file mode 100644 index 0000000..af0d1d9 --- /dev/null +++ b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/controller/OfflineDataFileController.java @@ -0,0 +1,60 @@ +package cn.datax.service.data.metadata.controller; + +import cn.datax.common.base.BaseController; +import cn.datax.common.core.JsonPage; +import cn.datax.common.core.R; +import cn.datax.service.data.metadata.api.entity.OfflineDataFileEntity; +import cn.datax.service.data.metadata.api.query.OfflineDataFileQuery; +import cn.datax.service.data.metadata.service.OfflineDataFileService; +import cn.hutool.core.util.StrUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@RequestMapping("/offline") +public class OfflineDataFileController extends BaseController { + + @Autowired + private OfflineDataFileService offlineDataFileService; + + /** + * 通过ID查询信息 + * + * @param id + * @return + */ + @ApiOperation(value = "获取详细信息", notes = "根据url的id来获取详细信息") + @ApiImplicitParam(name = "id", value = "ID", required = true, dataType = "String", paramType = "path") + @GetMapping("/{id}") + public R getOfflineDataFileById(@PathVariable String id) { + OfflineDataFileEntity offlineDataFileEntity = offlineDataFileService.getOfflineDataFileById(id); + return R.ok().setData(offlineDataFileEntity); + } + + /** + * 分页查询信息 + * + * @return + */ + @GetMapping("/page") + public R getOfflineDataFilePage(OfflineDataFileQuery offlineDataFileQuery) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.like(StrUtil.isNotBlank(offlineDataFileQuery.getFileName()), "s.file_name", offlineDataFileQuery.getFileName()); + IPage page = offlineDataFileService.pageWithAuth(new Page<>(offlineDataFileQuery.getPageNum(), offlineDataFileQuery.getPageSize()), queryWrapper); + List collect = page.getRecords(); + JsonPage jsonPage = new JsonPage<>(page.getCurrent(), page.getSize(), page.getTotal(), collect); + return R.ok().setData(jsonPage); + } + + +} diff --git a/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/dao/OfflineDataFileDao.java b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/dao/OfflineDataFileDao.java new file mode 100644 index 0000000..724d6ba --- /dev/null +++ b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/dao/OfflineDataFileDao.java @@ -0,0 +1,24 @@ +package cn.datax.service.data.metadata.dao; + +import cn.datax.common.base.BaseDao; +import cn.datax.service.data.metadata.api.entity.OfflineDataFileEntity; +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.core.toolkit.Constants; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.io.Serializable; +import java.util.List; + +@Mapper +public interface OfflineDataFileDao extends BaseDao { + + @Override + OfflineDataFileEntity selectById(Serializable id); + + @Override + List selectList(@Param(Constants.WRAPPER) Wrapper queryWrapper); + + > E selectPageWithAuth(E page, @Param(Constants.WRAPPER) Wrapper queryWrapper, @Param("roles") List roles); +} diff --git a/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/MetadataSourceService.java b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/MetadataSourceService.java index c7b9a67..9c7953a 100644 --- a/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/MetadataSourceService.java +++ b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/MetadataSourceService.java @@ -6,9 +6,12 @@ import cn.datax.common.database.core.DbColumn; import cn.datax.common.database.core.DbTable; import cn.datax.service.data.metadata.api.dto.MetadataSourceDto; import cn.datax.service.data.metadata.api.entity.MetadataSourceEntity; +import cn.datax.service.data.metadata.api.entity.OfflineDataFileEntity; import com.aspose.words.Document; import com.baomidou.mybatisplus.core.conditions.Wrapper; import com.baomidou.mybatisplus.core.metadata.IPage; +import org.apache.hc.core5.http.io.entity.FileEntity; +import org.springframework.web.multipart.MultipartFile; import java.util.List; import java.util.Map; @@ -54,4 +57,7 @@ public interface MetadataSourceService extends BaseService > E pageWithAuth(E page, Wrapper queryWrapper); List> getMetadatablood(String datasourceId, String tableName); + + OfflineDataFileEntity uploadFile(MultipartFile file, String type); + } diff --git a/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/OfflineDataFileService.java b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/OfflineDataFileService.java new file mode 100644 index 0000000..c27e62e --- /dev/null +++ b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/OfflineDataFileService.java @@ -0,0 +1,23 @@ +package cn.datax.service.data.metadata.service; + +import cn.datax.common.base.BaseService; +import cn.datax.service.data.metadata.api.entity.MetadataSourceEntity; +import cn.datax.service.data.metadata.api.entity.OfflineDataFileEntity; +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + +/** + *

+ * 数据源信息表 服务类 + *

+ * + * @author AllDataDC + * @date 2022-11-14 + */ +public interface OfflineDataFileService extends BaseService { + + OfflineDataFileEntity getOfflineDataFileById(String id); + + > E pageWithAuth(E page, Wrapper queryWrapper); + +} diff --git a/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/impl/MetadataSourceServiceImpl.java b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/impl/MetadataSourceServiceImpl.java index 2f8c7ff..47718da 100644 --- a/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/impl/MetadataSourceServiceImpl.java +++ b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/java/cn/datax/service/data/metadata/service/impl/MetadataSourceServiceImpl.java @@ -12,6 +12,7 @@ import cn.datax.common.database.core.DbTable; import cn.datax.common.exception.DataException; import cn.datax.common.redis.service.RedisService; import cn.datax.common.utils.SecurityUtil; +import cn.datax.common.utils.ThrowableUtil; import cn.datax.service.data.market.api.entity.DataApiEntity; import cn.datax.service.data.market.api.feign.DataApiServiceFeign; import cn.datax.service.data.metadata.api.dto.DbSchema; @@ -30,7 +31,10 @@ import cn.datax.service.data.standard.api.entity.ContrastEntity; import cn.datax.service.data.standard.api.feign.StandardServiceFeign; import cn.datax.service.data.visual.api.entity.DataSetEntity; import cn.datax.service.data.visual.api.feign.VisualServiceFeign; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.StrUtil; +import cn.hutool.system.SystemUtil; import com.aspose.words.Document; import com.aspose.words.MailMerge; import com.aspose.words.net.System.Data.DataRelation; @@ -48,7 +52,10 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; +import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.sql.Connection; import java.sql.ResultSet; @@ -115,6 +122,9 @@ public class MetadataSourceServiceImpl extends BaseServiceImpl + * 数据源信息表 服务实现类 + *

+ * + * @author AllDataDC + * @date 2022-11-14 + */ +@Service +@Transactional(propagation = Propagation.SUPPORTS, readOnly = true, rollbackFor = Exception.class) +public class OfflineDataFileServiceImpl extends BaseServiceImpl implements OfflineDataFileService { + + @Autowired + private OfflineDataFileDao offlineDataFileDao; + + @Override + public OfflineDataFileEntity getOfflineDataFileById(String id) { + OfflineDataFileEntity offlineDataFileEntity = super.getById(id); + return offlineDataFileEntity; + } + + @Override + public > E pageWithAuth(E page, Wrapper queryWrapper) { + boolean admin = SecurityUtil.isAdmin(); + List roles = new ArrayList<>(); + if (!admin) { + roles = SecurityUtil.getUserRoleIds(); + } + return offlineDataFileDao.selectPageWithAuth(page, queryWrapper, roles); + } + +} diff --git a/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/resources/mapper/OfflineDataFileMapper.xml b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/resources/mapper/OfflineDataFileMapper.xml new file mode 100644 index 0000000..70fc530 --- /dev/null +++ b/studio/modules/data-metadata-service-parent/data-metadata-service/src/main/resources/mapper/OfflineDataFileMapper.xml @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + id, + create_by, + create_time, + switch_time, + remark, + offline_type, + is_switch, + original_file_name, file_name, file_size, file_path, content_type, file_type + + + + ${alias}.id, + ${alias}.create_by, + ${alias}.create_time, + ${alias}.switch_time, + ${alias}.remark, + ${alias}.offline_type, + ${alias}.is_switch, + ${alias}.original_file_name, ${alias}.file_name, ${alias}.file_size, ${alias}.file_path, ${alias}.content_type, ${alias}.file_type + + + + + + + + + diff --git a/studio/modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/entity/CheckReportEntity.java b/studio/modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/entity/CheckReportEntity.java index 3bb4eb6..cb405dd 100644 --- a/studio/modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/entity/CheckReportEntity.java +++ b/studio/modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/entity/CheckReportEntity.java @@ -61,6 +61,10 @@ public class CheckReportEntity implements Serializable { */ private String checkBatch; + private String checkReportType; + + private Integer checkTimeConsuming; + /** * 规则名称 */ diff --git a/studio/modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/vo/CheckReportVo.java b/studio/modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/vo/CheckReportVo.java index 68fe56f..f3f080d 100644 --- a/studio/modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/vo/CheckReportVo.java +++ b/studio/modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/vo/CheckReportVo.java @@ -25,6 +25,8 @@ public class CheckReportVo implements Serializable { private LocalDateTime checkDate; private String checkResult; private Integer checkTotalCount; + private String checkReportType; + private String checkTimeConsuming; private Integer checkErrorCount; private String ruleName; private String ruleType; diff --git a/studio/modules/data-quality-service-parent/data-quality-service/pom.xml b/studio/modules/data-quality-service-parent/data-quality-service/pom.xml index 514ca82..a955f67 100644 --- a/studio/modules/data-quality-service-parent/data-quality-service/pom.xml +++ b/studio/modules/data-quality-service-parent/data-quality-service/pom.xml @@ -88,7 +88,6 @@ com.platform data-metadata-service 0.4.x - compile diff --git a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/config/StartedUpRunner.java b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/config/StartedUpRunner.java index 56d4af7..b962ffe 100644 --- a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/config/StartedUpRunner.java +++ b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/config/StartedUpRunner.java @@ -47,7 +47,7 @@ public class StartedUpRunner implements ApplicationRunner { List list = scheduleJobService.list(Wrappers.lambdaQuery().eq(ScheduleJobEntity::getStatus, DataConstant.TrueOrFalse.TRUE.getKey())); if (CollUtil.isNotEmpty(list)) { list.forEach(job -> { - SchedulingRunnable task = new SchedulingRunnable(job.getId(), job.getBeanName(), job.getMethodName(), job.getMethodParams()); + SchedulingRunnable task = new SchedulingRunnable(job.getId(), job.getBeanName(), job.getMethodName(), job.getMethodParams(), job.getJobType()); cronTaskRegistrar.addCronTask(task, job.getCronExpression()); }); } diff --git a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/controller/CheckReportController.java b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/controller/CheckReportController.java index 21ebf02..3bb86c2 100644 --- a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/controller/CheckReportController.java +++ b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/controller/CheckReportController.java @@ -78,8 +78,6 @@ public class CheckReportController extends BaseController { queryWrapper.like(StrUtil.isNotBlank(checkReportQuery.getRuleSource()), "r.rule_source", checkReportQuery.getRuleSource()); queryWrapper.like(StrUtil.isNotBlank(checkReportQuery.getRuleTable()), "r.rule_table", checkReportQuery.getRuleTable()); queryWrapper.like(StrUtil.isNotBlank(checkReportQuery.getRuleColumn()), "r.rule_column", checkReportQuery.getRuleColumn()); - // 确定唯一核查报告 - queryWrapper.apply("c.check_batch = r.last_check_batch"); IPage page = checkReportService.page(new Page<>(checkReportQuery.getPageNum(), checkReportQuery.getPageSize()), queryWrapper); List collect = page.getRecords().stream().map(checkReportMapper::toVO).collect(Collectors.toList()); JsonPage jsonPage = new JsonPage<>(page.getCurrent(), page.getSize(), page.getTotal(), collect); diff --git a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/SchedulingRunnable.java b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/SchedulingRunnable.java index 407464b..c4003d2 100644 --- a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/SchedulingRunnable.java +++ b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/SchedulingRunnable.java @@ -25,11 +25,14 @@ public class SchedulingRunnable implements Runnable { private String params; - public SchedulingRunnable(String id, String beanName, String methodName, String params) { + private String jobType; + + public SchedulingRunnable(String id, String beanName, String methodName, String params, String jobType) { this.id = id; this.beanName = beanName; this.methodName = methodName; this.params = params; + this.jobType = jobType; } @Override @@ -47,6 +50,7 @@ public class SchedulingRunnable implements Runnable { } batch = DateUtil.format(LocalDateTime.now(), DatePattern.PURE_DATETIME_PATTERN); map.put("batch", batch); + map.put("jobType", jobType); ReflectionUtils.makeAccessible(method); method.invoke(target, map); } catch (Exception ex) { diff --git a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/task/QualityTask.java b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/task/QualityTask.java index fdfc290..c2b034e 100644 --- a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/task/QualityTask.java +++ b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/task/QualityTask.java @@ -65,6 +65,8 @@ public class QualityTask { tasks.add(task); }); List> futures; + long checkConsumeTime = 0L; + long startTime = System.currentTimeMillis(); try { futures = threadPoolExecutor.invokeAll(tasks); // 处理线程返回结果 @@ -76,13 +78,18 @@ public class QualityTask { } catch (Exception e) { e.printStackTrace(); } + checkConsumeTime = System.currentTimeMillis() - startTime; // 关闭线程池 threadPoolExecutor.shutdown(); // 核查报告 + long finalCheckConsumeTime = checkConsumeTime; result.forEach(s -> { // 插入核查结果正常的数据 String status = StrUtil.isBlank(s.getCheckResult()) ? DataConstant.TrueOrFalse.TRUE.getKey() : DataConstant.TrueOrFalse.FALSE.getKey(); if (StrUtil.isBlank(s.getCheckResult())) { + s.setCheckTimeConsuming((int) finalCheckConsumeTime); + s.setCheckResult(DataConstant.TrueOrFalse.TRUE.getKey()); + s.setCheckReportType((String) map.get("jobType")); s.setCheckBatch((String) map.get("batch")); checkReportService.save(s); // 更新最近核查批次号 diff --git a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/CheckRuleServiceImpl.java b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/CheckRuleServiceImpl.java index e1b3852..94278eb 100644 --- a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/CheckRuleServiceImpl.java +++ b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/CheckRuleServiceImpl.java @@ -5,7 +5,6 @@ import cn.datax.common.core.RedisConstant; import cn.datax.common.database.constants.DbType; import cn.datax.common.redis.service.RedisService; import cn.datax.service.data.metadata.api.entity.MetadataTableEntity; -import cn.datax.service.data.metadata.dao.MetadataTableDao; import cn.datax.service.data.quality.api.dto.*; import cn.datax.service.data.quality.api.entity.CheckRuleEntity; import cn.datax.service.data.quality.api.enums.RuleItem; @@ -46,8 +45,8 @@ public class CheckRuleServiceImpl extends BaseServiceImpl queryWrapper = new QueryWrapper<>(); queryWrapper.eq("source_id", checkRuleDto.getRuleSourceId()); queryWrapper.eq("table_name",currentString); - MetadataTableEntity metadataTableEntity = metadataTableDao.selectOne(queryWrapper); + //MetadataTableEntity metadataTableEntity = metadataTableDao.selectOne(queryWrapper); checkRule = checkRuleMapper.toEntity(checkRuleDto); checkRule.setRuleTable(currentString); - checkRule.setRuleTableId(metadataTableEntity.getId()); + //checkRule.setRuleTableId(metadataTableEntity.getId()); checkRuleDao.insert(checkRule); } return checkRule; diff --git a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/ScheduleJobServiceImpl.java b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/ScheduleJobServiceImpl.java index 5f0c945..31cad5a 100644 --- a/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/ScheduleJobServiceImpl.java +++ b/studio/modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/ScheduleJobServiceImpl.java @@ -73,7 +73,7 @@ public class ScheduleJobServiceImpl extends BaseServiceImpl + @@ -24,12 +25,12 @@ id, - check_rule_id, check_date, check_result, check_total_count, check_error_count, check_batch + check_rule_id, check_date, check_result, check_total_count, check_error_count, check_batch, check_report_type, check_time_consuming ${alias}.id, - ${alias}.check_rule_id, ${alias}.check_date, ${alias}.check_result, ${alias}.check_total_count, ${alias}.check_error_count, ${alias}.check_batch + ${alias}.check_rule_id, ${alias}.check_date, ${alias}.check_result, ${alias}.check_total_count, ${alias}.check_error_count, ${alias}.check_batch, ${alias}.check_report_type, ${alias}.check_time_consuming diff --git a/studio/modules/system-service-parent/system-service/src/main/java/com/platform/modules/security/rest/AuthorizationController.java b/studio/modules/system-service-parent/system-service/src/main/java/com/platform/modules/security/rest/AuthorizationController.java index 54fd40b..fe27edb 100644 --- a/studio/modules/system-service-parent/system-service/src/main/java/com/platform/modules/security/rest/AuthorizationController.java +++ b/studio/modules/system-service-parent/system-service/src/main/java/com/platform/modules/security/rest/AuthorizationController.java @@ -63,7 +63,7 @@ public class AuthorizationController { public ResponseEntity login(@Validated @RequestBody AuthUserDto authUser, HttpServletRequest request) throws Exception { // 密码解密 String password = RsaUtils.decryptByPrivateKey(RsaProperties.privateKey, authUser.getPassword()); - // 查询验证码 + /* // 查询验证码 String code = (String) redisUtils.get(authUser.getUuid()); // 清除验证码 redisUtils.del(authUser.getUuid()); @@ -72,7 +72,7 @@ public class AuthorizationController { } if (StringUtils.isBlank(authUser.getCode()) || !authUser.getCode().equalsIgnoreCase(code)) { throw new BadRequestException("验证码错误"); - } + }*/ UsernamePasswordAuthenticationToken authenticationToken = new UsernamePasswordAuthenticationToken(authUser.getUsername(), password); Authentication authentication = authenticationManagerBuilder.getObject().authenticate(authenticationToken);