From 5d1c9036a1299f0205aced41b652d6f89881f302 Mon Sep 17 00:00:00 2001 From: gaojia Date: Sat, 9 Mar 2024 20:29:55 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=AE=9E=E4=BE=8B=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...taX与DataX-Web的安装部署与应用.md | 161 +++++++++++++++++- 1 file changed, 159 insertions(+), 2 deletions(-) diff --git a/技术知识库/技术文档/数据/ETL工具DataX与DataX-Web的安装部署与应用.md b/技术知识库/技术文档/数据/ETL工具DataX与DataX-Web的安装部署与应用.md index 0c8817e..79ab338 100644 --- a/技术知识库/技术文档/数据/ETL工具DataX与DataX-Web的安装部署与应用.md +++ b/技术知识库/技术文档/数据/ETL工具DataX与DataX-Web的安装部署与应用.md @@ -74,8 +74,165 @@ job文件都放在/datax/job目录 python /opt/datax/bin/datax.py /opt/datax/job/xx.json **（三） 实际应用** -待补充。。 +如下为实际应用示例 +3.1 任务管理配置截图 +![输入图片说明](https://git.rzdata.net/rzdata/knowledge-image/raw/branch/master/imgs/2024-03-09/AYqrSONWnm2bHpSP.png) +3.2 全量数据同步json配置【注意：同步数据前，会删除目标表的全部数据】 +{ + "job": { + "setting": { + "speed": { + "channel": 3, + "byte": 1048576 + }, + "errorLimit": { + "record": 0, + "percentage": 0.02 + } + }, + "content": [ + { + "reader": { + "name": "oraclereader", + "parameter": { + "username": "qJFt0d+pL7uQPs57rbfG/Q==", + "password": "DjjcIe3E9n71mmKvI55MtA==", + "column": [ + "\"PAPER_ID\"", + "\"AUTH_OBJECT\"", + "\"OBJECT_TYPE\"", + "\"AUTH_VALUE\"" + ], + "splitPk": "", + "connection": [ + { + "table": [ + "T_CMS_D_PAPER_AUTH" + ], + "jdbcUrl": [ + "jdbc:oracle:thin:@//10.158.129.136:1521/wportal" + ] + } + ] + } + }, + "writer": { + "name": "mysqlwriter", + "parameter": { + "writeMode": "update", + "username": "TUiXezDzJBqHeHh+D2YEKA==", + "password": "E2+aeKRFsyypelKNid/dYP+cUf7U/tFGQFV59oIG608=", + "column": [ + "`PAPER_ID`", + "`AUTH_OBJECT`", + "`OBJECT_TYPE`", + "`AUTH_VALUE`" + ], + "preSql": [ + "truncate table t_cms_d_paper_auth" + ], + "connection": [ + { + "table": [ + "t_cms_d_paper_auth" + ], + "jdbcUrl": "jdbc:mysql://172.28.145.61:3306/cz_doc" + } + ] + } + } + } + ] + } +} +3.3 增量数据同步配置 +{ + "job": { + "setting": { + "speed": { + "channel": 3, + "byte": 1048576 + }, + "errorLimit": { + "record": 0, + "percentage": 0.02 + } + }, + "content": [ + { + "reader": { + "name": "oraclereader", + "parameter": { + "username": "qJFt0d+pL7uQPs57rbfG/Q==", + "password": "DjjcIe3E9n71mmKvI55MtA==", + "column": [ + "\"FILE_ID\"", + "\"FILENAME\"", + "\"FILE_PATH\"", + "\"FILE_SIZE\"", + "\"FILE_FORMAT\"", + "\"FILE_MD5\"", + "\"APP_ID\"", + "\"CREATE_DATE\"", + "\"UPDATE_DATE\"", + "\"DELETE_FLAG\"", + "\"DELETE_DATE\"", + "\"BIGDATA_FILE_ID\"", + "\"LABEL_NAME\"", + "\"IS_UPLOAD\"" + ], + "where": "TO_CHAR(create_date,'yyyy-MM-dd HH24:mi:ss') > TO_CHAR(TO_DATE(SYSDATE - 3),'yyyy-MM-dd HH24:mi:ss')", + "splitPk": "", + "connection": [ + { + "table": [ + "T_FILE" + ], + "jdbcUrl": [ + "jdbc:oracle:thin:@//10.158.129.136:1521/wportal" + ] + } + ] + } + }, + "writer": { + "name": "mysqlwriter", + "parameter": { + "writeMode": "update", + "username": "TUiXezDzJBqHeHh+D2YEKA==", + "password": "E2+aeKRFsyypelKNid/dYP+cUf7U/tFGQFV59oIG608=", + "column": [ + "`FILE_ID`", + "`FILENAME`", + "`FILE_PATH`", + "`FILE_SIZE`", + "`FILE_FORMAT`", + "`FILE_MD5`", + "`APP_ID`", + "`CREATE_DATE`", + "`UPDATE_DATE`", + "`DELETE_FLAG`", + "`DELETE_DATE`", + "`BIGDATA_FILE_ID`", + "`LABEL_NAME`", + "`IS_UPLOAD`" + ], + "connection": [ + { + "table": [ + "t_file" + ], + "jdbcUrl": "jdbc:mysql://172.28.145.61:3306/cz_doc" + } + ] + } + } + } + ] + } +} +可通过where 语句，设定同步几天之内的数据。 \ No newline at end of file