rzdata/knowledge
20
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

更新实例说明

Browse Source
This commit is contained in:
gaojia 2024-03-09 20:29:55 +08:00
parent 95620e0d7f
commit 5d1c9036a1
1 changed files with 159 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
技术知识库/技术文档/数据/ETL工具DataX与DataX-Web的安装部署与应用.md
View File

@ -74,8 +74,165 @@ job文件都放在/datax/job目录
python /opt/datax/bin/datax.py /opt/datax/job/xx.json
**（三） 实际应用**
待补充。。
如下为实际应用示例
3.1 任务管理配置截图
![输入图片说明](https://git.rzdata.net/rzdata/knowledge-image/raw/branch/master/imgs/2024-03-09/AYqrSONWnm2bHpSP.png)
3.2 全量数据同步json配置【注意：同步数据前，会删除目标表的全部数据】
{
"job": {
"setting": {
"speed": {
"channel": 3,
"byte": 1048576
},
"errorLimit": {
"record": 0,
"percentage": 0.02
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "qJFt0d+pL7uQPs57rbfG/Q==",
"password": "DjjcIe3E9n71mmKvI55MtA==",
"column": [
"\"PAPER_ID\"",
"\"AUTH_OBJECT\"",
"\"OBJECT_TYPE\"",
"\"AUTH_VALUE\""
],
"splitPk": "",
"connection": [
{
"table": [
"T_CMS_D_PAPER_AUTH"
],
"jdbcUrl": [
"jdbc:oracle:thin:@//10.158.129.136:1521/wportal"
]
}
]
}
},
"writer": {
"name": "mysqlwriter",
"parameter": {
"writeMode": "update",
"username": "TUiXezDzJBqHeHh+D2YEKA==",
"password": "E2+aeKRFsyypelKNid/dYP+cUf7U/tFGQFV59oIG608=",
"column": [
"`PAPER_ID`",
"`AUTH_OBJECT`",
"`OBJECT_TYPE`",
"`AUTH_VALUE`"
],
"preSql": [
"truncate table t_cms_d_paper_auth"
],
"connection": [
{
"table": [
"t_cms_d_paper_auth"
],
"jdbcUrl": "jdbc:mysql://172.28.145.61:3306/cz_doc"
}
]
}
}
}
]
}
}
3.3 增量数据同步配置
{
"job": {
"setting": {
"speed": {
"channel": 3,
"byte": 1048576
},
"errorLimit": {
"record": 0,
"percentage": 0.02
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "qJFt0d+pL7uQPs57rbfG/Q==",
"password": "DjjcIe3E9n71mmKvI55MtA==",
"column": [
"\"FILE_ID\"",
"\"FILENAME\"",
"\"FILE_PATH\"",
"\"FILE_SIZE\"",
"\"FILE_FORMAT\"",
"\"FILE_MD5\"",
"\"APP_ID\"",
"\"CREATE_DATE\"",
"\"UPDATE_DATE\"",
"\"DELETE_FLAG\"",
"\"DELETE_DATE\"",
"\"BIGDATA_FILE_ID\"",
"\"LABEL_NAME\"",
"\"IS_UPLOAD\""
],
"where": "TO_CHAR(create_date,'yyyy-MM-dd HH24:mi:ss') > TO_CHAR(TO_DATE(SYSDATE - 3),'yyyy-MM-dd HH24:mi:ss')",
"splitPk": "",
"connection": [
{
"table": [
"T_FILE"
],
"jdbcUrl": [
"jdbc:oracle:thin:@//10.158.129.136:1521/wportal"
]
}
]
}
},
"writer": {
"name": "mysqlwriter",
"parameter": {
"writeMode": "update",
"username": "TUiXezDzJBqHeHh+D2YEKA==",
"password": "E2+aeKRFsyypelKNid/dYP+cUf7U/tFGQFV59oIG608=",
"column": [
"`FILE_ID`",
"`FILENAME`",
"`FILE_PATH`",
"`FILE_SIZE`",
"`FILE_FORMAT`",
"`FILE_MD5`",
"`APP_ID`",
"`CREATE_DATE`",
"`UPDATE_DATE`",
"`DELETE_FLAG`",
"`DELETE_DATE`",
"`BIGDATA_FILE_ID`",
"`LABEL_NAME`",
"`IS_UPLOAD`"
],
"connection": [
{
"table": [
"t_file"
],
"jdbcUrl": "jdbc:mysql://172.28.145.61:3306/cz_doc"
}
]
}
}
}
]
}
}
可通过where 语句，设定同步几天之内的数据。
<!--stackedit_data:
eyJoaXN0b3J5IjpbLTk1NzE5MjI3OV19
eyJoaXN0b3J5IjpbMTY1OTMxMTc1MiwtOTU3MTkyMjc5XX0=
-->