文章
datax提速方案
主键是字符串类型,设置多个channel运行
{
"job": {
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"connection": [
{
"querySql": [
"SELECT id, field_1, field_2, field_3, field_4, field_5 FROM realtime_ods.table01 WHERE CRC32(id) % 4 = 0 ",
"SELECT id, field_1, field_2, field_3, field_4, field_5 FROM realtime_ods.table01 WHERE CRC32(id) % 4 = 1 ",
"SELECT id, field_1, field_2, field_3, field_4, field_5 FROM realtime_ods.table01 WHERE CRC32(id) % 4 = 2 ",
"SELECT id, field_1, field_2, field_3, field_4, field_5 FROM realtime_ods.table01 WHERE CRC32(id) % 4 = 3 "
],
"jdbcUrl": ["jdbc:mysql://192.168.0.1:9030/db_name?rewriteBatchedStatements=true"]
}
]
}
},
"writer": {
"name": "mysqlwriter",
"parameter": {
"username": "root",
"password": "root",
"column": ["id", "field_1", "field_2", "field_3", "field_4", "field_5"],
"preSql": [],
"postSql": [],
"connection": [
{
"jdbcUrl": "jdbc:mysql://192.168.0.1:4000/db_name?useSSL=false&serverTimezone=UTC",
"table": ["table01"]
}
],
"writeMode": "update"
}
}
}
],
"setting": {
"speed": {
"channel": 4,
"byte": -1
},
"errorLimit": {
"record": 100,
"percentage": 0.1
}
}
}
}
提速原理 4个channel 每个channel一个querySql 这样相当于提速4倍。