大数据

datax提速方案

主键是字符串类型,设置多个channel运行

{
  "job": {
    "content": [
      {
        "reader": {
          "name": "mysqlreader",
          "parameter": {
            "username": "root",
            "password": "root",
            "connection": [
              {
                "querySql": [
                  "SELECT id, field_1, field_2, field_3, field_4, field_5 FROM realtime_ods.table01 WHERE CRC32(id) % 4 = 0 ",
                  "SELECT id, field_1, field_2, field_3, field_4, field_5 FROM realtime_ods.table01 WHERE CRC32(id) % 4 = 1 ",
                  "SELECT id, field_1, field_2, field_3, field_4, field_5 FROM realtime_ods.table01 WHERE CRC32(id) % 4 = 2 ",
                  "SELECT id, field_1, field_2, field_3, field_4, field_5 FROM realtime_ods.table01 WHERE CRC32(id) % 4 = 3 "
                ],
                "jdbcUrl": ["jdbc:mysql://192.168.0.1:9030/db_name?rewriteBatchedStatements=true"]
              }
            ]
          }
        },
        "writer": {
          "name": "mysqlwriter",
          "parameter": {
            "username": "root",
            "password": "root",
            "column": ["id", "field_1", "field_2", "field_3", "field_4", "field_5"],
            "preSql": [],
            "postSql": [],
            "connection": [
              {
                "jdbcUrl": "jdbc:mysql://192.168.0.1:4000/db_name?useSSL=false&serverTimezone=UTC",
                "table": ["table01"]
              }
            ],
            "writeMode": "update"
          }
        }
      }
    ],
    "setting": {
      "speed": {
        "channel": 4,
        "byte": -1
      },
      "errorLimit": {
        "record": 100,
        "percentage": 0.1
      }
    }
  }
}

提速原理 4个channel 每个channel一个querySql 这样相当于提速4倍。