flinkx1.12同步mysql-hive 数据量700万,本地模式可以跑,yarn-perjob无法跑
- Dominant language
- Java
- Stars
- 4.1k
- Forks
- 1.7k
- PR merge metrics
- No merged PRs in 30d
Description
### Search before asking
- [X] I had searched in the [issues](https://github.com/DTStack/chunjun/issues) and found no similar issues.
### What happened



执行JSON:
{
"job": {
"content": [{
"reader": {
"parameter": {
"password": "123456",
"dataSourceId": 14,
"column": [{
"precision": 20,
"name": "id",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 32,
"name": "device_id",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 12,
"name": "point",
"columnDisplaySize": 12,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "hash",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "value",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 19,
"name": "acq_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 12,
"name": "ratio",
"columnDisplaySize": 12,
"type": "FLOAT"
}, {
"precision": 32,
"name": "max_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "min_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "out_param",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "item_code",
"columnDisplaySize": 8,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "descr",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 16,
"name": "save_hst",
"columnDisplaySize": 16,
"type": "SMALLINT"
}, {
"precision": 1,
"name": "del_flag",
"columnDisplaySize": 1,
"type": "TINYINT"
}, {
"precision": 20,
"name": "create_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 20,
"name": "update_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 19,
"name": "create_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 19,
"name": "update_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 50,
"name": "ext_first",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_second",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_third",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fourth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fifth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_sixth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_seventh",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_eighth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_ninth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_tenth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "device_add_id",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "today_start_value",
"columnDisplaySize": 10,
"type": "DECIMAL"
}, {
"precision": 75,
"name": "unit",
"columnDisplaySize": 75,
"type": "VARCHAR"
}],
"connection": [{
"jdbcUrl": ["jdbc:mysql://172.18.8.209:3306/Vasyslink_yag001?useSSL=false&useUnicode=true&characterEncoding=utf8"],
"table": ["tb_device_point_data_hst_24"]
}],
"splitPk": "id",
"username": "root"
},
"name": "mysqlreader"
},
"writer": {
"parameter": {
"tablesColumn": "{\"tb_device_point_data_hst_666\":[{\"key\":\"id\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"device_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"point\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"hash\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"value\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"acq_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ratio\",\"type\":\"float\",\"precision\":7,\"columnDisplaySize\":24},{\"key\":\"max_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"min_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"out_param\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"item_code\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"descr\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"save_hst\",\"type\":\"smallint\",\"precision\":5,\"columnDisplaySize\":6},{\"key\":\"del_flag\",\"type\":\"tinyint\",\"precision\":3,\"columnDisplaySize\":4},{\"key\":\"create_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"update_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"create_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"update_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_first\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_second\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_third\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fourth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fifth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_sixth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_seventh\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_eighth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_ninth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_tenth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"device_add_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"today_start_value\",\"type\":\"decimal\",\"precision\":8,\"columnDisplaySize\":10},{\"key\":\"unit\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647}]}",
"dataSourceId": 54,
"partition": "pt",
"jdbcUrl": "jdbc:hive2://172.18.8.208:10000/Vasyslink_yag001",
"defaultFS": "hdfs://172.18.8.207:8020",
"writeMode": "overwrite",
"maxFileSize": 1073741824,
"partitionValue":"",
"fieldDelimiter": "\t",
"partitionType": "DAY",
"fileType": "text",
"charsetName": "UTF-8"
},
"name": "hivewriter"
}
}],
"setting": {
"log": {
"isLogger": false
},
"errorLimit": {},
"speed": {
"bytes": 0,
"channel": 1
}
}
}
}
### What you expected to happen
{
"job": {
"content": [{
"reader": {
"parameter": {
"password": "123456",
"dataSourceId": 14,
"column": [{
"precision": 20,
"name": "id",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 32,
"name": "device_id",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 12,
"name": "point",
"columnDisplaySize": 12,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "hash",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "value",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 19,
"name": "acq_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 12,
"name": "ratio",
"columnDisplaySize": 12,
"type": "FLOAT"
}, {
"precision": 32,
"name": "max_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "min_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "out_param",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "item_code",
"columnDisplaySize": 8,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "descr",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 16,
"name": "save_hst",
"columnDisplaySize": 16,
"type": "SMALLINT"
}, {
"precision": 1,
"name": "del_flag",
"columnDisplaySize": 1,
"type": "TINYINT"
}, {
"precision": 20,
"name": "create_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 20,
"name": "update_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 19,
"name": "create_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 19,
"name": "update_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 50,
"name": "ext_first",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_second",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_third",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fourth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fifth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_sixth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_seventh",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_eighth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_ninth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_tenth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "device_add_id",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "today_start_value",
"columnDisplaySize": 10,
"type": "DECIMAL"
}, {
"precision": 75,
"name": "unit",
"columnDisplaySize": 75,
"type": "VARCHAR"
}],
"connection": [{
"jdbcUrl": ["jdbc:mysql://172.18.8.209:3306/Vasyslink_yag001?useSSL=false&useUnicode=true&characterEncoding=utf8"],
"table": ["tb_device_point_data_hst_24"]
}],
"splitPk": "id",
"username": "root"
},
"name": "mysqlreader"
},
"writer": {
"parameter": {
"tablesColumn": "{\"tb_device_point_data_hst_666\":[{\"key\":\"id\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"device_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"point\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"hash\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"value\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"acq_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ratio\",\"type\":\"float\",\"precision\":7,\"columnDisplaySize\":24},{\"key\":\"max_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"min_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"out_param\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"item_code\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"descr\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"save_hst\",\"type\":\"smallint\",\"precision\":5,\"columnDisplaySize\":6},{\"key\":\"del_flag\",\"type\":\"tinyint\",\"precision\":3,\"columnDisplaySize\":4},{\"key\":\"create_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"update_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"create_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"update_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_first\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_second\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_third\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fourth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fifth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_sixth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_seventh\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_eighth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_ninth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_tenth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"device_add_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"today_start_value\",\"type\":\"decimal\",\"precision\":8,\"columnDisplaySize\":10},{\"key\":\"unit\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647}]}",
"dataSourceId": 54,
"partition": "pt",
"jdbcUrl": "jdbc:hive2://172.18.8.208:10000/Vasyslink_yag001",
"defaultFS": "hdfs://172.18.8.207:8020",
"writeMode": "overwrite",
"maxFileSize": 1073741824,
"partitionValue":"",
"fieldDelimiter": "\t",
"partitionType": "DAY",
"fileType": "text",
"charsetName": "UTF-8"
},
"name": "hivewriter"
}
}],
"setting": {
"log": {
"isLogger": false
},
"errorLimit": {},
"speed": {
"bytes": 0,
"channel": 1
}
}
}
}
### How to reproduce
通过日志发现:mysqlsource端 在yarn模式下执行SQL超时,无法获取返回结果,但是local模式下是可以执行的:
SELECT `id`, `device_id`, `point`, `hash`, `value`, `acq_time`, `ratio`, `max_val`, `min_val`, `out_param`, `item_code`, `descr`, `save_hst`, `del_flag`, `create_user`, `update_user`, `create_time`, `update_time`, `ext_first`, `ext_second`, `ext_third`, `ext_fourth`, `ext_fifth`, `ext_sixth`, `ext_seventh`, `ext_eighth`, `ext_ninth`, `ext_tenth`, `device_add_id`, `today_start_value`, `unit` FROM `tb_device_point_data_hst_01` WHERE 1=1
### Anything else
_No response_
### Version
1.12_release
### Are you willing to submit PR?
- [X] Yes I am willing to submit a PR!
### Code of Conduct
- [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
Contributor guide
Assessment
This issue has not been assessed yet.