chunjun标准模式下mysql-hive 并发执行问题
- Dominant language
- Java
- Stars
- 4.1k
- Forks
- 1.7k
- PR merge metrics
- No merged PRs in 30d
Description
### Search before asking
- [X] I had searched in the [issues](https://github.com/DTStack/chunjun/issues) and found no similar issues.
### What happened
来源数据mysql总数据只有8300万左右
1、source端:执行并发操作,并没有按照切分键均匀查询 切分键为Id
2、hivesink端:执行期间出现两个问题:a、执行过程中移动文件出问题,比如:并发20,先执行完的通道,会将.data文件删除,后面还在执行的通道,找不到.data文件 b|、通道多了 很多通道没有数据直接关闭



### What you expected to happen
1、执行JSON
{
"job": {
"content": [{
"reader": {
"parameter": {
"password": "123456",
"dataSourceId": 14,
"column": [{
"precision": 20,
"name": "id",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 32,
"name": "device_id",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 12,
"name": "point",
"columnDisplaySize": 12,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "hash",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "value",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 19,
"name": "acq_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 12,
"name": "ratio",
"columnDisplaySize": 12,
"type": "FLOAT"
}, {
"precision": 32,
"name": "max_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "min_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "out_param",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "item_code",
"columnDisplaySize": 8,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "descr",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 16,
"name": "save_hst",
"columnDisplaySize": 16,
"type": "SMALLINT"
}, {
"precision": 20,
"name": "create_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 20,
"name": "update_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 19,
"name": "create_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 19,
"name": "update_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 50,
"name": "ext_first",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_second",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_third",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fourth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fifth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_sixth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_seventh",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_eighth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_ninth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_tenth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "device_add_id",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "today_start_value",
"columnDisplaySize": 10,
"type": "DECIMAL"
}, {
"precision": 75,
"name": "unit",
"columnDisplaySize": 75,
"type": "VARCHAR"
}],
"connection": [{
"jdbcUrl": ["jdbc:mysql://172.18.8.209:3306/Vasyslink_yag001?useSSL=false&useUnicode=true&characterEncoding=utf8&useCursorFetch=true"],
"table": ["tb_device_point_data_hst_01"]
}],
"splitPk": "id",
"username": "root"
},
"name": "mysqlreader"
},
"writer": {
"parameter": {
"tablesColumn": "{\"tb_device_point_data_hst_996\":[{\"key\":\"id\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"device_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"point\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"hash\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"value\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"acq_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ratio\",\"type\":\"float\",\"precision\":7,\"columnDisplaySize\":24},{\"key\":\"max_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"min_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"out_param\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"item_code\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"descr\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"save_hst\",\"type\":\"smallint\",\"precision\":5,\"columnDisplaySize\":6},{\"key\":\"create_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"update_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"create_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"update_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_first\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_second\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_third\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fourth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fifth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_sixth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_seventh\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_eighth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_ninth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_tenth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"device_add_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"today_start_value\",\"type\":\"decimal\",\"precision\":8,\"columnDisplaySize\":10},{\"key\":\"unit\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647}]}",
"dataSourceId": 54,
"partition": "pt",
"jdbcUrl": "jdbc:hive2://172.18.8.208:10000/Vasyslink_yag001",
"defaultFS": "hdfs://172.18.8.207:8020",
"writeMode": "overwrite",
"maxFileSize": 1073741824,
"fieldDelimiter": "\t",
"partitionType" : "DAY",
"fileType": "text",
"charsetName": "UTF-8"
},
"name": "hivewriter"
}
}],
"setting": {
"log": {
"isLogger": false
},
"errorLimit": {},
"speed": {
"channel": 10
}
}
}
}
### How to reproduce
{
"job": {
"content": [{
"reader": {
"parameter": {
"password": "123456",
"dataSourceId": 14,
"column": [{
"precision": 20,
"name": "id",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 32,
"name": "device_id",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 12,
"name": "point",
"columnDisplaySize": 12,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "hash",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "value",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 19,
"name": "acq_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 12,
"name": "ratio",
"columnDisplaySize": 12,
"type": "FLOAT"
}, {
"precision": 32,
"name": "max_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "min_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "out_param",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "item_code",
"columnDisplaySize": 8,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "descr",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 16,
"name": "save_hst",
"columnDisplaySize": 16,
"type": "SMALLINT"
}, {
"precision": 20,
"name": "create_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 20,
"name": "update_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 19,
"name": "create_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 19,
"name": "update_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 50,
"name": "ext_first",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_second",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_third",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fourth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fifth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_sixth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_seventh",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_eighth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_ninth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_tenth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "device_add_id",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "today_start_value",
"columnDisplaySize": 10,
"type": "DECIMAL"
}, {
"precision": 75,
"name": "unit",
"columnDisplaySize": 75,
"type": "VARCHAR"
}],
"connection": [{
"jdbcUrl": ["jdbc:mysql://172.18.8.209:3306/Vasyslink_yag001?useSSL=false&useUnicode=true&characterEncoding=utf8&useCursorFetch=true"],
"table": ["tb_device_point_data_hst_01"]
}],
"splitPk": "id",
"username": "root"
},
"name": "mysqlreader"
},
"writer": {
"parameter": {
"tablesColumn": "{\"tb_device_point_data_hst_996\":[{\"key\":\"id\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"device_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"point\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"hash\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"value\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"acq_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ratio\",\"type\":\"float\",\"precision\":7,\"columnDisplaySize\":24},{\"key\":\"max_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"min_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"out_param\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"item_code\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"descr\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"save_hst\",\"type\":\"smallint\",\"precision\":5,\"columnDisplaySize\":6},{\"key\":\"create_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"update_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"create_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"update_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_first\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_second\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_third\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fourth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fifth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_sixth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_seventh\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_eighth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_ninth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_tenth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"device_add_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"today_start_value\",\"type\":\"decimal\",\"precision\":8,\"columnDisplaySize\":10},{\"key\":\"unit\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647}]}",
"dataSourceId": 54,
"partition": "pt",
"jdbcUrl": "jdbc:hive2://172.18.8.208:10000/Vasyslink_yag001",
"defaultFS": "hdfs://172.18.8.207:8020",
"writeMode": "overwrite",
"maxFileSize": 1073741824,
"fieldDelimiter": "\t",
"partitionType" : "DAY",
"fileType": "text",
"charsetName": "UTF-8"
},
"name": "hivewriter"
}
}],
"setting": {
"log": {
"isLogger": false
},
"errorLimit": {},
"speed": {
"channel": 10
}
}
}
}
### Anything else
_No response_
### Version
master
### Are you willing to submit PR?
- [X] Yes I am willing to submit a PR!
### Code of Conduct
- [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
Contributor guide
Assessment
This issue has not been assessed yet.