DTStack / DTStack/chunjun

chunjun标准模式下mysql-hive 并发执行问题

Open
#1,165 1 comment 0 reactions 0 assignees View on GitHub
bug
Dominant language
Java
Stars
4.1k
Forks
1.7k
PR merge metrics
No merged PRs in 30d

Description

### Search before asking

- [X] I had searched in the [issues](https://github.com/DTStack/chunjun/issues) and found no similar issues.

### What happened

来源数据mysql总数据只有8300万左右
1、source端:执行并发操作,并没有按照切分键均匀查询 切分键为Id
2、hivesink端:执行期间出现两个问题:a、执行过程中移动文件出问题,比如:并发20,先执行完的通道,会将.data文件删除,后面还在执行的通道,找不到.data文件 b|、通道多了 很多通道没有数据直接关闭
![image](https://user-images.githubusercontent.com/34857750/185526441-53b82214-386f-4ae5-8ee4-a7f5e5280ef1.png)
![image](https://user-images.githubusercontent.com/34857750/185526144-4ea1ae75-b2af-4792-af29-4632b55d5947.png)
![image](https://user-images.githubusercontent.com/34857750/185526688-01ea68a9-dfe6-415c-89c8-b28172e33dea.png)

### What you expected to happen

1、执行JSON
{
"job": {
"content": [{
"reader": {
"parameter": {
"password": "123456",
"dataSourceId": 14,
"column": [{
"precision": 20,
"name": "id",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 32,
"name": "device_id",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 12,
"name": "point",
"columnDisplaySize": 12,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "hash",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "value",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 19,
"name": "acq_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 12,
"name": "ratio",
"columnDisplaySize": 12,
"type": "FLOAT"
}, {
"precision": 32,
"name": "max_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "min_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "out_param",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "item_code",
"columnDisplaySize": 8,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "descr",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 16,
"name": "save_hst",
"columnDisplaySize": 16,
"type": "SMALLINT"
}, {
"precision": 20,
"name": "create_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 20,
"name": "update_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 19,
"name": "create_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 19,
"name": "update_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 50,
"name": "ext_first",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_second",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_third",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fourth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fifth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_sixth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_seventh",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_eighth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_ninth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_tenth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "device_add_id",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "today_start_value",
"columnDisplaySize": 10,
"type": "DECIMAL"
}, {
"precision": 75,
"name": "unit",
"columnDisplaySize": 75,
"type": "VARCHAR"
}],
"connection": [{
"jdbcUrl": ["jdbc:mysql://172.18.8.209:3306/Vasyslink_yag001?useSSL=false&useUnicode=true&characterEncoding=utf8&useCursorFetch=true"],
"table": ["tb_device_point_data_hst_01"]
}],

"splitPk": "id",
"username": "root"
},
"name": "mysqlreader"
},
"writer": {
"parameter": {
"tablesColumn": "{\"tb_device_point_data_hst_996\":[{\"key\":\"id\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"device_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"point\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"hash\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"value\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"acq_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ratio\",\"type\":\"float\",\"precision\":7,\"columnDisplaySize\":24},{\"key\":\"max_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"min_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"out_param\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"item_code\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"descr\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"save_hst\",\"type\":\"smallint\",\"precision\":5,\"columnDisplaySize\":6},{\"key\":\"create_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"update_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"create_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"update_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_first\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_second\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_third\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fourth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fifth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_sixth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_seventh\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_eighth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_ninth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_tenth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"device_add_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"today_start_value\",\"type\":\"decimal\",\"precision\":8,\"columnDisplaySize\":10},{\"key\":\"unit\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647}]}",
"dataSourceId": 54,
"partition": "pt",
"jdbcUrl": "jdbc:hive2://172.18.8.208:10000/Vasyslink_yag001",
"defaultFS": "hdfs://172.18.8.207:8020",
"writeMode": "overwrite",
"maxFileSize": 1073741824,

"fieldDelimiter": "\t",

"partitionType" : "DAY",
"fileType": "text",
"charsetName": "UTF-8"
},
"name": "hivewriter"
}
}],
"setting": {
"log": {
"isLogger": false
},
"errorLimit": {},
"speed": {

"channel": 10
}
}
}
}

### How to reproduce

{
"job": {
"content": [{
"reader": {
"parameter": {
"password": "123456",
"dataSourceId": 14,
"column": [{
"precision": 20,
"name": "id",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 32,
"name": "device_id",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 12,
"name": "point",
"columnDisplaySize": 12,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "hash",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "value",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 19,
"name": "acq_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 12,
"name": "ratio",
"columnDisplaySize": 12,
"type": "FLOAT"
}, {
"precision": 32,
"name": "max_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "min_val",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "out_param",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "item_code",
"columnDisplaySize": 8,
"type": "VARCHAR"
}, {
"precision": 32,
"name": "descr",
"columnDisplaySize": 32,
"type": "VARCHAR"
}, {
"precision": 16,
"name": "save_hst",
"columnDisplaySize": 16,
"type": "SMALLINT"
}, {
"precision": 20,
"name": "create_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 20,
"name": "update_user",
"columnDisplaySize": 20,
"type": "BIGINT"
}, {
"precision": 19,
"name": "create_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 19,
"name": "update_time",
"columnDisplaySize": 19,
"type": "DATETIME"
}, {
"precision": 50,
"name": "ext_first",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_second",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_third",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fourth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_fifth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_sixth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_seventh",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_eighth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_ninth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "ext_tenth",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 50,
"name": "device_add_id",
"columnDisplaySize": 50,
"type": "VARCHAR"
}, {
"precision": 8,
"name": "today_start_value",
"columnDisplaySize": 10,
"type": "DECIMAL"
}, {
"precision": 75,
"name": "unit",
"columnDisplaySize": 75,
"type": "VARCHAR"
}],
"connection": [{
"jdbcUrl": ["jdbc:mysql://172.18.8.209:3306/Vasyslink_yag001?useSSL=false&useUnicode=true&characterEncoding=utf8&useCursorFetch=true"],
"table": ["tb_device_point_data_hst_01"]
}],

"splitPk": "id",
"username": "root"
},
"name": "mysqlreader"
},
"writer": {
"parameter": {
"tablesColumn": "{\"tb_device_point_data_hst_996\":[{\"key\":\"id\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"device_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"point\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"hash\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"value\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"acq_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ratio\",\"type\":\"float\",\"precision\":7,\"columnDisplaySize\":24},{\"key\":\"max_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"min_val\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"out_param\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"item_code\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"descr\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"save_hst\",\"type\":\"smallint\",\"precision\":5,\"columnDisplaySize\":6},{\"key\":\"create_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"update_user\",\"type\":\"bigint\",\"precision\":19,\"columnDisplaySize\":20},{\"key\":\"create_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"update_time\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_first\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_second\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_third\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fourth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_fifth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_sixth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_seventh\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_eighth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_ninth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"ext_tenth\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"device_add_id\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647},{\"key\":\"today_start_value\",\"type\":\"decimal\",\"precision\":8,\"columnDisplaySize\":10},{\"key\":\"unit\",\"type\":\"string\",\"precision\":2147483647,\"columnDisplaySize\":2147483647}]}",
"dataSourceId": 54,
"partition": "pt",
"jdbcUrl": "jdbc:hive2://172.18.8.208:10000/Vasyslink_yag001",
"defaultFS": "hdfs://172.18.8.207:8020",
"writeMode": "overwrite",
"maxFileSize": 1073741824,

"fieldDelimiter": "\t",

"partitionType" : "DAY",
"fileType": "text",
"charsetName": "UTF-8"
},
"name": "hivewriter"
}
}],
"setting": {
"log": {
"isLogger": false
},
"errorLimit": {},
"speed": {

"channel": 10
}
}
}
}

### Anything else

_No response_

### Version

master

### Are you willing to submit PR?

- [X] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.