DTStack / DTStack/chunjun

flinkx1.12 mysql同步hive 无法断点续传(yarn-perjob模式)

Open
#1,139 7 comments 0 reactions 0 assignees View on GitHub
bug
Dominant language
Java
Stars
4.1k
Forks
1.7k
PR merge metrics
No merged PRs in 30d

Description

### Search before asking

- [X] I had searched in the [issues](https://github.com/DTStack/chunjun/issues) and found no similar issues.

### What happened

![1660201162(1)](https://user-images.githubusercontent.com/34857750/184080171-68db0352-f5b5-4689-baa2-ce546106b580.png)
{
"job": {
"setting": {
"errorLimit": {},
"speed": {
"channel": 1,
"bytes": 0
},
"log": {
"isLogger": false
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"column": [
{
"name": "id",
"type": "INT",
"precision": 10,
"columnDisplaySize": 10
},
{
"name": "address",
"type": "VARCHAR"

}
],
"username": "root",
"password": "123456",
"increColumn":"id",

"connection": [
{
"table": [
"mysqlreader2"
],
"jdbcUrl": [
"jdbc:mysql://172.18.8.77:3306/zk_test"
]
}
],
"dataSourceId": 21
}
},
"writer": {
"name": "hivewriter",
"parameter": {
"jdbcUrl": "jdbc:hive2://172.18.8.208:10000/Vasyslink_yag001",
"fileType": "text",
"fieldDelimiter": "\t",
"writeMode": "append",
"charsetName": "UTF-8",
"maxFileSize": 1073741824,
"tablesColumn": "{\"dept22\":[{\"key\":\"deptno\",\"type\":\"int\",\"precision\":10,\"columnDisplaySize\":11},{\"key\":\"address\",\"type\":\"string\"}]}",
"defaultFS": "hdfs://172.18.8.207:8020",
"dataSourceId": 15,
"partition": "pt",
"partitionType": "USERDEFINED",
"partitionValue": "2022041000"
}
}
}
]
}
}

### What you expected to happen

下次执行从上次记录位置开始执行

### How to reproduce

执行json

{
"job": {
"setting": {
"errorLimit": {},
"speed": {
"channel": 1,
"bytes": 0
},
"log": {
"isLogger": false
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"column": [
{
"name": "id",
"type": "INT",
"precision": 10,
"columnDisplaySize": 10
},
{
"name": "address",
"type": "VARCHAR"

}
],
"username": "root",
"password": "123456",
"increColumn":"id",

"connection": [
{
"table": [
"mysqlreader2"
],
"jdbcUrl": [
"jdbc:mysql://172.18.8.77:3306/zk_test"
]
}
],
"dataSourceId": 21
}
},
"writer": {
"name": "hivewriter",
"parameter": {
"jdbcUrl": "jdbc:hive2://172.18.8.208:10000/Vasyslink_yag001",
"fileType": "text",
"fieldDelimiter": "\t",
"writeMode": "append",
"charsetName": "UTF-8",
"maxFileSize": 1073741824,
"tablesColumn": "{\"dept22\":[{\"key\":\"deptno\",\"type\":\"int\",\"precision\":10,\"columnDisplaySize\":11},{\"key\":\"address\",\"type\":\"string\"}]}",
"defaultFS": "hdfs://172.18.8.207:8020",
"dataSourceId": 15,
"partition": "pt",
"partitionType": "USERDEFINED",
"partitionValue": "2022041000"
}
}
}
]
}
}

### Anything else

_No response_

### Version

1.12_release

### Are you willing to submit PR?

- [X] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.