insert with static partition config leads to wrong schema being written
- Dominant language
- Java
- Stars
- 6.2k
- Forks
- 2.5k
- Avg merge
- 2d 8h
- Merged PRs (30d)
- 111
Description
{code:java}
test("Test enable hoodie.datasource.write.drop.partition.columns when write") {
withSQLConf("hoodie.sql.bulk.insert.enable" -> "false") {
Seq("cow").foreach { tableType =>
withRecordType()(withTempDir { tmp =>
val tableName = generateTableName
spark.sql(
s"""
| create table $tableName (
| id int,
| name string,
| price double,
| ts long,
| dt string
| ) using hudi
| partitioned by (dt)
| location '${tmp.getCanonicalPath}/$tableName'
| tblproperties (
| primaryKey = 'id',
| preCombineField = 'ts',
| type = '$tableType',
| hoodie.datasource.write.drop.partition.columns = 'true'
| )
""".stripMargin)
spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a1', 10, 1000)")
spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (2, 'a2', 20, 1000)")
checkAnswer(s"select id, name, price, ts, dt from $tableName")(
Seq(1, "a1", 10, 1000, "2021-12-25"),
Seq(2, "a2", 20, 1000, "2021-12-25")
)
})
}
}
} {code}
extra commit metadata does not contain the partition column `dt`
{code:java}
"schema": "{\"type\":\"record\",\"name\":\"h2_record\",\"namespace\":\"hoodie.h2\",\"fields\":[{\"name\":\"id\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"price\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"ts\",\"type\":[\"null\",\"long\"],\"default\":null}]}"{code}
## JIRA info
- Link: https://issues.apache.org/jira/browse/HUDI-8956
- Type: Bug
- Fix version(s):
- 1.1.0
---
## Comments
11/Feb/25 08:23;ktblsva;but why column dt should be in schema if you use this property?
hoodie.datasource.write.drop.partition.columns = 'true';;;
Contributor guide
No contributing guide indexed for this repository
Assessment
This issue has not been assessed yet.