[c++] arrow::json::TableReader::Make ignores nullable=False in explicit_schema parse_options
- Dominant language
- C++
- Stars
- 17.1k
- Forks
- 4.3k
- Avg merge
- 3d 13h
- Merged PRs (30d)
- 88
Description
When using arrow::json::TableReader::Make and sending explicit schema with one of it fields is struct of REQURIED field, the nullability is not passing to the parsed column type. see test:
```
#include
#include
#include
#include
#include
#include
#include "parquet/schema.h"
#include
#include
#include
#include
#include
#include
std::tuple, std::shared_ptr>
CreateProperties() {
parquet::WriterProperties::Builder properties_builder;
properties_builder.max_row_group_length(std::size_t(parquet::DEFAULT_MAX_ROW_GROUP_LENGTH));
properties_builder.data_pagesize(std::size_t(parquet::kDefaultDataPageSize));
properties_builder.write_batch_size(std::size_t(parquet::DEFAULT_WRITE_BATCH_SIZE));
auto properties = properties_builder.build();
parquet::ArrowWriterProperties::Builder arrow_properties_builder;
auto arrow_properties = arrow_properties_builder.build();
return std::make_tuple(properties, arrow_properties);
}
```
```
int main() {
auto schema = arrow::schema({
arrow::field("nullable_field", arrow::int32(), true),
arrow::field("non_nullable_field", arrow::int32(), false),
arrow::field("struct_1", arrow::struct_({
arrow::field("non_nullable_struct_field", arrow::int32(), false) //**This nullability is ignored
}))
});
std::string json_data = R"(
{"nullable_field": 1, "non_nullable_field": 2, "struct_1": {"non_nullable_struct_field": 7}}
)";
auto buffer = std::make_shared(json_data);
arrow::json::ParseOptions parse_options = arrow::json::ParseOptions::Defaults();
arrow::json::ReadOptions read_options = arrow::json::ReadOptions::Defaults();
read_options.use_threads = false;
read_options.block_size = json_data.size();
parse_options.explicit_schema = schema;
auto input = std::make_shared(buffer);
auto json_reader_result = arrow::json::TableReader::Make(arrow::default_memory_pool(), input, read_options, parse_options);
if (!json_reader_result.ok()) {
throw std::runtime_error("Failed to create JSON TableReader: " + json_reader_result.status().ToString());
}
auto json_reader = *json_reader_result;
auto read_table_result = json_reader->Read();
if (!read_table_result.ok()) {
throw std::runtime_error("Failed to read table from JSON: " + read_table_result.status().ToString());
}
std::shared_ptr table = *read_table_result;
auto typed_table = arrow::Table::Make(schema, table->columns());
std::shared_ptr out_stream;
PARQUET_ASSIGN_OR_THROW(out_stream, arrow::io::BufferOutputStream::Create());
auto [properties, arrow_properties] = CreateProperties();
// The fail is in this function it calls "RETURN_NOT_OK(table.Validate());" and fails due to diff in schemas: the "columns" schema is missing the "Nullability" of the inner struct field (uses OPTIONAL as default) but the explicit schema saves it as REQUIRED
PARQUET_THROW_NOT_OK(
parquet::arrow::WriteTable(*typed_table, arrow::default_memory_pool(), out_stream, std::size_t(parquet::DEFAULT_MAX_ROW_GROUP_LENGTH), properties,
arrow_properties)
);
return 0;
}
```
it fails on:
terminate called after throwing an instance of 'parquet::ParquetStatusException'
what(): Invalid: Column data for field 2 with type struct is inconsistent with schema struct
Aborted
### Component(s)
C++
Contributor guide
Assessment
This issue has not been assessed yet.