apache / apache/arrow

[C++] Enable Substrait ReadRel Projection in Acero

Open
#14,965 3 comments 0 reactions 0 assignees View on GitHub
Component: C++ Type: enhancement
Dominant language
C++
Stars
17.1k
Forks
4.3k
Avg merge
3d 18h
Merged PRs (30d)
91

Description

### Describe the enhancement requested

At the moment, Acero-Substrait integration doesn't support projection in the `ReadRel`. To enable this of the main contibutions is to consume `MaskExpressions` in Acero. A sample Substrait plan is shown below. At the moment this plan cannot be consumed using Acero.

```json
{
"extensions": [
{
"extensionFunction": {
"functionAnchor": 1,
"name": "equal"
}
}
],
"relations": [
{
"root": {
"input": {
"project": {
"input": {
"project": {
"input": {
"join": {
"left": {
"read": {
"baseSchema": {
"names": [
"o_orderkey",
"o_custkey",
"o_orderstatus",
"o_totalprice",
"o_orderdate",
"o_orderpriority",
"o_clerk",
"o_shippriority",
"o_comment"
],
"struct": {
"types": [
{
"i32": {
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"i32": {
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"varchar": {
"length": 1,
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"decimal": {
"scale": 2,
"precision": 15,
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"date": {
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"varchar": {
"length": 15,
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"varchar": {
"length": 15,
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"i32": {
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"varchar": {
"length": 78,
"nullability": "NULLABILITY_NULLABLE"
}
}
],
"nullability": "NULLABILITY_REQUIRED"
}
},
"projection": {
"select": {
"structItems": [
{
"field": 1
}
]
},
"maintainSingularStruct": true
},
"namedTable": {
"names": [
"orders"
]
}
}
},
"right": {
"read": {
"baseSchema": {
"names": [
"c_custkey",
"c_name",
"c_address",
"c_nationkey",
"c_phone",
"c_acctbal",
"c_mktsegment",
"c_comment"
],
"struct": {
"types": [
{
"i32": {
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"varchar": {
"length": 18,
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"varchar": {
"length": 40,
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"i32": {
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"varchar": {
"length": 15,
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"decimal": {
"scale": 2,
"precision": 15,
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"varchar": {
"length": 10,
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"varchar": {
"length": 116,
"nullability": "NULLABILITY_NULLABLE"
}
}
],
"nullability": "NULLABILITY_REQUIRED"
}
},
"projection": {
"select": {
"structItems": [
{}
]
},
"maintainSingularStruct": true
},
"namedTable": {
"names": [
"customer"
]
}
}
},
"expression": {
"scalarFunction": {
"functionReference": 1,
"outputType": {
"bool": {
"nullability": "NULLABILITY_NULLABLE"
}
},
"arguments": [
{
"value": {
"selection": {
"directReference": {
"structField": {}
},
"rootReference": {}
}
}
},
{
"value": {
"selection": {
"directReference": {
"structField": {
"field": 1
}
},
"rootReference": {}
}
}
}
]
}
},
"type": "JOIN_TYPE_INNER"
}
},
"expressions": [
{
"selection": {
"directReference": {
"structField": {}
},
"rootReference": {}
}
},
{
"selection": {
"directReference": {
"structField": {
"field": 1
}
},
"rootReference": {}
}
}
]
}
},
"expressions": [
{
"selection": {
"directReference": {
"structField": {}
},
"rootReference": {}
}
}
]
}
},
"names": [
"o_custkey"
]
}
}
]
}
```

### Component(s)

C++

Contributor guide

Open the contributing guide

Research direction

Start with the Acero-Substrait integration's current ReadRel handling and the sample Substrait plan in this issue, focusing on the projection and its MaskExpressions. Done means Acero can consume the shown plan with projected ReadRel inputs, including the nested projections described in the request.

Written by the indexing model from the issue text.

Assessment

Tech stack
cpp
Domain
data-engineering
Issue type
Feature
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
35/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.