Dq engine could not be used with custom udfs
- Dominant language
- C++
- Stars
- 2.2k
- Forks
- 219
- PR merge metrics
- No merged PRs in 30d
Description
Hi! Encountered an error with dq engine. It seems that dq does not recognizes custom udfs supplied in query and fails. Wrote a simple udf:
MySum UDF Module
```cpp
#include
#include
#include
#include
using namespace NKikimr;
using namespace NUdf;
class TSumFunction: public TBoxedValue {
public:
static const TStringRef& Name() {
static auto name = ::TStringRef::Of("Sum");
return name;
}
static bool DeclareSignature(
bool typesOnly,
IFunctionTypeInfoBuilder& builder,
TType* userType
) {
Y_UNUSED(userType);
builder.SimpleSignature();
if (!typesOnly) {
builder.Implementation(new TSumFunction);
}
return true;
}
private:
TUnboxedValue Run(
const IValueBuilder* valueBuilder,
const TUnboxedValuePod* args
) const override {
Y_UNUSED(valueBuilder);
return TUnboxedValuePod(args[0].Get() + args[1].Get());
}
};
class TMySumModule: public IUdfModule {
public:
void CleanupOnTerminate() const final {}
TStringRef Name() const {
return TStringRef::Of("MySum");
}
void GetAllFunctions(IFunctionsSink& sink) const final {
sink.Add(TSumFunction::Name());
}
void BuildFunctionTypeInfo(
const TStringRef& name,
TType* userType,
const TStringRef& typeConfig,
ui32 flags,
IFunctionTypeInfoBuilder& builder) const override
{
try {
Y_UNUSED(typeConfig);
if (TSumFunction::Name() == name) {
TSumFunction::DeclareSignature(flags & TFlags::TypesOnly, builder, userType);
}
} catch (std::exception& e) {
builder.SetError(CurrentExceptionMessage());
}
}
};
REGISTER_MODULES(TMySumModule)
```
And trying to use it in query
```
USE cluster;
PRAGMA File("MySum", "yt://example.cluster.com/path/to/udf.so");
PRAGMA Udf("MySum");
SELECT MySum::Sum(1, 2)
from //path/to/table
```
And it fails with an error `contrib/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp:693 BuildUploadList(): requirement udfPathWithMd5.Defined() failed`
Setting pragma DqEngine = 'disabled' solves the issue. We are currently using version 0.0.10 of query-tracker
Contributor guide
Assessment
This issue has not been assessed yet.