apache / apache/tvm

[Bug] InternalError: Check failed: (ret_ex.dtype() == var.dtype()) is false

Open
#17,937 0 comments 0 reactions 0 assignees View on GitHub
needs-triage type: bug
Dominant language
Python
Stars
13.7k
Forks
4k
Avg merge
2d 1h
Merged PRs (30d)
112

Description

### Actual behavior

```
Traceback (most recent call last):
File "/data/qshenaf/remote_pc/TirFuzz/bugs/05-03_20-50/topi.nn.group_norm_4.py", line 20, in
tvm.build(opt_mod, target='llvm')
File "/data/qshenaf/envs/tvm/python/tvm/driver/build_module.py", line 59, in build
return tvm.tir.build(mod, target, pipeline)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/python/tvm/tir/build.py", line 173, in build
mod = pipeline(mod)
^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/python/tvm/ir/transform.py", line 238, in __call__
return _ffi_transform_api.RunPass(self, mod)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "tvm/_ffi/_cython/./packed_func.pxi", line 339, in tvm._ffi._cy3.core.PackedFuncBase.__call__
File "tvm/_ffi/_cython/./packed_func.pxi", line 270, in tvm._ffi._cy3.core.FuncCall
File "tvm/_ffi/_cython/./packed_func.pxi", line 259, in tvm._ffi._cy3.core.FuncCall3
File "tvm/_ffi/_cython/./base.pxi", line 185, in tvm._ffi._cy3.core.CHECK_CALL
File "/data/qshenaf/envs/tvm/python/tvm/_ffi/base.py", line 468, in raise_last_ffi_error
raise py_err
File "tvm/_ffi/_cython/./packed_func.pxi", line 56, in tvm._ffi._cy3.core.tvm_callback
File "/data/qshenaf/envs/tvm/python/tvm/tir/pipeline.py", line 122, in _pipeline
mod = tvm.ir.transform.Sequential(passes)(mod)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/python/tvm/ir/transform.py", line 238, in __call__
return _ffi_transform_api.RunPass(self, mod)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "tvm/_ffi/_cython/./packed_func.pxi", line 339, in tvm._ffi._cy3.core.PackedFuncBase.__call__
File "tvm/_ffi/_cython/./packed_func.pxi", line 270, in tvm._ffi._cy3.core.FuncCall
File "tvm/_ffi/_cython/./packed_func.pxi", line 259, in tvm._ffi._cy3.core.FuncCall3
File "tvm/_ffi/_cython/./base.pxi", line 185, in tvm._ffi._cy3.core.CHECK_CALL
File "/data/qshenaf/envs/tvm/src/tir/ir/transform.cc", line 121, in tvm::tir::transform::PrimFuncPassNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
func = pass_func(std::move(func), mod, pass_ctx);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc", line 255, in operator()
return PlanAndUpdateBufferAllocationLocation(std::move(f));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc", line 247, in tvm::tir::PlanAndUpdateBufferAllocationLocation(tvm::tir::PrimFunc)
fptr->body = locator(fptr->body);
^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc", line 178, in tvm::tir::BufferAllocationLocator::VisitStmt_(tvm::tir::BlockNode const*)
Stmt stmt = StmtMutator::VisitStmt_(op);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 211, in tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)
return MutateArray(self, arr, fmutate);
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 184, in tvm::runtime::Array::value, void>::type> tvm::tir::StmtMutator::Internal::MutateArray const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator*, tvm::runtime::Array::value, void>::type> const&, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Arrayconst&)::{lambda(tvm::tir::Stmt const&)#1})
Array copy = arr.Map(fmutate);
^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/include/tvm/runtime/container/array.h", line 652, in tvm::runtime::Array::value, void>::type> tvm::runtime::Array::Map const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1}) const
return Array(MapHelper(data_, fmap));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/include/tvm/runtime/container/array.h", line 877, in tvm::runtime::ObjectPtr tvm::runtime::Array::MapHelper const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::runtime::ObjectPtr, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1})
U mapped = fmap(DowncastNoCheck(*it));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 210, in tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1}::operator()(tvm::tir::Stmt const&) const
auto fmutate = [self](const Stmt& s) { return self->VisitStmt(s); };
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc", line 145, in tvm::tir::BufferAllocationLocator::VisitStmt_(tvm::tir::ForNode const*)
auto node = Downcast(StmtMutator::VisitStmt_(op));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 211, in tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)
return MutateArray(self, arr, fmutate);
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 184, in tvm::runtime::Array::value, void>::type> tvm::tir::StmtMutator::Internal::MutateArray const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator*, tvm::runtime::Array::value, void>::type> const&, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Arrayconst&)::{lambda(tvm::tir::Stmt const&)#1})
Array copy = arr.Map(fmutate);
^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/include/tvm/runtime/container/array.h", line 652, in tvm::runtime::Array::value, void>::type> tvm::runtime::Array::Map const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1}) const
return Array(MapHelper(data_, fmap));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/include/tvm/runtime/container/array.h", line 877, in tvm::runtime::ObjectPtr tvm::runtime::Array::MapHelper const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::runtime::ObjectPtr, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1})
U mapped = fmap(DowncastNoCheck(*it));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 210, in tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1}::operator()(tvm::tir::Stmt const&) const
auto fmutate = [self](const Stmt& s) { return self->VisitStmt(s); };
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc", line 156, in tvm::tir::BufferAllocationLocator::VisitStmt_(tvm::tir::ForNode const*)
node.CopyOnWrite()->body = InjectOpaqueBlock(node->body, new_block_alloc_bufs);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc", line 219, in tvm::tir::BufferAllocationLocator::InjectOpaqueBlock(tvm::tir::Stmt, tvm::runtime::Array const&)
GetBlockReadWriteRegion(opaque_block, buffer_data_to_buffer_);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/analysis/block_access_region_detector.cc", line 385, in tvm::tir::GetBlockReadWriteRegion(tvm::tir::Block const&, tvm::runtime::Map const&)
detector(block);
^
File "/data/qshenaf/envs/tvm/src/tir/analysis/block_access_region_detector.cc", line 135, in tvm::tir::BlockReadWriteDetector::operator()(tvm::tir::Stmt const&)
StmtExprVisitor::operator()(stmt);
^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 144, in tvm::tir::StmtVisitor::VisitStmt_(tvm::tir::BlockNode const*)
this->VisitStmt(op->body);
^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 119, in tvm::tir::StmtVisitor::VisitStmt_(tvm::tir::SeqStmtNode const*)
VisitArray(op->seq, [this](const Stmt& s) { this->VisitStmt(s); });
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/functor_common.h", line 35, in VisitArray >
fvisit(arr[i]);
^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 119, in operator()
VisitArray(op->seq, [this](const Stmt& s) { this->VisitStmt(s); });
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/analysis/block_access_region_detector.cc", line 167, in tvm::tir::BlockReadWriteDetector::VisitStmt_(tvm::tir::ForNode const*)
StmtVisitor::VisitStmt_(op);
^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 48, in tvm::tir::StmtVisitor::VisitStmt_(tvm::tir::ForNode const*)
this->VisitStmt(op->body);
^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/analysis/block_access_region_detector.cc", line 258, in tvm::tir::BlockReadWriteDetector::VisitStmt_(tvm::tir::BlockRealizeNode const*)
Substitute(range->min, vmap), Substitute(range->extent, vmap))),
^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 758, in tvm::tir::Substitute(tvm::PrimExpr, std::function (tvm::tir::Var const&)>)
return IRSubstitute(vmap)(std::move(expr));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 660, in tvm::tir::IRSubstitute::VisitExpr_(tvm::tir::VarNode const*)
ICHECK(ret_ex.dtype() == var.dtype()) << "substituting " << var << ":" << var.dtype()
^^^^^^^^^^^^^^^^^^^^^^^^^^^
tvm.error.InternalError: Traceback (most recent call last):
37: tvm::tir::transform::PrimFuncPassNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
at /data/qshenaf/envs/tvm/src/tir/ir/transform.cc:121
36: operator()
at /data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc:255
35: tvm::tir::PlanAndUpdateBufferAllocationLocation(tvm::tir::PrimFunc)
at /data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc:247
34: tvm::tir::BufferAllocationLocator::VisitStmt_(tvm::tir::BlockNode const*)
at /data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc:178
33: tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:211
32: tvm::runtime::Array::value, void>::type> tvm::tir::StmtMutator::Internal::MutateArray const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator*, tvm::runtime::Array::value, void>::type> const&, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1})
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:184
31: tvm::runtime::Array::value, void>::type> tvm::runtime::Array::Map const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1}) const
at /data/qshenaf/envs/tvm/include/tvm/runtime/container/array.h:652
30: tvm::runtime::ObjectPtr tvm::runtime::Array::MapHelper const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::runtime::ObjectPtr, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1})
at /data/qshenaf/envs/tvm/include/tvm/runtime/container/array.h:877
29: tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1}::operator()(tvm::tir::Stmt const&) const
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:210
28: tvm::tir::BufferAllocationLocator::VisitStmt_(tvm::tir::ForNode const*)
at /data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc:145
27: tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:211
26: tvm::runtime::Array::value, void>::type> tvm::tir::StmtMutator::Internal::MutateArray const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator*, tvm::runtime::Array::value, void>::type> const&, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1})
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:184
25: tvm::runtime::Array::value, void>::type> tvm::runtime::Array::Map const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1}) const
at /data/qshenaf/envs/tvm/include/tvm/runtime/container/array.h:652
24: tvm::runtime::ObjectPtr tvm::runtime::Array::MapHelper const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::runtime::ObjectPtr, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1})
at /data/qshenaf/envs/tvm/include/tvm/runtime/container/array.h:877
23: tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array const&)::{lambda(tvm::tir::Stmt const&)#1}::operator()(tvm::tir::Stmt const&) const
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:210
22: tvm::tir::BufferAllocationLocator::VisitStmt_(tvm::tir::ForNode const*)
at /data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc:156
21: tvm::tir::BufferAllocationLocator::InjectOpaqueBlock(tvm::tir::Stmt, tvm::runtime::Array const&)
at /data/qshenaf/envs/tvm/src/tir/transforms/plan_update_buffer_allocation_location.cc:219
20: tvm::tir::GetBlockReadWriteRegion(tvm::tir::Block const&, tvm::runtime::Map const&)
at /data/qshenaf/envs/tvm/src/tir/analysis/block_access_region_detector.cc:385
19: tvm::tir::BlockReadWriteDetector::operator()(tvm::tir::Stmt const&)
at /data/qshenaf/envs/tvm/src/tir/analysis/block_access_region_detector.cc:135
18: tvm::tir::StmtVisitor::VisitStmt_(tvm::tir::BlockNode const*)
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:144
17: tvm::tir::StmtVisitor::VisitStmt_(tvm::tir::SeqStmtNode const*)
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:119
16: VisitArray >
at /data/qshenaf/envs/tvm/src/tir/ir/functor_common.h:35
15: operator()
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:119
14: tvm::tir::BlockReadWriteDetector::VisitStmt_(tvm::tir::ForNode const*)
at /data/qshenaf/envs/tvm/src/tir/analysis/block_access_region_detector.cc:167
13: tvm::tir::StmtVisitor::VisitStmt_(tvm::tir::ForNode const*)
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:48
12: tvm::tir::BlockReadWriteDetector::VisitStmt_(tvm::tir::BlockRealizeNode const*)
at /data/qshenaf/envs/tvm/src/tir/analysis/block_access_region_detector.cc:258
11: tvm::tir::Substitute(tvm::PrimExpr, std::function (tvm::tir::Var const&)>)
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:758
10: _ZThn16_N3tvm3tir15StmtExprMu
9: _ZThn16_N3tvm3tir15StmtExprMu
8: _ZThn16_N3tvm3tir15StmtExprMu
7: _ZThn16_N3tvm3tir15StmtExprMu
6: _ZThn16_N3tvm3tir15StmtExprMu
5: _ZThn16_N3tvm3tir15StmtExprMu
4: _ZThn16_N3tvm3tir15StmtExprMu
3: _ZThn16_N3tvm3tir15StmtExprMu
2: _ZThn16_N3tvm3tir15StmtExprMu
1: _ZThn16_N3tvm3tir12IRSubstitu
0: tvm::tir::IRSubstitute::VisitExpr_(tvm::tir::VarNode const*)
at /data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc:660
File "/data/qshenaf/envs/tvm/src/tir/ir/stmt_functor.cc", line 660
InternalError: Check failed: (ret_ex.dtype() == var.dtype()) is false: substituting v_ax4:int32 -> T.Ramp(0, 1, 3):int32x3

```

### Environment

tvm-0.21.dev0

### Steps to reproduce

```
import tvm
from tvm import te, topi, tir
from tvm import meta_schedule as ms

data = te.placeholder((2, 48, 28, 28, 3), dtype='float16', name='data')
gamma = te.placeholder((48,), dtype='float16', name='gamma')
beta = te.placeholder((48,), dtype='float16', name='beta')
op_config = {'data': data, 'gamma': gamma, 'beta': beta, 'num_groups': 12, 'channel_axis': 1, 'axes': [2, 3, 4], 'epsilon': 1e-08, }
op_output = topi.nn.group_norm(**op_config)

sch = tir.Schedule(te.create_prim_func([data, gamma, beta, op_output]).with_attr('target', tvm.target.Target('llvm')))

database = ms.tir_integration.tune_tir(sch.mod, target='llvm --num-cores=16', work_dir='./tune_tmp', max_trials_global=1, num_trials_per_iter=1)
sch = ms.tir_integration.compile_tir(database, sch.mod, 'llvm --num-cores=16')

passes = [tir.transform.VectorizeLoop(True),tir.transform.InjectVirtualThread()]
with tvm.ir.transform.PassContext(opt_level=4):
opt_mod = tvm.ir.transform.Sequential(passes)(sch.mod)
tvm.build(opt_mod, target='llvm')

```

### Triage

* needs-triage
* tir:transform
* tir:schedule
*tune:meta_schedule

Contributor guide

No contributing guide indexed for this repository

Research direction

Start with the reported topi.nn.group_norm_4.py reproduction and the tvm.build call, then trace the failure through src/tir/transforms/plan_update_buffer_allocation_location.cc and src/tir/analysis/block_access_region_detector.cc. Inspect the dtype substitution path at src/tir/ir/stmt_functor.cc and determine a focused regression test. Done means the reproduction builds without the reported InternalError and the regression test passes.

Written by the indexing model from the issue text.

Assessment

Tech stack
cpp, python
Domain
compilers
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
28/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.