flagos-ai / flagos-ai/FlagTree

[BUG][TLE.DSA][ascend] wrong result when replace tle.dsa.extract_slice with tle.dsa.extract_element

Open
#1,012 3 comments 0 reactions 0 assignees View on GitHub
ascend
Dominant language
Python
Stars
350
Forks
149
Avg merge
2d 4h
Merged PRs (30d)
87

Description

device: Ascend910_9382

flagtree: 0.6.0+ascend3.2

testcase:

The kernel invokes extract_element 8 times during the retrieval of topk_indices, using indices 24, 9, 1, 0, 25, 19, 16, 24 respectively. However, when USE_ELEM=True is set, the dumped results show that the actual indices used in these eight calls are 24, 24, 9, 1, 0, 25, 19, 16.

```
import random
import torch
import triton
import triton.language as tl
import triton.experimental.tle.language as tle

# return max(x, y), min(x, y)
@triton.jit
def _topk_swap(x_val, x_idx, y_val, y_idx):
mask = (x_val > y_val) | ((x_val == y_val) & (x_idx < y_idx))
max_val = tl.where(mask, x_val, y_val)
max_idx = tl.where(mask, x_idx, y_idx)
min_val = tl.where(not mask, x_val, y_val)
min_idx = tl.where(not mask, x_idx, y_idx)
return max_val, max_idx, min_val, min_idx

@triton.jit
def _kernel(
scores_ptr,
bias_ptr,
topk_indices_ptr,
dump_ptr,
dump_ptr2,
topk: tl.constexpr,
USE_ELEM: tl.constexpr,
):
WARP_SIZE: tl.constexpr = 32
NUM_WARPS: tl.constexpr = 8
neg_inf: tl.constexpr = float("-inf")
MAX_IDX: tl.constexpr = 65535
scores_stride0: tl.constexpr = WARP_SIZE * NUM_WARPS

token_id = tl.program_id(0)
scores_ptr += token_id * scores_stride0
dump_ptr += token_id * scores_stride0
dump_ptr2 += token_id * scores_stride0
topk_indices_ptr += token_id * topk

# 1. get score_bias
warps = tl.arange(0, NUM_WARPS)
lane = tl.arange(0, WARP_SIZE)
offs = warps[:, None] * WARP_SIZE + lane[None, :]
score_ub = tle.dsa.alloc([NUM_WARPS, WARP_SIZE], dtype=tl.bfloat16, mem_addr_space=tle.dsa.ascend.UB)
bias_ub = tle.dsa.alloc([NUM_WARPS, WARP_SIZE], dtype=tl.float32, mem_addr_space=tle.dsa.ascend.UB)
tle.dsa.copy(scores_ptr + offs, score_ub, [NUM_WARPS, WARP_SIZE])
tle.dsa.copy(bias_ptr + offs, bias_ub, [NUM_WARPS, WARP_SIZE])
score = tle.dsa.to_tensor(score_ub).to(tl.float32)
bias_val = tle.dsa.to_tensor(bias_ub)
score_bias = score + bias_val

# 2. get topk_group score
# topk_group is [5, 2, 6, 3] with current seed
group_idx0 = 5; group_idx1 = 2; group_idx2 = 6; group_idx3 = 3
expert_idx_group0 = group_idx0 * WARP_SIZE + lane
expert_idx_group1 = group_idx1 * WARP_SIZE + lane
expert_idx_group2 = group_idx2 * WARP_SIZE + lane
expert_idx_group3 = group_idx3 * WARP_SIZE + lane
expert_score_group0 = tle.dsa.extract_slice(score_bias, offsets=(group_idx0, 0), sizes=(1, WARP_SIZE), strides=(1, 1))
expert_score_group0 = tl.reshape(expert_score_group0, WARP_SIZE)
expert_score_group1 = tle.dsa.extract_slice(score_bias, offsets=(group_idx1, 0), sizes=(1, WARP_SIZE), strides=(1, 1))
expert_score_group1 = tl.reshape(expert_score_group1, WARP_SIZE)
expert_score_group2 = tle.dsa.extract_slice(score_bias, offsets=(group_idx2, 0), sizes=(1, WARP_SIZE), strides=(1, 1))
expert_score_group2 = tl.reshape(expert_score_group2, WARP_SIZE)
expert_score_group3 = tle.dsa.extract_slice(score_bias, offsets=(group_idx3, 0), sizes=(1, WARP_SIZE), strides=(1, 1))
expert_score_group3 = tl.reshape(expert_score_group3, WARP_SIZE)

# 3. swap 4 elem per lane
# TOPK_SWAP(0, 2); TOPK_SWAP(1, 3); TOPK_SWAP(0, 1); TOPK_SWAP(2, 3); TOPK_SWAP(1, 2);
expert_score_group0, expert_idx_group0, expert_score_group2, expert_idx_group2 = _topk_swap(
expert_score_group0, expert_idx_group0, expert_score_group2, expert_idx_group2
)
expert_score_group1, expert_idx_group1, expert_score_group3, expert_idx_group3 = _topk_swap(
expert_score_group1, expert_idx_group1, expert_score_group3, expert_idx_group3
)
expert_score_group0, expert_idx_group0, expert_score_group1, expert_idx_group1 = _topk_swap(
expert_score_group0, expert_idx_group0, expert_score_group1, expert_idx_group1
)
expert_score_group2, expert_idx_group2, expert_score_group3, expert_idx_group3 = _topk_swap(
expert_score_group2, expert_idx_group2, expert_score_group3, expert_idx_group3
)
expert_score_group1, expert_idx_group1, expert_score_group2, expert_idx_group2 = _topk_swap(
expert_score_group1, expert_idx_group1, expert_score_group2, expert_idx_group2
)

# 4. get topk
top_experts = tl.full([WARP_SIZE], 0, dtype=tl.int32)
lane_idx = tl.full((), MAX_IDX, dtype=tl.int32)
for kk in tl.static_range(0, topk):
if kk > 0:
if not USE_ELEM:
sub_score3 = tle.dsa.extract_slice(expert_score_group3, offsets=(lane_idx,), sizes=(1,), strides=(1,))
sub_score2 = tle.dsa.extract_slice(expert_score_group2, offsets=(lane_idx,), sizes=(1,), strides=(1,))
sub_score1 = tle.dsa.extract_slice(expert_score_group1, offsets=(lane_idx,), sizes=(1,), strides=(1,))
else:
sub_score3_elem = tle.dsa.extract_element(expert_score_group3, indice=(lane_idx,))
sub_score3 = tl.full([1], sub_score3_elem, dtype=tl.float32)
sub_score2_elem = tle.dsa.extract_element(expert_score_group2, indice=(lane_idx,))
sub_score2 = tl.full([1], sub_score2_elem, dtype=tl.float32)
sub_score1_elem = tle.dsa.extract_element(expert_score_group1, indice=(lane_idx,))
sub_score1 = tl.full([1], sub_score1_elem, dtype=tl.float32)
invalid_score = tl.full([1], neg_inf, dtype=tl.float32)
expert_score_group3 = tle.dsa.insert_slice(expert_score_group3, invalid_score, offsets=(lane_idx,), sizes=(1,), strides=(1,))
expert_score_group2 = tle.dsa.insert_slice(expert_score_group2, sub_score3, offsets=(lane_idx,), sizes=(1,), strides=(1,))
expert_score_group1 = tle.dsa.insert_slice(expert_score_group1, sub_score2, offsets=(lane_idx,), sizes=(1,), strides=(1,))
expert_score_group0 = tle.dsa.insert_slice(expert_score_group0, sub_score1, offsets=(lane_idx,), sizes=(1,), strides=(1,))

if not USE_ELEM:
sub_idx3 = tle.dsa.extract_slice(expert_idx_group3, offsets=(lane_idx,), sizes=(1,), strides=(1,))
sub_idx2 = tle.dsa.extract_slice(expert_idx_group2, offsets=(lane_idx,), sizes=(1,), strides=(1,))
sub_idx1 = tle.dsa.extract_slice(expert_idx_group1, offsets=(lane_idx,), sizes=(1,), strides=(1,))
else:
sub_idx3_elem = tle.dsa.extract_element(expert_idx_group3, indice=(lane_idx,))
sub_idx3 = tl.full([1], sub_idx3_elem, dtype=tl.int32)
sub_idx2_elem = tle.dsa.extract_element(expert_idx_group2, indice=(lane_idx,))
sub_idx2 = tl.full([1], sub_idx2_elem, dtype=tl.int32)
sub_idx1_elem = tle.dsa.extract_element(expert_idx_group1, indice=(lane_idx,))
sub_idx1 = tl.full([1], sub_idx1_elem, dtype=tl.int32)
invalid_idx = tl.full([1], MAX_IDX, dtype=tl.int32)
expert_idx_group3 = tle.dsa.insert_slice(expert_idx_group3, invalid_idx, offsets=(lane_idx,), sizes=(1,), strides=(1,))
expert_idx_group2 = tle.dsa.insert_slice(expert_idx_group2, sub_idx3, offsets=(lane_idx,), sizes=(1,), strides=(1,))
expert_idx_group1 = tle.dsa.insert_slice(expert_idx_group1, sub_idx2, offsets=(lane_idx,), sizes=(1,), strides=(1,))
expert_idx_group0 = tle.dsa.insert_slice(expert_idx_group0, sub_idx1, offsets=(lane_idx,), sizes=(1,), strides=(1,))

_2, lane_idx = tl.max(expert_score_group0, axis=-1, return_indices=True, return_indices_tie_break_left=True)
tl.store(dump_ptr + 32 * kk + lane, expert_idx_group0)
tl.store(dump_ptr2 + kk, lane_idx)
out_idx = tle.dsa.extract_element(expert_idx_group0, indice=(lane_idx,))
tl.store(dump_ptr2 + topk + kk, out_idx)
out_idx_tensor = tl.full([1], out_idx, dtype=tl.int32)
top_experts = tle.dsa.insert_slice(top_experts, out_idx_tensor, offsets=(kk,), sizes=(1,), strides=(1,))

tl.store(topk_indices_ptr + lane, top_experts, mask=lane < topk)

torch.manual_seed(42)
torch.cuda.manual_seed(42)
device = "npu"
n_token = 1
n_expert = 256
n_group = 8
topk = 8
topk_group = 4
routed_scaling_factor = 1.0

scores = torch.randn((n_token, n_expert), dtype=torch.bfloat16, device=device)
bias = torch.randn((n_expert,), dtype=torch.float32, device=device)
top_k_indices = torch.empty((n_token, topk), dtype=torch.int32, device=device)
dump_buf = torch.zeros((n_token, n_expert), dtype=torch.int32, device=device)
dump_buf2 = torch.zeros((n_token, n_expert), dtype=torch.int32, device=device)

_kernel[(n_token,)](scores, bias, top_k_indices, dump_buf, dump_buf2, topk, USE_ELEM=True, num_warps=1)

#print(f"top_k_indices: {top_k_indices}")
#print("")
for i in range(topk):
#print(f"========= {i} ==========")
extract_source = dump_buf[0, 32*i: 32*(i+1)]
extract_offset = dump_buf2[0, i]
extract_result = dump_buf2[0, topk + i]
assert extract_result == extract_source[extract_offset] # Fail if USE_ELEM=True
#print(f"extract source: {extract_source}")
#print(f"extract offset: {extract_offset}")
#print(f"extract result: {extract_result}, expected result: {extract_source[extract_offset]}")
#print("")
#print("")
```

Contributor guide

Open the contributing guide

Research direction

Start by running the inline Python reproducer on the specified Ascend910_9382 device and compare the USE_ELEM=True extract_element path with the extract_slice path. Trace the TLE.DSA extract_element lowering and its handling of lane_idx, then use the final assertion and dumped indices to verify that all eight extracted values match their source positions.

Written by the indexing model from the issue text.

Assessment

Tech stack
python
Domain
compilers
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Active
Clarity
Mostly clear
Newbie friendliness
64/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.