RuntimeError: PassManager::run failed

I get the following error when i try to use PLAID from pylate which is using fastkmeans , i get this error before the last week, but solve it with downgrade for pytorch from 2.7 to 2.6 but it didn't work now with me!

My GPU is Titan RTX

+ nvidia-cublas-cu12==12.6.4.
+ nvidia-cuda-cupti-cu12==12.6.80
+ nvidia-cuda-nvrtc-cu12==12.6.77
+ nvidia-cuda-runtime-cu12==12.6.77
+ nvidia-cudnn-cu12==9.5.1.17
+ nvidia-cufft-cu12==11.3.0.4
+ nvidia-curand-cu12==10.3.7.77
+ nvidia-cusolver-cu12==11.7.1.2
+ nvidia-cusparse-cu12==12.5.4.2
+ nvidia-cusparselt-cu12==0.6.3
+ nvidia-nccl-cu12==2.26.2
+ nvidia-nvjitlink-cu12==12.6.85
+ nvidia-nvtx-cu12==12.6.77
+ sympy==1.14.0
+ torch==2.7.1
+ triton==3.3.1
```python
from pylate import indexes, models, retrieve

index = indexes.PLAID(
index_folder="pylate-index",
index_name="index",
override=True,
)

retriever = retrieve.ColBERT(index=index)
documents_ids = ["1", "2", "3"]

documents = [
"ColBERT’s late-interaction keeps token-level embeddings to deliver cross-encoder-quality ranking at near-bi-encoder speed, enabling fine-grained relevance, robustness across domains, and hardware-friendly scalable search.",

"PLAID compresses ColBERT token vectors via product quantization to shrink storage by 10×, uses two-stage centroid scoring for sub-200 ms latency, and plugs directly into existing ColBERT pipelines.",

"PyLate is a library built on top of Sentence Transformers, designed to simplify and optimize fine-tuning, inference, and retrieval with state-of-the-art ColBERT models. It enables easy fine-tuning on both single and multiple GPUs, providing flexibility for various hardware setups. PyLate also streamlines document retrieval and allows you to load a wide range of models, enabling you to construct ColBERT models from most pre-trained language models.",
]

# Encode the documents
documents_embeddings = model.encode(
documents,
batch_size=32,
is_query=False, # Encoding documents
show_progress_bar=True,
)

# Add the documents ids and embeddings to the PLAID index
index.add_documents(
documents_ids=documents_ids,
documents_embeddings=documents_embeddings,
)
```

[Jun 07, 22:10:48] #> Note: Output directory pylate-index/index already exists

{
"ncells": 8,
"centroid_score_threshold": 0.35,
"ndocs": 8192,
"load_index_with_mmap": false,
"search_batch_size": 262144,
"index_path": null,
"index_bsize": 1,
"nbits": 2,
"kmeans_niters": 4,
"resume": false,
"dim": 128,
"index_name": "index",
"overwrite": true,
"root": "pylate-index",
"experiment": "default",
"index_root": "pylate-index",
"name": "2025-06\/07\/22.09.21",
"rank": 0,
"nranks": 1,
"amp": true,
"gpus": 2,
"avoid_fork_if_possible": true
}
[Jun 07, 22:10:48] [0] # of sampled PIDs = 3 sampled_pids[:3] = [1, 0, 2]
[Jun 07, 22:10:48] [0] avg_doclen_est = 68.66666412353516 len(doclens) = 3
[Jun 07, 22:10:48] [0] Creating 128 partitions.
[Jun 07, 22:10:48] [0] *Estimated* 205 embeddings.
[Jun 07, 22:10:48] [0] #> Saving the indexing plan to pylate-index/index/plan.json ..

Unsupported conversion from f16 to f16
LLVM ERROR: Unsupported rounding mode for conversion.
#blocked = #ttg.blocked<{sizePerThread = [4, 4], threadsPerWarp = [1, 32], warpsPerCTA = [4, 1], order = [1, 0]}>
#blocked1 = #ttg.blocked<{sizePerThread = [1], threadsPerWarp = [32], warpsPerCTA = [4], order = [0]}>
#blocked2 = #ttg.blocked<{sizePerThread = [1, 8], threadsPerWarp = [8, 4], warpsPerCTA = [4, 1], order = [1, 0]}>
#blocked3 = #ttg.blocked<{sizePerThread = [8, 1], threadsPerWarp = [4, 8], warpsPerCTA = [1, 4], order = [0, 1]}>
#shared = #ttg.swizzled_shared<{vec = 1, perPhase = 1, maxPhase = 1, order = [1, 0]}>
#shared1 = #ttg.swizzled_shared<{vec = 1, perPhase = 1, maxPhase = 1, order = [0, 1]}>
#smem = #ttg.shared_memory
module attributes {"ttg.num-ctas" = 1 : i32, "ttg.num-warps" = 4 : i32, ttg.target = "cuda:75", "ttg.threads-per-warp" = 32 : i32} {
tt.func public @_kmeans_kernel(%arg0: !tt.ptr {tt.divisibility = 16 : i32}, %arg1: !tt.ptr {tt.divisibility = 16 : i32}, %arg2: !tt.ptr {tt.divisibility = 16 : i32}, %arg3: !tt.ptr {tt.divisibility = 16 : i32}, %arg4: !tt.ptr {tt.divisibility = 16 : i32}, %arg5: !tt.ptr {tt.divisibility = 16 : i32}, %arg6: i32, %arg7: i32 {tt.divisibility = 16 : i32}) attributes {noinline = false} {
%cst = arith.constant dense<0.000000e+00> : tensor<128x128xf32, #blocked>
%cst_0 = arith.constant dense<-2.000000e+00> : tensor<128x128xf32, #blocked>
%cst_1 = arith.constant dense<0.000000e+00> : tensor<128xf16, #blocked1>
%c0_i32 = arith.constant 0 : i32
%c32_i32 = arith.constant 32 : i32
%c127_i32 = arith.constant 127 : i32
%c16_i32 = arith.constant 16 : i32
%c128_i32 = arith.constant 128 : i32
%cst_2 = arith.constant dense<128> : tensor<128x1xi32, #blocked2>
%cst_3 = arith.constant dense<0.000000e+00> : tensor<128xf32, #blocked1>
%0 = tt.get_program_id x : i32
%1 = arith.addi %arg6, %c127_i32 : i32
%2 = arith.divsi %1, %c128_i32 : i32
%3 = arith.addi %arg7, %c127_i32 : i32
%4 = arith.divsi %3, %c128_i32 : i32
%5 = arith.muli %4, %c16_i32 : i32
%6 = arith.divsi %0, %5 : i32
%7 = arith.muli %6, %c16_i32 : i32
%8 = arith.subi %2, %7 : i32
%9 = arith.minsi %8, %c16_i32 : i32
%10 = arith.remsi %0, %5 : i32
%11 = arith.remsi %10, %9 : i32
%12 = arith.addi %7, %11 : i32
%13 = arith.divsi %10, %9 : i32
%14 = arith.muli %12, %c128_i32 : i32
%15 = arith.muli %13, %c128_i32 : i32
%16 = tt.make_range {end = 128 : i32, start = 0 : i32} : tensor<128xi32, #blocked1>
%17 = tt.make_range {end = 128 : i32, start = 0 : i32} : tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>>
%18 = tt.splat %14 : i32 -> tensor<128xi32, #blocked1>
%19 = tt.splat %14 : i32 -> tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>>
%20 = arith.addi %18, %16 : tensor<128xi32, #blocked1>
%21 = arith.addi %19, %17 : tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>>
%22 = tt.splat %15 : i32 -> tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>>
%23 = tt.splat %15 : i32 -> tensor<128xi32, #blocked1>
%24 = tt.splat %15 : i32 -> tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked}>>
%25 = arith.addi %22, %17 : tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>>
%26 = arith.addi %23, %16 : tensor<128xi32, #blocked1>
%27 = tt.splat %arg6 : i32 -> tensor<128xi32, #blocked1>
%28 = tt.splat %arg6 : i32 -> tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>>
%29 = arith.cmpi slt, %20, %27 : tensor<128xi32, #blocked1>
%30 = arith.cmpi slt, %21, %28 : tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>>
%31 = tt.splat %arg7 : i32 -> tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>>
%32 = tt.splat %arg7 : i32 -> tensor<128xi32, #blocked1>
%33 = arith.cmpi slt, %25, %31 : tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>>
%34 = arith.cmpi slt, %26, %32 : tensor<128xi32, #blocked1>
%35 = tt.splat %arg1 : !tt.ptr -> tensor<128x!tt.ptr, #blocked1>
%36 = tt.addptr %35, %20 : tensor<128x!tt.ptr, #blocked1>, tensor<128xi32, #blocked1>
%37 = tt.load %36, %29, %cst_1 : tensor<128x!tt.ptr, #blocked1>
%38 = tt.splat %arg3 : !tt.ptr -> tensor<128x!tt.ptr, #blocked1>
%39 = tt.addptr %38, %26 : tensor<128x!tt.ptr, #blocked1>, tensor<128xi32, #blocked1>
%40 = tt.load %39, %34, %cst_1 : tensor<128x!tt.ptr, #blocked1>
%41 = tt.make_range {end = 32 : i32, start = 0 : i32} : tensor<32xi32, #ttg.slice<{dim = 0, parent = #blocked2}>>
%42 = tt.expand_dims %21 {axis = 1 : i32} : tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>> -> tensor<128x1xi32, #blocked2>
%43 = arith.muli %42, %cst_2 : tensor<128x1xi32, #blocked2>
%44 = tt.splat %arg0 : !tt.ptr -> tensor<128x1x!tt.ptr, #blocked2>
%45 = tt.addptr %44, %43 : tensor<128x1x!tt.ptr, #blocked2>, tensor<128x1xi32, #blocked2>
%46 = tt.broadcast %45 : tensor<128x1x!tt.ptr, #blocked2> -> tensor<128x32x!tt.ptr, #blocked2>
%47 = tt.expand_dims %30 {axis = 1 : i32} : tensor<128xi1, #ttg.slice<{dim = 1, parent = #blocked2}>> -> tensor<128x1xi1, #blocked2>
%48 = tt.broadcast %47 : tensor<128x1xi1, #blocked2> -> tensor<128x32xi1, #blocked2>
%49 = tt.expand_dims %25 {axis = 1 : i32} : tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked2}>> -> tensor<128x1xi32, #blocked2>
%50 = arith.muli %49, %cst_2 : tensor<128x1xi32, #blocked2>
%51 = tt.splat %arg2 : !tt.ptr -> tensor<128x1x!tt.ptr, #blocked2>
%52 = tt.addptr %51, %50 : tensor<128x1x!tt.ptr, #blocked2>, tensor<128x1xi32, #blocked2>
%53 = tt.broadcast %52 : tensor<128x1x!tt.ptr, #blocked2> -> tensor<128x32x!tt.ptr, #blocked2>
%54 = tt.expand_dims %33 {axis = 1 : i32} : tensor<128xi1, #ttg.slice<{dim = 1, parent = #blocked2}>> -> tensor<128x1xi1, #blocked2>
%55 = tt.broadcast %54 : tensor<128x1xi1, #blocked2> -> tensor<128x32xi1, #blocked2>
%56 = scf.for %arg8 = %c0_i32 to %c128_i32 step %c32_i32 iter_args(%arg9 = %cst) -> (tensor<128x128xf32, #blocked>) : i32 {
%91 = tt.splat %arg8 : i32 -> tensor<32xi32, #ttg.slice<{dim = 0, parent = #blocked2}>>
%92 = arith.addi %91, %41 : tensor<32xi32, #ttg.slice<{dim = 0, parent = #blocked2}>>
%93 = tt.expand_dims %92 {axis = 0 : i32} : tensor<32xi32, #ttg.slice<{dim = 0, parent = #blocked2}>> -> tensor<1x32xi32, #blocked2>
%94 = tt.broadcast %93 : tensor<1x32xi32, #blocked2> -> tensor<128x32xi32, #blocked2>
%95 = tt.addptr %46, %94 : tensor<128x32x!tt.ptr, #blocked2>, tensor<128x32xi32, #blocked2>
%96 = tt.load %95, %48 : tensor<128x32x!tt.ptr, #blocked2>
%97 = tt.addptr %53, %94 : tensor<128x32x!tt.ptr, #blocked2>, tensor<128x32xi32, #blocked2>
%98 = tt.load %97, %55 : tensor<128x32x!tt.ptr, #blocked2>
%99 = tt.trans %98 {order = array} : tensor<128x32xf16, #blocked2> -> tensor<32x128xf16, #blocked3>
%100 = tt.fp_to_fp %96 : tensor<128x32xf16, #blocked2> -> tensor<128x32xf32, #blocked2>
%101 = ttg.local_alloc %100 : (tensor<128x32xf32, #blocked2>) -> !ttg.memdesc<128x32xf32, #shared, #smem>
%102 = ttg.local_load %101 : !ttg.memdesc<128x32xf32, #shared, #smem> -> tensor<128x32xf32, #ttg.dot_op<{opIdx = 0, parent = #blocked}>>
%103 = tt.fp_to_fp %99 : tensor<32x128xf16, #blocked3> -> tensor<32x128xf32, #blocked3>
%104 = ttg.local_alloc %103 : (tensor<32x128xf32, #blocked3>) -> !ttg.memdesc<32x128xf32, #shared1, #smem>
%105 = ttg.local_load %104 : !ttg.memdesc<32x128xf32, #shared1, #smem> -> tensor<32x128xf32, #ttg.dot_op<{opIdx = 1, parent = #blocked}>>
%106 = tt.dot %102, %105, %arg9, inputPrecision = tf32 : tensor<128x32xf32, #ttg.dot_op<{opIdx = 0, parent = #blocked}>> * tensor<32x128xf32, #ttg.dot_op<{opIdx = 1, parent = #blocked}>> -> tensor<128x128xf32, #blocked>
scf.yield %106 : tensor<128x128xf32, #blocked>
}
%57 = ttg.convert_layout %37 : tensor<128xf16, #blocked1> -> tensor<128xf16, #ttg.slice<{dim = 1, parent = #blocked}>>
%58 = tt.expand_dims %57 {axis = 1 : i32} : tensor<128xf16, #ttg.slice<{dim = 1, parent = #blocked}>> -> tensor<128x1xf16, #blocked>
%59 = ttg.convert_layout %40 : tensor<128xf16, #blocked1> -> tensor<128xf16, #ttg.slice<{dim = 0, parent = #blocked}>>
%60 = tt.expand_dims %59 {axis = 0 : i32} : tensor<128xf16, #ttg.slice<{dim = 0, parent = #blocked}>> -> tensor<1x128xf16, #blocked>
%61 = tt.broadcast %58 : tensor<128x1xf16, #blocked> -> tensor<128x128xf16, #blocked>
%62 = tt.broadcast %60 : tensor<1x128xf16, #blocked> -> tensor<128x128xf16, #blocked>
%63 = arith.addf %61, %62 : tensor<128x128xf16, #blocked>
%64 = arith.extf %63 : tensor<128x128xf16, #blocked> to tensor<128x128xf32, #blocked>
%65 = math.fma %56, %cst_0, %64 : tensor<128x128xf32, #blocked>
%66 = tt.make_range {end = 128 : i32, start = 0 : i32} : tensor<128xi32, #ttg.slice<{dim = 0, parent = #blocked}>>
%67 = tt.expand_dims %66 {axis = 0 : i32} : tensor<128xi32, #ttg.slice<{dim = 0, parent = #blocked}>> -> tensor<1x128xi32, #blocked>
%68 = tt.broadcast %67 : tensor<1x128xi32, #blocked> -> tensor<128x128xi32, #blocked>
%69:2 = "tt.reduce"(%65, %68) <{axis = 1 : i32}> ({
^bb0(%arg8: f32, %arg9: i32, %arg10: f32, %arg11: i32):
%91 = arith.cmpf oeq, %arg8, %arg10 : f32
%92 = arith.cmpi slt, %arg9, %arg11 : i32
%93 = arith.andi %91, %92 : i1
%94 = arith.cmpf olt, %arg8, %arg10 : f32
%95 = arith.ori %94, %93 : i1
%96 = arith.select %95, %arg8, %arg10 : f32
%97 = arith.select %95, %arg9, %arg11 : i32
tt.reduce.return %96, %97 : f32, i32
}) : (tensor<128x128xf32, #blocked>, tensor<128x128xi32, #blocked>) -> (tensor<128xf32, #ttg.slice<{dim = 1, parent = #blocked}>>, tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked}>>)
%70 = ttg.convert_layout %69#0 : tensor<128xf32, #ttg.slice<{dim = 1, parent = #blocked}>> -> tensor<128xf32, #blocked1>
%71 = tt.splat %arg4 : !tt.ptr -> tensor<128x!tt.ptr, #blocked1>
%72 = tt.addptr %71, %20 : tensor<128x!tt.ptr, #blocked1>, tensor<128xi32, #blocked1>
%73 = tt.bitcast %70 : tensor<128xf32, #blocked1> -> tensor<128xi32, #blocked1>
%74 = tt.bitcast %72 : tensor<128x!tt.ptr, #blocked1> -> tensor<128x!tt.ptr, #blocked1>
%75 = arith.cmpf oge, %70, %cst_3 : tensor<128xf32, #blocked1>
%76 = arith.cmpf olt, %70, %cst_3 : tensor<128xf32, #blocked1>
%77 = arith.andi %29, %75 : tensor<128xi1, #blocked1>
%78 = tt.atomic_rmw min, acq_rel, gpu, %74, %73, %77 : (tensor<128x!tt.ptr, #blocked1>, tensor<128xi32, #blocked1>, tensor<128xi1, #blocked1>) -> tensor<128xi32, #blocked1>
%79 = arith.andi %29, %76 : tensor<128xi1, #blocked1>
%80 = tt.atomic_rmw umax, acq_rel, gpu, %74, %73, %79 : (tensor<128x!tt.ptr, #blocked1>, tensor<128xi32, #blocked1>, tensor<128xi1, #blocked1>) -> tensor<128xi32, #blocked1>
%81 = arith.select %75, %78, %80 : tensor<128xi1, #blocked1>, tensor<128xi32, #blocked1>
%82 = tt.bitcast %81 : tensor<128xi32, #blocked1> -> tensor<128xf32, #blocked1>
%83 = arith.cmpf olt, %70, %82 : tensor<128xf32, #blocked1>
%84 = tt.splat %arg5 : !tt.ptr -> tensor<128x!tt.ptr, #blocked1>
%85 = tt.addptr %84, %20 : tensor<128x!tt.ptr, #blocked1>, tensor<128xi32, #blocked1>
%86 = arith.addi %24, %69#1 : tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked}>>
%87 = tt.load %85 : tensor<128x!tt.ptr, #blocked1>
%88 = ttg.convert_layout %86 : tensor<128xi32, #ttg.slice<{dim = 1, parent = #blocked}>> -> tensor<128xi32, #blocked1>
%89 = arith.extsi %88 : tensor<128xi32, #blocked1> to tensor<128xi64, #blocked1>
%90 = arith.select %83, %89, %87 : tensor<128xi1, #blocked1>, tensor<128xi64, #blocked1>
tt.store %85, %90, %29 : tensor<128x!tt.ptr, #blocked1>
tt.return
}
}

{-#
external_resources: {
mlir_reproducer: {
pipeline: "builtin.module(triton-nvidia-mma-lowering, tritongpu-combine-tensor-select-and-if, tritongpu-allocate-warp-groups, convert-scf-to-cf, allocate-shared-memory, triton-tensor-memory-allocation, tritongpu-global-scratch-memory-allocation, convert-triton-gpu-to-llvm{compute-capability=75 ptx-version=84}, canonicalize{ max-iterations=10 max-num-rewrites=-1 region-simplify=normal test-convergence=false top-down=true}, cse, convert-nv-gpu-to-llvm, convert-warp-specialize-to-llvm, canonicalize{ max-iterations=10 max-num-rewrites=-1 region-simplify=normal test-convergence=false top-down=true}, cse, symbol-dce, enable-line-info)",
disable_threading: false,
verify_each: true
}
}
#-}
/home/ai/miniconda3/envs/nlp/lib/python3.12/site-packages/fastkmeans/triton_kernels.py:33:0: error: Failures have been detected while processing an MLIR pass pipeline
/home/ai/miniconda3/envs/nlp/lib/python3.12/site-packages/fastkmeans/triton_kernels.py:33:0: note: Pipeline failed while executing [`ConvertTritonGPUToLLVM` on 'builtin.module' operation]: reproducer generated at `std::errs, please share the reproducer above with Triton project.`

Reproduce the failure through the PLAID index.add_documents call using the provided PyLate example and the reported PyTorch, Triton, CUDA package, and Titan RTX environment. Start by tracing the fastkmeans indexing path around the generated _kmeans_kernel and determine which configuration triggers the PassManager::run error; done means this indexing example completes without the LLVM rounding-mode failure.

Written by the indexing model from the issue text.