Faiss Test Failure when using cuvs=25.08

When I tried to update Faiss to use the 25.08 version of cuvs, InnerProduct metric tests are failing for CAGRA. Inserted some additional log statements for debugging in graph_core.cuh to find out what is going on and looks like there are a bunch of duplicates in the intermediate graph created by IVF-PQ.

test_cagra.py:79: in do_compute_GT
    index.train(fp16_data, faiss.Float16)
../../../../miniconda3/envs/faiss-cmake/lib/python3.11/site-packages/faiss/class_wrappers.py:305: in replacement_train
    self.train_c(n, swig_ptr(x), faiss.Float16)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <faiss.swigfaiss_avx2.GpuIndexCagra; proxy of <Swig Object of type 'faiss::gpu::GpuIndexCagra *' at 0x7f1dc00dae50> >
args = (10000, <Swig Object of type 'uint16_t *' at 0x7f1dc00dafd0>, 1)

    def train(self, *args):
>       return _swigfaiss_avx2.GpuIndexCagra_train(self, *args)
E       RuntimeError: C++ exception RAFT failure at file=/raid/tarangj/cuvs/cpp/src/neighbors/detail/cagra/graph_core.cuh line=1573: Could not generate an intermediate CAGRA graph because the initial kNN graph contains too many invalid or duplicated neighbor nodes. This error can occur, for example, if too many overflows occur during the norm computation between the dataset vectors.
E       Obtained 63 stack frames
E       #1 in /raid/tarangj/miniconda3/envs/faiss-cmake/lib/libcuvs.so(+0x46c46d) [0x7f1e52dec46d]
E       #2 in /raid/tarangj/miniconda3/envs/faiss-cmake/lib/libcuvs.so: void cuvs::neighbors::cagra::detail::graph::optimize<unsigned int, raft::host_device_accessor<std::experimental::default_accessor<unsigned int>, (raft::memory_type)0> >(raft::resources const&, std::experimental::mdspan<unsigned int, std::experimental::extents<long, 18446744073709551615ul, 18446744073709551615ul>, std::experimental::layout_right, raft::host_device_accessor<std::experimental::default_accessor<unsigned int>, (raft::memory_type)0> >, std::experimental::mdspan<unsigned int, std::experimental::extents<long, 18446744073709551615ul, 18446744073709551615ul>, std::experimental::layout_right, raft::host_device_accessor<std::experimental::default_accessor<unsigned int>, (raft::memory_type)0> >, bool, bool) +0x206e [0x7f1e536cd21e]
E       #3 in /raid/tarangj/miniconda3/envs/faiss-cmake/lib/libcuvs.so: cuvs::neighbors::cagra::index<__half, unsigned int> cuvs::neighbors::cagra::detail::build<__half, unsigned int, raft::host_device_accessor<std::experimental::default_accessor<__half const>, (raft::memory_type)0> >(raft::resources const&, cuvs::neighbors::cagra::index_params const&, std::experimental::mdspan<__half const, std::experimental::extents<long, 18446744073709551615ul, 18446744073709551615ul>, std::experimental::layout_right, raft::host_device_accessor<std::experimental::default_accessor<__half const>, (raft::memory_type)0> >) +0xc93 [0x7f1e537254f3]
E       #4 in /raid/tarangj/miniconda3/envs/faiss-cmake/lib/libcuvs.so: cuvs::neighbors::cagra::build(raft::resources const&, cuvs::neighbors::cagra::index_params const&, std::experimental::mdspan<__half const, std::experimental::extents<long, 18446744073709551615ul, 18446744073709551615ul>, std::experimental::layout_right, raft::host_device_accessor<std::experimental::default_accessor<__half const>, (raft::memory_type)0> >) +0x21 [0x7f1e537101a1]
E       #5 in /raid/tarangj/faiss/build/faiss/libfaiss_avx2.so: faiss::gpu::CuvsCagra<__half>::train(long, __half const*) +0x30e [0x7f1e5fffa2de]
E       #6 in /raid/tarangj/faiss/build/faiss/libfaiss_avx2.so: faiss::gpu::GpuIndexCagra::train(long, void const*, faiss::NumericType) +0x28d [0x7f1e5ffef2ad]
E       #7 in /raid/tarangj/miniconda3/envs/faiss-cmake/lib/python3.11/site-packages/faiss/_swigfaiss_avx2.so(+0x33f3dc) [0x7f1e61e7d3dc]
E       #8 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x1f91b8) [0x558ac55a11b8]
E       #9 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: PyObject_Call +0xbe [0x558ac55ba8ee]
E       #10 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x8e2c [0x558ac559444c]
E       #11 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x227a92) [0x558ac55cfa92]
E       #12 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x227316) [0x558ac55cf316]
E       #13 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: PyObject_Call +0x134 [0x558ac55ba964]
E       #14 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x4a29 [0x558ac5590049]
E       #15 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyFunction_Vectorcall +0x17f [0x558ac55b0a4f]
E       #16 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_FastCallDictTstate +0xe0 [0x558ac5583320]
E       #17 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_Call_Prepend +0x69 [0x558ac55b84c9]
E       #18 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x2df799) [0x558ac5687799]
E       #19 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_MakeTpCall +0x29b [0x558ac557e6ab]
E       #20 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x70a [0x558ac558bd2a]
E       #21 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyFunction_Vectorcall +0x17f [0x558ac55b0a4f]
E       #22 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x4a29 [0x558ac5590049]
E       #23 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyFunction_Vectorcall +0x17f [0x558ac55b0a4f]
E       #24 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_FastCallDictTstate +0xe0 [0x558ac5583320]
E       #25 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_Call_Prepend +0x69 [0x558ac55b84c9]
E       #26 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x2df799) [0x558ac5687799]
E       #27 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: PyObject_Call +0xbe [0x558ac55ba8ee]
E       #28 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x4a29 [0x558ac5590049]
E       #29 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyFunction_Vectorcall +0x17f [0x558ac55b0a4f]
E       #30 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x4a29 [0x558ac5590049]
E       #31 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyFunction_Vectorcall +0x17f [0x558ac55b0a4f]
E       #32 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x4a29 [0x558ac5590049]
E       #33 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyFunction_Vectorcall +0x17f [0x558ac55b0a4f]
E       #34 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_FastCallDictTstate +0xe0 [0x558ac5583320]
E       #35 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_Call_Prepend +0x69 [0x558ac55b84c9]
E       #36 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x2df799) [0x558ac5687799]
E       #37 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_MakeTpCall +0x29b [0x558ac557e6ab]
E       #38 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x70a [0x558ac558bd2a]
E       #39 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyFunction_Vectorcall +0x17f [0x558ac55b0a4f]
E       #40 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x4a29 [0x558ac5590049]
E       #41 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyFunction_Vectorcall +0x17f [0x558ac55b0a4f]
E       #42 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_FastCallDictTstate +0xe0 [0x558ac5583320]
E       #43 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_Call_Prepend +0x69 [0x558ac55b84c9]
E       #44 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x2df799) [0x558ac5687799]
E       #45 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_MakeTpCall +0x29b [0x558ac557e6ab]
E       #46 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x70a [0x558ac558bd2a]
E       #47 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyFunction_Vectorcall +0x17f [0x558ac55b0a4f]
E       #48 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x4a29 [0x558ac5590049]
E       #49 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyFunction_Vectorcall +0x17f [0x558ac55b0a4f]
E       #50 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_FastCallDictTstate +0xe0 [0x558ac5583320]
E       #51 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_Call_Prepend +0x69 [0x558ac55b84c9]
E       #52 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x2df799) [0x558ac5687799]
E       #53 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyObject_MakeTpCall +0x29b [0x558ac557e6ab]
E       #54 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyEval_EvalFrameDefault +0x70a [0x558ac558bd2a]
E       #55 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x29b0ad) [0x558ac56430ad]
E       #56 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: PyEval_EvalCode +0x9f [0x558ac56427ef]
E       #57 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x2b872a) [0x558ac566072a]
E       #58 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x2b43b3) [0x558ac565c3b3]
E       #59 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11(+0x2c9780) [0x558ac5671780]
E       #60 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyRun_SimpleFileObject +0x1bc [0x558ac567111c]
E       #61 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: _PyRun_AnyFileObject +0x44 [0x558ac5670ef4]
E       #62 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: Py_RunMain +0x383 [0x558ac566b643]
E       #63 in /raid/tarangj/miniconda3/envs/faiss-cmake/bin/python3.11: Py_BytesMain +0x37 [0x558ac5632a17]

../../../../miniconda3/envs/faiss-cmake/lib/python3.11/site-packages/faiss/swigfaiss_avx2.py:12459: RuntimeError
--------------------------------------------- Captured stderr call ---------------------------------------------
[901803][12:56:57:364791][info  ] optimizing graph
[901803][12:56:57:365312][info  ] Input graph analysis:
[901803][12:56:57:365321][info  ]   graph_size: 10000
[901803][12:56:57:365326][info  ]   knn_graph_degree (input): 128
[901803][12:56:57:365331][info  ]   output_graph_degree: 64
[901803][12:56:57:365340][info  ]   Sample of first 10 nodes: invalid=0, self_loops=0
[901803][12:56:57:365414][info  ] Initial use_gpu setting: true
[901803][12:56:57:365729][info  ] Successfully allocated GPU memory for pruning
[901803][12:56:57:366008][info  ] Final use_gpu setting for pruning: true
[901914][12:56:57:382239][info  ] Node 1500 failed: found 14/64 neighbors, duplicates=114, out_of_range=0, last_detour_count=116
[901911][12:56:57:382239][info  ] Node 1125 failed: found 56/64 neighbors, duplicates=72, out_of_range=0, last_detour_count=255
[901935][12:56:57:382259][info  ] Node 4126 failed: found 54/64 neighbors, duplicates=74, out_of_range=0, last_detour_count=86
[901932][12:56:57:382322][info  ] Node 3754 failed: found 49/64 neighbors, duplicates=79, out_of_range=0, last_detour_count=167
[901929][12:56:57:382342][info  ] Node 3376 failed: found 48/64 neighbors, duplicates=80, out_of_range=0, last_detour_count=85
[901925][12:56:57:382325][info  ] Node 2875 failed: found 55/64 neighbors, duplicates=73, out_of_range=0, last_detour_count=79
[901979][12:56:57:382333][info  ] Node 9626 failed: found 38/64 neighbors, duplicates=90, out_of_range=0, last_detour_count=126
[901981][12:56:57:382350][info  ] Node 9876 failed: found 59/64 neighbors, duplicates=69, out_of_range=0, last_detour_count=10
[901941][12:56:57:382328][info  ] Node 4877 failed: found 45/64 neighbors, duplicates=83, out_of_range=0, last_detour_count=92
[901976][12:56:57:382396][info  ] Node 9251 failed: found 47/64 neighbors, duplicates=81, out_of_range=0, last_detour_count=68
[901911][12:56:57:382436][info  ]   Neighbors for node 1125: 4892(0) 1080(61) 6949(1) 6931(0) 575(1) 3682(0) 8858(3) 5619(0) 5162(0) 3791(1) 
[901914][12:56:57:382438][info  ]   Neighbors for node 1500: 125(0) 68(0) 3961(0) 8462(1) 2093(1) 6719(0) 4610(1) 291(1) 9222(3) 1965(0) 
[901911][12:56:57:382510][info  ]   Detour count histogram for node 1125: 0:82 1:12 2:10 3:7 4:4 5:2 6:1 7:3 8:2 9:1 255:1 
[901914][12:56:57:382533][info  ]   Detour count histogram for node 1500: 0:120 1:5 3:2 
[901914][12:56:57:382546][info  ]   Input graph duplicates for node 1500: 114
[901911][12:56:57:382551][info  ]   Input graph duplicates for node 1125: 72
[901803][12:56:57:390101][info  ] Graph pruning statistics:
[901803][12:56:57:390118][info  ]   Total nodes: 10000
[901803][12:56:57:390170][info  ]   Failed nodes: 446 (4.46%)
[901803][12:56:57:390177][info  ]   Total duplicates encountered: 102848
[901803][12:56:57:390182][info  ]   Total out-of-range neighbors: 0
[901803][12:56:57:390187][info  ]   Total valid neighbors added: 632069
[901803][12:56:57:390191][info  ]   Distribution of pk values:
[901803][12:56:57:390196][info  ]     pk=1: 3 nodes
[901803][12:56:57:390201][info  ]     pk=2: 1 nodes
[901803][12:56:57:390206][info  ]     pk=5: 1 nodes
[901803][12:56:57:390211][info  ]     pk=6: 1 nodes
[901803][12:56:57:390216][info  ]     pk=9: 1 nodes
[901803][12:56:57:390221][info  ]     pk=10: 1 nodes
[901803][12:56:57:390225][info  ]     pk=11: 1 nodes
[901803][12:56:57:390230][info  ]     pk=13: 2 nodes
[901803][12:56:57:390235][info  ]     pk=14: 3 nodes
[901803][12:56:57:390240][info  ]     pk=15: 1 nodes
[901803][12:56:57:390244][info  ]     pk=17: 5 nodes
[901803][12:56:57:390249][info  ]     pk=18: 2 nodes
[901803][12:56:57:390254][info  ]     pk=19: 4 nodes
[901803][12:56:57:390259][info  ]     pk=20: 4 nodes
[901803][12:56:57:390264][info  ]     pk=21: 3 nodes
[901803][12:56:57:390268][info  ]     pk=22: 5 nodes
[901803][12:56:57:390273][info  ]     pk=23: 1 nodes
[901803][12:56:57:390278][info  ]     pk=24: 7 nodes
[901803][12:56:57:390283][info  ]     pk=25: 3 nodes
[901803][12:56:57:390287][info  ]     pk=26: 3 nodes
[901803][12:56:57:390292][info  ]     pk=27: 4 nodes
[901803][12:56:57:390297][info  ]     pk=28: 4 nodes
[901803][12:56:57:390327][info  ]     pk=29: 4 nodes
[901803][12:56:57:390333][info  ]     pk=30: 7 nodes
[901803][12:56:57:390338][info  ]     pk=31: 2 nodes
[901803][12:56:57:390342][info  ]     pk=32: 5 nodes
[901803][12:56:57:390347][info  ]     pk=33: 5 nodes
[901803][12:56:57:390352][info  ]     pk=34: 6 nodes
[901803][12:56:57:390357][info  ]     pk=35: 9 nodes
[901803][12:56:57:390361][info  ]     pk=36: 5 nodes
[901803][12:56:57:390366][info  ]     pk=37: 7 nodes
[901803][12:56:57:390371][info  ]     pk=38: 4 nodes
[901803][12:56:57:390376][info  ]     pk=39: 8 nodes
[901803][12:56:57:390380][info  ]     pk=40: 9 nodes
[901803][12:56:57:390385][info  ]     pk=41: 7 nodes
[901803][12:56:57:390390][info  ]     pk=42: 10 nodes
[901803][12:56:57:390395][info  ]     pk=43: 6 nodes
[901803][12:56:57:390400][info  ]     pk=44: 10 nodes
[901803][12:56:57:390404][info  ]     pk=45: 9 nodes
[901803][12:56:57:390409][info  ]     pk=46: 11 nodes
[901803][12:56:57:390414][info  ]     pk=47: 9 nodes
[901803][12:56:57:390419][info  ]     pk=48: 12 nodes
[901803][12:56:57:390424][info  ]     pk=49: 16 nodes
[901803][12:56:57:390428][info  ]     pk=50: 15 nodes
[901803][12:56:57:390433][info  ]     pk=51: 15 nodes
[901803][12:56:57:390438][info  ]     pk=52: 10 nodes
[901803][12:56:57:390443][info  ]     pk=53: 9 nodes
[901803][12:56:57:390447][info  ]     pk=54: 17 nodes
[901803][12:56:57:390452][info  ]     pk=55: 10 nodes
[901803][12:56:57:390457][info  ]     pk=56: 16 nodes
[901803][12:56:57:390462][info  ]     pk=57: 13 nodes
[901803][12:56:57:390467][info  ]     pk=58: 24 nodes
[901803][12:56:57:390471][info  ]     pk=59: 16 nodes
[901803][12:56:57:390476][info  ]     pk=60: 14 nodes
[901803][12:56:57:390481][info  ]     pk=61: 29 nodes
[901803][12:56:57:390486][info  ]     pk=62: 17 nodes
[901803][12:56:57:390491][info  ]     pk=63: 20 nodes
[901803][12:56:57:390495][info  ]     pk=64: 9554 nodes
=========================================== short test summary info ============================================
FAILED test_cagra.py::TestComputeGT::test_compute_GT_IP - RuntimeError: C++ exception RAFT failure at file=/raid/tarangj/cuvs/cpp/src/neighbors/detail/cagra/graph_co...
FAILED test_cagra.py::TestComputeGTFP16::test_compute_GT_IP - RuntimeError: C++ exception RAFT failure at file=/raid/tarangj/cuvs/cpp/src/neighbors/detail/cagra/graph_co...

Reproduce the failure from test_cagra.py:79 using cuvs 25.08, then inspect graph_core.cuh around line 1573 and the logged intermediate-graph analysis. Trace why the InnerProduct CAGRA training path produces duplicated neighbors, and consider the issue done when the test completes without the invalid-or-duplicated-neighbor exception.