Need help with identifying source of error in distributed dgl job

Hi,

DGL example script used to work fine for me. However, since a recent system maintenance, dist-DGL stopped working. I am attaching the error below. I need help in narrowing down the source of the error. I am running the distributed example with dgl launch script.

Looks like it is running into some issues with node_split(). Is this related to some issue in the filesystem where my partitions are stored? Or is this a network configuration issue?

```
Traceback (most recent call last):
Traceback (most recent call last):
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
Traceback (most recent call last):
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
main(args)
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
main(args)
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
main(args)
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
train_nid = dgl.distributed.node_split(train_nid = dgl.distributed.node_split(

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
train_nid = dgl.distributed.node_split(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
Traceback (most recent call last):
Traceback (most recent call last):
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
Traceback (most recent call last):
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
Traceback (most recent call last):
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
part_nid = _split_even_to_part(partition_book, nodes)
part_nid = _split_even_to_part(partition_book, nodes)
part_nid = _split_even_to_part(partition_book, nodes)
Traceback (most recent call last):
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
Traceback (most recent call last):
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
^^ ^^ ^^ ^^ ^^^ ^^ ^^ ^^ ^^ ^^ ^^ ^^^^ ^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^Traceback (most recent call last):
^
^ File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1487, in _split_even_to_part
^^^^ File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
^ File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1487, in _split_even_to_part
^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1487, in _split_even_to_part
main(args)
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
main(args)
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
main(args)
nonzero_count = elements.count_nonzero()
nonzero_count = elements.count_nonzero()
^ ^ ^ File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
main(args)
main(args)
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
main(args)
^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 302, in count_nonzero
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 302, in count_nonzero
train_nid = dgl.distributed.node_split(
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
main(args)
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
nonzero_count = elements.count_nonzero()
^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 302, in count_nonzero
train_nid = dgl.distributed.node_split(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
train_nid = dgl.distributed.node_split(
train_nid = dgl.distributed.node_split(
^^^^^ ^^train_nid = dgl.distributed.node_split(^
^^^^^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
^^^^ File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
train_nid = dgl.distributed.node_split(
^^^^^^ ^^train_nid = dgl.distributed.node_split(^
^^^^ ^ ^ ^ ^ ^ ^ ^ ^ ^^ ^ ^ ^ ^
^^^^^^^^^^^^^^^^ File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
part_nid = _split_even_to_part(partition_book, nodes)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1487, in _split_even_to_part
nonzero_count = elements.count_nonzero()
^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 302, in count_nonzero
part_nid = _split_even_to_part(partition_book, nodes)
^^^^^^^^^^ ^^part_nid = _split_even_to_part(partition_book, nodes)^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1507, in _split_even_to_part
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1507, in _split_even_to_part
part_nid = _split_even_to_part(partition_book, nodes)
^^^^^^ ^part_nid = _split_even_to_part(partition_book, nodes)^
^^^^^^^^^^^^^^^^^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^
^ part_nid = _split_even_to_part(partition_book, nodes)
^^^^^ File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1507, in _split_even_to_part
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1507, in _split_even_to_part
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1507, in _split_even_to_part
part_nid = _split_even_to_part(partition_book, nodes)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1507, in _split_even_to_part
elements[idx : min(idx + block_size, num_elements)]
~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 205, in __getitem__
elements[idx : min(idx + block_size, num_elements)]
elements[idx : min(idx + block_size, num_elements)]
~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^elements[idx : min(idx + block_size, num_elements)]^
^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 205, in __getitem__
~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 205, in __getitem__
^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 205, in __getitem__
elements[idx : min(idx + block_size, num_elements)]
~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 205, in __getitem__
elements[idx : min(idx + block_size, num_elements)]
~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 205, in __getitem__
Traceback (most recent call last):
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
main(args)
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
train_nid = dgl.distributed.node_split(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
return self.kvstore.pull(name=self._name, id_tensor=idx)
return self.kvstore.count_nonzero(name=self._name)return self.kvstore.count_nonzero(name=self._name)

return self.kvstore.pull(name=self._name, id_tensor=idx)
return self.kvstore.pull(name=self._name, id_tensor=idx)
return self.kvstore.pull(name=self._name, id_tensor=idx)
^^^^^^^^^^^^^^^^^ ^^ ^ ^ ^ ^ ^^ ^ ^ ^ ^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^ File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1463, in pull
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1463, in pull
^ ^ ^ ^ ^ ^ ^return self.kvstore.count_nonzero(name=self._name)^ ^
^^^^^^^^^^^^^^^^^^^^^^ ^^return self.kvstore.count_nonzero(name=self._name)^
^^^^^^^^^^^^ ^^ ^^ ^^ ^^^ ^^ ^^ ^^ ^^ ^ ^ ^^ ^^ ^
^^ ^^ ^^ File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1558, in count_nonzero
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^ File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1558, in count_nonzero
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^ ^^^^^^^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1463, in pull
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1463, in pull
return self.kvstore.pull(name=self._name, id_tensor=idx)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ part_nid = _split_even_to_part(partition_book, nodes)
return self.kvstore.pull(name=self._name, id_tensor=idx)
^^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^ File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1507, in _split_even_to_part
^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1463, in pull
^^^^^^^
^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1558, in count_nonzero
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1558, in count_nonzero
^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1463, in pull
Traceback (most recent call last):
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 577, in
main(args)
File "/global/u1/s/sark777/Distributed_DGL/src/prefetch/node_classification.py", line 405, in main
elements[idx : min(idx + block_size, num_elements)]
~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 205, in __getitem__
train_nid = dgl.distributed.node_split(
return rpc.fast_pull(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1675, in node_split
^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/rpc.py", line 1166, in fast_pull
return rpc.fast_pull(
^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/rpc.py", line 1166, in fast_pull
return rpc.fast_pull(
^^ ^^return rpc.fast_pull(^
^^^^ ^ ^ ^ ^ ^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/rpc.py", line 1166, in fast_pull
^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/rpc.py", line 1166, in fast_pull
total += F.count_nonzero(self._data_store[name][local_id])
total += F.count_nonzero(self._data_store[name][local_id])
return rpc.fast_pull(
^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/rpc.py", line 1166, in fast_pull
return rpc.fast_pull(
^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/rpc.py", line 1166, in fast_pull
return self.kvstore.pull(name=self._name, id_tensor=idx)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1463, in pull
~~ ~~ ~ ~ ~ ~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~^~^~^~^~^~^~^~^^^^
^^^^^IndexError^^: ^index 630098 is out of bounds for dimension 0 with size 630098

IndexError: index 630098 is out of bounds for dimension 0 with size 630098
total += F.count_nonzero(self._data_store[name][local_id])
~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
IndexError: index 630098 is out of bounds for dimension 0 with size 630098
total += F.count_nonzero(self._data_store[name][local_id])
~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
IndexError: index 630098 is out of bounds for dimension 0 with size 630098
part_nid = _split_even_to_part(partition_book, nodes)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ res_tensor = _CAPI_DGLRPCFastPull(
^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_graph.py", line 1507, in _split_even_to_part
res_tensor = _CAPI_DGLRPCFastPull(
^res_tensor = _CAPI_DGLRPCFastPull(^
^^^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
^^^^^ File "dgl/_ffi/_cython/./function.pxi", line 295, in dgl._ffi._cy3.core.FunctionBase.__call__
^^^^^^^^^^^^^^^^
File "dgl/_ffi/_cython/./function.pxi", line 295, in dgl._ffi._cy3.core.FunctionBase.__call__
^^^^^^^^^^^^^^^^^^^^^
File "dgl/_ffi/_cython/./function.pxi", line 295, in dgl._ffi._cy3.core.FunctionBase.__call__
res_tensor = _CAPI_DGLRPCFastPull(
^^^^^^^^^^^^^^^^^^^^^
File "dgl/_ffi/_cython/./function.pxi", line 295, in dgl._ffi._cy3.core.FunctionBase.__call__
res_tensor = _CAPI_DGLRPCFastPull(
^^^^^^^^^^^^^^^^^^^^^
File "dgl/_ffi/_cython/./function.pxi", line 295, in dgl._ffi._cy3.core.FunctionBase.__call__
res_tensor = _CAPI_DGLRPCFastPull(
^^^^^^^^^^^^^^^^^^^^^
File "dgl/_ffi/_cython/./function.pxi", line 295, in dgl._ffi._cy3.core.FunctionBase.__call__
return rpc.fast_pull(
^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/rpc.py", line 1166, in fast_pull
File "dgl/_ffi/_cython/./function.pxi", line 241, in dgl._ffi._cy3.core.FuncCall
File "dgl/_ffi/_cython/./function.pxi", line 241, in dgl._ffi._cy3.core.FuncCall
File "dgl/_ffi/_cython/./function.pxi", line 241, in dgl._ffi._cy3.core.FuncCall
File "dgl/_ffi/_cython/./function.pxi", line 241, in dgl._ffi._cy3.core.FuncCall
res_tensor = _CAPI_DGLRPCFastPull( elements[idx : min(idx + block_size, num_elements)]
~~~~~~~~^^^^^^^^^^^^^ File "dgl/_ffi/_cython/./function.pxi", line 241, in dgl._ffi._cy3.core.FuncCall
^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^
File "dgl/_ffi/_cython/./function.pxi", line 295, in dgl._ffi._cy3.core.FunctionBase.__call__
File "dgl/_ffi/_cython/./function.pxi", line 241, in dgl._ffi._cy3.core.FuncCall
^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/dist_tensor.py", line 205, in __getitem__
dgl._ffi.base.DGLError: [02:00:36] /opt/dgl/src/rpc/rpc.cc:484: Check failed: l_id < local_data_shape[0] (18446744073707713820 vs. 630098) :
Stack trace:
[bt] (0) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a4b80) [0x7fdaa8ea4b80]
[bt] (1) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9869) [0x7fdaa8ea9869]
[bt] (2) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9e74) [0x7fdaa8ea9e74]
[bt] (3) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(DGLFuncCall+0x48) [0x7fdaa8d16108]
[bt] (4) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x162a3) [0x7fdaa85f72a3]
[bt] (5) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x1690b) [0x7fdaa85f790b]
[bt] (6) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyObject_MakeTpCall+0x254) [0x502a14]
[bt] (7) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyEval_EvalFrameDefault+0x753) [0x50ed83]
[bt] (8) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python() [0x57b0d0]

dgl._ffi.base.DGLError: [02:00:36] /opt/dgl/src/rpc/rpc.cc:484: Check failed: l_id < local_data_shape[0] (18446744073708921518 vs. 630098) :
Stack trace:
[bt] (0) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a4b80) [0x7f52ac8a4b80]
[bt] (1) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9869) [0x7f52ac8a9869]
[bt] (2) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9e74) [0x7f52ac8a9e74]
[bt] (3) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(DGLFuncCall+0x48) [0x7f52ac716108]
[bt] (4) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x162a3) [0x7f52abff72a3]
[bt] (5) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x1690b) [0x7f52abff790b]
[bt] (6) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyObject_MakeTpCall+0x254) [0x502a14]
[bt] (7) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyEval_EvalFrameDefault+0x753) [0x50ed83]
[bt] (8) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python() [0x57b0d0]

dgl._ffi.base.DGLError: [02:00:36] /opt/dgl/src/rpc/rpc.cc:484: Check failed: l_id < local_data_shape[0] (18446744073708921518 vs. 630098) :
Stack trace:
[bt] (0) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a4b80) [0x7ffac42a4b80]
[bt] (1) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9869) [0x7ffac42a9869]
[bt] (2) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9e74) [0x7ffac42a9e74]
[bt] (3) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(DGLFuncCall+0x48) [0x7ffac4116108]
[bt] (4) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x162a3) [0x7ffac39f72a3]
[bt] (5) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x1690b) [0x7ffac39f790b]
[bt] (6) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyObject_MakeTpCall+0x254) [0x502a14]
[bt] (7) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyEval_EvalFrameDefault+0x753) [0x50ed83]
[bt] (8) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python() [0x57b0d0]

dgl._ffi.base.DGLError: [02:00:36] /opt/dgl/src/rpc/rpc.cc:484: Check failed: l_id < local_data_shape[0] (18446744073707713820 vs. 630098) :
Stack trace:
[bt] (0) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a4b80) [0x7f3c7bea4b80]
[bt] (1) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9869) [0x7f3c7bea9869]
[bt] (2) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9e74) [0x7f3c7bea9e74]
[bt] (3) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(DGLFuncCall+0x48) [0x7f3c7bd16108]
[bt] (4) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x162a3) [0x7f3c7b5f72a3]
[bt] (5) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x1690b) [0x7f3c7b5f790b]
[bt] (6) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyObject_MakeTpCall+0x254) [0x502a14]
[bt] (7) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyEval_EvalFrameDefault+0x753) [0x50ed83]
[bt] (8) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python() [0x57b0d0]

return self.kvstore.pull(name=self._name, id_tensor=idx)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
dgl._ffi.base.DGLError File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/kvstore.py", line 1463, in pull
: File "dgl/_ffi/_cython/./function.pxi", line 241, in dgl._ffi._cy3.core.FuncCall
[02:00:36] /opt/dgl/src/rpc/rpc.cc:484: Check failed: l_id < local_data_shape[0] (18446744073707713820 vs. 630098) :
Stack trace:
[bt] (0) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a4b80) [0x7f282dea4b80]
[bt] (1) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9869) [0x7f282dea9869]
[bt] (2) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9e74) [0x7f282dea9e74]
[bt] (3) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(DGLFuncCall+0x48) [0x7f282dd16108]
[bt] (4) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x162a3) [0x7f282d5f72a3]
[bt] (5) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x1690b) [0x7f282d5f790b]
[bt] (6) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyObject_MakeTpCall+0x254) [0x502a14]
[bt] (7) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyEval_EvalFrameDefault+0x753) [0x50ed83]
[bt] (8) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python() [0x57b0d0]

dgl._ffi.base.DGLError: [02:00:36] /opt/dgl/src/rpc/rpc.cc:484: Check failed: l_id < local_data_shape[0] (18446744073708921518 vs. 630098) :
Stack trace:
[bt] (0) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a4b80) [0x7fe434ea4b80]
[bt] (1) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9869) [0x7fe434ea9869]
[bt] (2) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9e74) [0x7fe434ea9e74]
[bt] (3) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(DGLFuncCall+0x48) [0x7fe434d16108]
[bt] (4) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x162a3) [0x7fe4345f72a3]
[bt] (5) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x1690b) [0x7fe4345f790b]
[bt] (6) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyObject_MakeTpCall+0x254) [0x502a14]
[bt] (7) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyEval_EvalFrameDefault+0x753) [0x50ed83]
[bt] (8) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python() [0x57b0d0]

dgl._ffi.base.DGLError: [02:00:36] /opt/dgl/src/rpc/rpc.cc:484: Check failed: l_id < local_data_shape[0] (18446744073708921518 vs. 630098) :
Stack trace:
[bt] (0) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a4b80) [0x7f6c524a4b80]
[bt] (1) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9869) [0x7f6c524a9869]
[bt] (2) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9e74) [0x7f6c524a9e74]
[bt] (3) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(DGLFuncCall+0x48) [0x7f6c52316108]
[bt] (4) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x162a3) [0x7f6c51bf72a3]
[bt] (5) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x1690b) [0x7f6c51bf790b]
[bt] (6) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyObject_MakeTpCall+0x254) [0x502a14]
[bt] (7) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyEval_EvalFrameDefault+0x753) [0x50ed83]
[bt] (8) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python() [0x57b0d0]

return rpc.fast_pull(
^^^^^^^^^^^^^^
File "/global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/distributed/rpc.py", line 1166, in fast_pull
res_tensor = _CAPI_DGLRPCFastPull(
^^^^^^^^^^^^^^^^^^^^^
File "dgl/_ffi/_cython/./function.pxi", line 295, in dgl._ffi._cy3.core.FunctionBase.__call__
File "dgl/_ffi/_cython/./function.pxi", line 241, in dgl._ffi._cy3.core.FuncCall
dgl._ffi.base.DGLError: [02:00:36] /opt/dgl/src/rpc/rpc.cc:484: Check failed: l_id < local_data_shape[0] (18446744073707713820 vs. 630098) :
Stack trace:
[bt] (0) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a4b80) [0x7f63c1aa4b80]
[bt] (1) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9869) [0x7f63c1aa9869]
[bt] (2) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(+0x8a9e74) [0x7f63c1aa9e74]
[bt] (3) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/libdgl.so(DGLFuncCall+0x48) [0x7f63c1916108]
[bt] (4) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x162a3) [0x7f63c11f72a3]
[bt] (5) /global/homes/s/sark777/.conda/envs/dgl-nightly/lib/python3.11/site-packages/dgl/_ffi/_cy3/core.cpython-311-x86_64-linux-gnu.so(+0x1690b) [0x7f63c11f790b]
[bt] (6) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyObject_MakeTpCall+0x254) [0x502a14]
[bt] (7) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python(_PyEval_EvalFrameDefault+0x753) [0x50ed83]
[bt] (8) /global/homes/s/sark777/.conda/envs/dgl-nightly/bin/python() [0x57b0d0]
```