RuntimeError: nonzero is not supported for tensors with more than INT_MAX elements

If you do not know the root cause of the problem, please post according to this template:

## Instructions To Reproduce the Issue:

Check https://stackoverflow.com/help/minimal-reproducible-example for how to ask good questions.
Simplify the steps to reproduce the issue using suggestions from the above link, and provide them below:

1. Full runnable code or full changes you made:
```
If making changes to the project itself, please use output of the following command:
git rev-parse HEAD; git diff
import os
from detectron2.checkpoint import DetectionCheckpointer
from detectron2.config import get_cfg
from detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, launch
from detectron2.evaluation import COCOEvaluator, PascalVOCDetectionEvaluator
from detectron2.layers import get_norm
from detectron2.modeling.roi_heads import ROI_HEADS_REGISTRY, Res5ROIHeads
from detectron2.data.datasets import register_coco_instances
from detectron2 import model_zoo

@ROI_HEADS_REGISTRY.register()
class Res5ROIHeadsExtraNorm(Res5ROIHeads):
"""
As described in the MOCO paper, there is an extra BN layer
following the res5 stage.
"""

def _build_res5_block(self, cfg):
seq, out_channels = super()._build_res5_block(cfg)
norm = cfg.MODEL.RESNETS.NORM
norm = get_norm(norm, out_channels)
seq.add_module("norm", norm)
return seq, out_channels

class Trainer(DefaultTrainer):
@classmethod
def build_evaluator(cls, cfg, dataset_name, output_folder=None):
if output_folder is None:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
if "coco" in dataset_name:
return COCOEvaluator(dataset_name, cfg, True, output_folder)
elif "voc" in dataset_name:
return PascalVOCDetectionEvaluator(dataset_name)
else:
return COCOEvaluator(dataset_name, cfg, True, output_folder)

def setup(args):
cfg = get_cfg()

cfg.merge_from_file(args.config_file)

cfg.merge_from_list(args.opts)

cfg.DATASETS.TRAIN = ("BDD_train",)
cfg.DATALOADER.NUM_WORKERS = 2
cfg.DATASETS.TEST = ("BDD_val",)
cfg.SOLVER.IMS_PER_BATCH = 4
cfg.SOLVER.BASE_LR = 0.0125 # pick a good LearningRate
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 10 # No. of BDD classes
detector_name = 'BYOL_BDD'
cfg.OUTPUT_DIR = os.path.join(os.getcwd(),detector_name)
cfg.freeze()
default_setup(cfg, args)

return cfg

def main(args):

cfg = setup(args)
register_coco_instances("BDD_train", {}, "/BDD/BDD_train_v2.json", "/BDD/images/train/")
register_coco_instances("BDD_val", {}, "/BDD/BDD_valid.json", "/BDD/images/valid")
if args.eval_only:
model = Trainer.build_model(cfg)
DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(
cfg.MODEL.WEIGHTS, resume=args.resume
)
res = Trainer.test(cfg, model)
return res

trainer = Trainer(cfg)
trainer.resume_or_load(resume=args.resume)
return trainer.train()

if __name__ == "__main__":
args = default_argument_parser().parse_args()
print("Command Line Args:", args)
launch(
main,
args.num_gpus,
num_machines=args.num_machines,
machine_rank=args.machine_rank,
dist_url=args.dist_url,
args=(args,),
)

```
2. What exact command you run: python train.py
3. __Full logs__ or other relevant observations:
```
GPU Driver '470' detected
Command Line Args: Namespace(config_file='./configs/coco_R_50_C4_2x.yaml', resume=False, eval_only=False, num_gpus=4, num_machines=1, machine_rank=0, dist_url='tcp://127.0.0.1:52795', opts=['MODEL.WEIGHTS', './byol_detectron_model.pkl', 'SOLVER.IMS_PER_BATCH', '16', 'SOLVER.BASE_LR', '0.1'])

[07/05 15:08:47 detectron2]: Rank of current process: 0. World size: 4

[07/05 15:08:57 detectron2]: Command line arguments: Namespace(config_file='./configs/coco_R_50_C4_2x.yaml', resume=False, eval_only=False, num_gpus=4, num_machines=1, machine_rank=0, dist_url='tcp://127.0.0.1:52795', opts=['MODEL.WEIGHTS', './byol_detectron_model.pkl', 'SOLVER.IMS_PER_BATCH', '16', 'SOLVER.BASE_LR', '0.1'])
[07/05 15:08:58 d2.engine.defaults]: Model:
GeneralizedRCNN(
(backbone): ResNet(
(stem): BasicStem(
(conv1): Conv2d(
3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False
(norm): SyncBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(res2): Sequential(
(0): BottleneckBlock(
(shortcut): Conv2d(
64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv1): Conv2d(
64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(1): BottleneckBlock(
(conv1): Conv2d(
256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(2): BottleneckBlock(
(conv1): Conv2d(
256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
)
(res3): Sequential(
(0): BottleneckBlock(
(shortcut): Conv2d(
256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv1): Conv2d(
256, 128, kernel_size=(1, 1), stride=(2, 2), bias=False
(norm): SyncBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(1): BottleneckBlock(
(conv1): Conv2d(
512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(2): BottleneckBlock(
(conv1): Conv2d(
512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(3): BottleneckBlock(
(conv1): Conv2d(
512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
)
(res4): Sequential(
(0): BottleneckBlock(
(shortcut): Conv2d(
512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False
(norm): SyncBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv1): Conv2d(
512, 256, kernel_size=(1, 1), stride=(2, 2), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(1): BottleneckBlock(
(conv1): Conv2d(
1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(2): BottleneckBlock(
(conv1): Conv2d(
1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(3): BottleneckBlock(
(conv1): Conv2d(
1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(4): BottleneckBlock(
(conv1): Conv2d(
1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(5): BottleneckBlock(
(conv1): Conv2d(
1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
)
)
(proposal_generator): RPN(
(rpn_head): StandardRPNHead(
(conv): Conv2d(
1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)
(activation): ReLU()
)
(objectness_logits): Conv2d(1024, 15, kernel_size=(1, 1), stride=(1, 1))
(anchor_deltas): Conv2d(1024, 60, kernel_size=(1, 1), stride=(1, 1))
)
(anchor_generator): DefaultAnchorGenerator(
(cell_anchors): BufferList()
)
)
(roi_heads): Res5ROIHeadsExtraNorm(
(pooler): ROIPooler(
(level_poolers): ModuleList(
(0): ROIAlign(output_size=(14, 14), spatial_scale=0.0625, sampling_ratio=0, aligned=True)
)
)
(res5): Sequential(
(0): BottleneckBlock(
(shortcut): Conv2d(
1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False
(norm): SyncBatchNorm(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv1): Conv2d(
1024, 512, kernel_size=(1, 1), stride=(2, 2), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(1): BottleneckBlock(
(conv1): Conv2d(
2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(2): BottleneckBlock(
(conv1): Conv2d(
2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv2): Conv2d(
512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False
(norm): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(conv3): Conv2d(
512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
(norm): SyncBatchNorm(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(norm): SyncBatchNorm(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(box_predictor): FastRCNNOutputLayers(
(cls_score): Linear(in_features=2048, out_features=11, bias=True)
(bbox_pred): Linear(in_features=2048, out_features=40, bias=True)
)
)
)

[07/05 15:09:09 d2.data.build]: Using training sampler TrainingSampler
[07/05 15:09:16 d2.data.common]: Serializing the dataset using:
[07/05 15:09:16 d2.data.common]: Serializing 10099 elements to byte tensors and concatenating them all ...
[07/05 15:09:17 d2.data.common]: Serialized dataset takes 41.70 MiB
[07/05 15:09:23 d2.data.datasets.coco]: Loading /home/kalikhademi/orange_juan_gilbert/kalikhademi/dissertation_vision/dataset/BDD/BDD_train_v2.json takes 5.26 seconds.
[07/05 15:09:23 d2.data.datasets.coco]: Loaded 10099 images in COCO format from /home/kalikhademi/orange_juan_gilbert/kalikhademi/dissertation_vision/dataset/BDD/BDD_train_v2.json
[07/05 15:09:26 d2.data.build]: Removed 0 images with no usable annotations. 10099 images left.
[07/05 15:09:27 d2.data.dataset_mapper]: [DatasetMapper] Augmentations used in training: [ResizeShortestEdge(short_edge_length=(640, 672, 704, 736, 768, 800), max_size=1333, sample_style='choice'), RandomFlip()]
[07/05 15:09:27 d2.data.build]: Using training sampler TrainingSampler
[07/05 15:09:35 d2.data.common]: Serializing the dataset using:
[07/05 15:09:35 d2.data.common]: Serializing 10099 elements to byte tensors and concatenating them all ...
[07/05 15:09:37 d2.data.common]: Serialized dataset takes 41.70 MiB
[07/05 15:09:37 d2.checkpoint.detection_checkpoint]: [DetectionCheckpointer] Loading from ./byol_detectron_model.pkl ...
[07/05 15:09:37 fvcore.common.checkpoint]: [Checkpointer] Loading from ./byol_detectron_model.pkl ...
[07/05 15:09:37 fvcore.common.checkpoint]: Reading a file from 'solo-learn'
[07/05 15:09:37 d2.checkpoint.c2_model_loading]: Following weights matched with model:

[07/05 15:09:37 d2.engine.train_loop]: Starting training from iteration 0
/home/.local/lib/python3.10/site-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3483.)
return _VF.meshgrid(tensors, **kwargs) # type: ignore[attr-defined]
/home/.local/lib/python3.10/site-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3483.)
return _VF.meshgrid(tensors, **kwargs) # type: ignore[attr-defined]
/home/.local/lib/python3.10/site-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3483.)
return _VF.meshgrid(tensors, **kwargs) # type: ignore[attr-defined]
/home/kalikhademi/.local/lib/python3.10/site-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3483.)
return _VF.meshgrid(tensors, **kwargs) # type: ignore[attr-defined]
[07/05 15:10:24 d2.utils.events]: eta: 13:42:05 iter: 19 total_loss: 4.511 loss_cls: 2.39 loss_box_reg: 2.054 loss_rpn_cls: 0.00556 loss_rpn_loc: 0.1136 time: 1.3124 last_time: 1.0732 data_time: 0.7075 last_data_time: 0.0107 lr: 0.00024976 max_mem: 2179M
[07/05 15:11:09 d2.utils.events]: eta: 12:09:50 iter: 39 total_loss: 4.483 loss_cls: 2.267 loss_box_reg: 2.132 loss_rpn_cls: 0.004844 loss_rpn_loc: 0.1037 time: 1.2017 last_time: 1.1382 data_time: 0.0222 last_data_time: 0.0135 lr: 0.00049951 max_mem: 3035M
[07/05 15:11:32 d2.utils.events]: eta: 12:05:08 iter: 59 total_loss: 4.263 loss_cls: 1.972 loss_box_reg: 2.153 loss_rpn_cls: 0.004295 loss_rpn_loc: 0.09861 time: 1.1728 last_time: 1.2492 data_time: 0.0184 last_data_time: 0.0191 lr: 0.00074926 max_mem: 3035M
[07/05 15:11:54 d2.utils.events]: eta: 11:56:29 iter: 79 total_loss: 4.064 loss_cls: 1.982 loss_box_reg: 1.952 loss_rpn_cls: 0.003846 loss_rpn_loc: 0.08443 time: 1.1614 last_time: 0.9913 data_time: 0.0268 last_data_time: 0.0263 lr: 0.00099901 max_mem: 3035M
[07/05 15:12:18 d2.utils.events]: eta: 12:04:24 iter: 99 total_loss: 3.832 loss_cls: 1.494 loss_box_reg: 2.278 loss_rpn_cls: 0.003217 loss_rpn_loc: 0.06982 time: 1.1606 last_time: 0.9454 data_time: 0.0243 last_data_time: 0.0148 lr: 0.0012488 max_mem: 3035M
[07/05 15:12:40 d2.utils.events]: eta: 12:02:03 iter: 119 total_loss: 3.744 loss_cls: 1.663 loss_box_reg: 2.002 loss_rpn_cls: 0.002504 loss_rpn_loc: 0.05743 time: 1.1491 last_time: 1.0477 data_time: 0.0244 last_data_time: 0.0278 lr: 0.0014985 max_mem: 3035M
[07/05 15:13:01 d2.utils.events]: eta: 11:50:28 iter: 139 total_loss: 3.964 loss_cls: 1.429 loss_box_reg: 2.317 loss_rpn_cls: 0.002214 loss_rpn_loc: 0.04595 time: 1.1319 last_time: 0.9427 data_time: 0.0263 last_data_time: 0.0322 lr: 0.0017483 max_mem: 3035M
[07/05 15:13:22 d2.utils.events]: eta: 11:45:46 iter: 159 total_loss: 4.074 loss_cls: 1.564 loss_box_reg: 2.516 loss_rpn_cls: 0.002159 loss_rpn_loc: 0.04597 time: 1.1222 last_time: 1.2356 data_time: 0.0193 last_data_time: 0.0105 lr: 0.001998 max_mem: 3035M

Traceback (most recent call last):
File "/second_study_codebase/detectron2/downstream/object_detection/train_object_detection.py", line 99, in
launch(
File "/second_study_codebase/detectron2/detectron2/engine/launch.py", line 69, in launch
mp.start_processes(
File "/home/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 197, in start_processes
while not context.join():
File "/home/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 160, in join
raise ProcessRaisedException(msg, error_index, failed_process.pid)
torch.multiprocessing.spawn.ProcessRaisedException:

-- Process 2 terminated with the following error:
Traceback (most recent call last):
File "/home/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 69, in _wrap
fn(i, *args)
File "/second_study_codebase/detectron2/detectron2/engine/launch.py", line 123, in _distributed_worker
main_func(*args)
File "/second_study_codebase/detectron2/downstream/object_detection/train_object_detection.py", line 93, in main
return trainer.train()
File "/second_study_codebase/detectron2/detectron2/engine/defaults.py", line 484, in train
super().train(self.start_iter, self.max_iter)
File "/second_study_codebase/detectron2/detectron2/engine/train_loop.py", line 155, in train
self.run_step()
File "/second_study_codebase/detectron2/detectron2/engine/defaults.py", line 494, in run_step
self._trainer.run_step()
File "/second_study_codebase/detectron2/detectron2/engine/train_loop.py", line 310, in run_step
loss_dict = self.model(data)
File "/home/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1156, in forward
output = self._run_ddp_forward(*inputs, **kwargs)
File "/home/kalikhademi/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1110, in _run_ddp_forward
return module_to_run(*inputs[0], **kwargs[0]) # type: ignore[index]
File "/home/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/second_study_codebase/detectron2/detectron2/modeling/meta_arch/rcnn.py", line 161, in forward
proposals, proposal_losses = self.proposal_generator(images, features, gt_instances)
File "/home/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/second_study_codebase/detectron2/detectron2/modeling/proposal_generator/rpn.py", line 471, in forward
gt_labels, gt_boxes = self.label_and_sample_anchors(anchors, gt_instances)
File "/home/.local/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/second_study_codebase/detectron2/detectron2/modeling/proposal_generator/rpn.py", line 340, in label_and_sample_anchors
matched_idxs, gt_labels_i = retry_if_cuda_oom(self.anchor_matcher)(match_quality_matrix)
File "/second_study_codebase/detectron2/detectron2/utils/memory.py", line 70, in wrapped
return func(*args, **kwargs)
File "/second_study_codebase/detectron2/detectron2/modeling/matcher.py", line 102, in __call__
self.set_low_quality_matches_(match_labels, match_quality_matrix)
File "/second_study_codebase/detectron2/detectron2/modeling/matcher.py", line 121, in set_low_quality_matches_
pred_inds_with_highest_quality = nonzero_tuple(
File "/second_study_codebase/detectron2/detectron2/layers/wrappers.py", line 153, in nonzero_tuple
return x.nonzero()
RuntimeError: nonzero is not supported for tensors with more than INT_MAX elements, file a support request
```
## Expected behavior:

If there are no obvious crash in "full logs" provided above,
please tell us the expected behavior.

If you expect a model to converge / work better, we do not help with such issues, unless
a model fails to reproduce the results in detectron2 model zoo, or proves existence of bugs.

## Environment:

Paste the output of the following command:
```
wget -nc -nv https://github.com/facebookresearch/detectron2/raw/main/detectron2/utils/collect_env.py && python collect_env.py
```
------------------------------- --------------------------------------------------------------------------------------------------------------------------
sys.platform linux
Python 3.10.11 | packaged by conda-forge | (main, May 10 2023, 18:58:44) [GCC 11.3.0]
numpy 1.25.0
detectron2 0.6 @/orange/juangilbert/kalikhademi/dissertation_vision/dissertation_codebase/second_study_codebase/detectron2/detectron2
Compiler GCC 7.3
CUDA compiler not available
DETECTRON2_ENV_MODULE
PyTorch 2.0.1+cu117 @/home/kalikhademi/.local/lib/python3.10/site-packages/torch
PyTorch debug build False
torch._C._GLIBCXX_USE_CXX11_ABI False
GPU available No: torch.cuda.is_available() == False
Pillow 9.5.0
torchvision 0.15.2+cu117 @/home/kalikhademi/.local/lib/python3.10/site-packages/torchvision
fvcore 0.1.5.post20221221
iopath 0.1.9
cv2 Not found
------------------------------- --------------------------------------------------------------------------------------------------------------------------
PyTorch built with:
- GCC 9.3
- C++ Version: 201703
- Intel(R) oneAPI Math Kernel Library Version 2022.2-Product Build 20220804 for Intel(R) 64 architecture applications
- Intel(R) MKL-DNN v2.7.3 (Git Hash 6dbeffbae1f23cbbeae17adb7b5b13f1f37c080e)
- OpenMP 201511 (a.k.a. OpenMP 4.5)
- LAPACK is enabled (usually provided by MKL)
- NNPACK is enabled
- CPU capability usage: AVX2
- Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CUDA_VERSION=11.7, CUDNN_VERSION=8.5.0, CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, CXX_FLAGS= -D_GLIBCXX_USE_CXX11_ABI=0 -fabi-version=11 -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO -DLIBKINETO_NOROCTRACER -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -O2 -fPIC -Wall -Wextra -Werror=return-type -Werror=non-virtual-dtor -Werror=bool-operation -Wnarrowing -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wunused-local-typedefs -Wno-unused-parameter -Wno-unused-function -Wno-unused-result -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-stringop-overflow -Wno-psabi -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Werror=cast-function-type -Wno-stringop-overflow, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_DISABLE_GPU_ASSERTS=ON, TORCH_VERSION=2.0.1, USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=1, USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF,

If your issue looks like an installation issue / environment issue,
please first check common issues in https://detectron2.readthedocs.io/tutorials/install.html#common-installation-issues