[2025-03-27 11:31:44,995 INFO train.py line 205 1936026] train_data samples: '7206'
/home/hy/PointCT/output/model/model_last.pth
/pytorch/aten/src/ATen/native/cuda/Loss.cu:247: nll_loss_forward_reduce_cuda_kernel_2d: block: [0,0,0], thread: [4,0,0] Assertion t >= 0 && t < n_classes failed.
Traceback (most recent call last):
File "train.py", line 414, in
main()
File "train.py", line 89, in main
main_worker(args.train_gpu, args.ngpus_per_node, args)
File "train.py", line 228, in main_worker
loss_train, mIoU_train, mAcc_train, allAcc_train = train(train_loader, model, criterion, optimizer, epoch)
File "train.py", line 287, in train
loss.backward()
File "/home/hy/anaconda3/envs/PCT/lib/python3.7/site-packages/torch/_tensor.py", line 307, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
File "/home/hy/anaconda3/envs/PCT/lib/python3.7/site-packages/torch/autograd/init.py", line 156, in backward
allow_unreachable=True, accumulate_grad=True) # allow_unreachable flag
RuntimeError: CUDA error: CUBLAS_STATUS_ALLOC_FAILED when calling cublasCreate(handle)
[2025-03-27 11:31:44,995 INFO train.py line 205 1936026] train_data samples: '7206'
/home/hy/PointCT/output/model/model_last.pth
/pytorch/aten/src/ATen/native/cuda/Loss.cu:247: nll_loss_forward_reduce_cuda_kernel_2d: block: [0,0,0], thread: [4,0,0] Assertion
t >= 0 && t < n_classesfailed.Traceback (most recent call last):
File "train.py", line 414, in
main()
File "train.py", line 89, in main
main_worker(args.train_gpu, args.ngpus_per_node, args)
File "train.py", line 228, in main_worker
loss_train, mIoU_train, mAcc_train, allAcc_train = train(train_loader, model, criterion, optimizer, epoch)
File "train.py", line 287, in train
loss.backward()
File "/home/hy/anaconda3/envs/PCT/lib/python3.7/site-packages/torch/_tensor.py", line 307, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
File "/home/hy/anaconda3/envs/PCT/lib/python3.7/site-packages/torch/autograd/init.py", line 156, in backward
allow_unreachable=True, accumulate_grad=True) # allow_unreachable flag
RuntimeError: CUDA error: CUBLAS_STATUS_ALLOC_FAILED when calling
cublasCreate(handle)