confusion about the cuda stability.

[xjock]

import os
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
import cv2
import torch
import torchvision
from vpf import PyNvCodec as nvc
from vpf import PytorchNvCodec as pnvc
print("PyTorch Version {}".format(torch.__version__))
print("Cuda Version {}".format(torch.version.cuda))
print("CUDNN Version {}".format(torch.backends.cudnn.version()))

device = torch.device('cuda:0')
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(
    pretrained=True)
model.float()
model.to(device)
model.eval()

gpuID = 0
encFilePath = '/home/lhcz/sdb/data/bazhou_videos/videos/videos/20210103/6912003160731176192.mp4'
nvDec = nvc.PyNvDecoder(encFilePath, gpuID)
to_rgb = nvc.PySurfaceConverter(nvDec.Width(), nvDec.Height(), nvc.PixelFormat.NV12, nvc.PixelFormat.RGB, gpuID)
to_planar = nvc.PySurfaceConverter(nvDec.Width(), nvDec.Height(), nvc.PixelFormat.RGB, nvc.PixelFormat.RGB_PLANAR,
                                   gpuID)

cnt_frame = 0
while True:
    rawSurface = nvDec.DecodeSingleSurface()
    if (rawSurface.Empty()):
        break
    rgb_byte = to_rgb.Execute(rawSurface)
    rgb_planar = to_planar.Execute(rgb_byte)
    surfPlane = rgb_planar.PlanePtr()
    surface_tensor = pnvc.makefromDevicePtrUint8(surfPlane.GpuMem(), surfPlane.Width(), surfPlane.Height(),
                                                 surfPlane.Pitch(), surfPlane.ElemSize())
    


    print(surface_tensor)



    print(cnt_frame)
    cnt_frame += 1
    # surface_tensor = surface_tensor.type(dtype=torch.cuda.FloatTensor)
    # do inference here
    img_tensor = (surface_tensor.resize_(1, 3, nvDec.Height(), nvDec.Width()) / 255.)#.float().cuda()
    out = model(img_tensor)
    boxes = out[0]['boxes']
    labels = out[0]['labels']
    scores = out[0]['scores']
    for idx in range(boxes.shape[0]):
        if scores[idx] >= 0.8:
            x1, y1, x2, y2 = boxes[idx][0], boxes[idx][1], boxes[idx][2], boxes[idx][3]
            print(int(x1), int(y1), int(x2), int(y2), float(scores[idx]), str(labels[idx].item()))

The above code runs well with the line print(surface_tensor)

However, if I comment the line print(surface_tensor), the above code return the following error

/root/anaconda3/envs/insightface/bin/python /home/lhcz/sdb/dev/vpf_test3.py
PyTorch Version 1.6.0
Cuda Version 10.2
CUDNN Version 7605
Decoding on GPU 0
/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torchvision/ops/boxes.py:101: UserWarning: This overload of nonzero is deprecated:
nonzero()
Consider using one of the following signatures instead:
nonzero(*, bool as_tuple) (Triggered internally at /pytorch/torch/csrc/utils/python_arg_parser.cpp:766.)
keep = keep.nonzero().squeeze(1)
257 436 438 913 0.997122585773468 1
330 599 636 1080 0.9901776909828186 1
469 714 666 1069 0.9539728760719299 62
169 834 385 1027 0.8340146541595459 62
436 861 557 1043 0.8114858865737915 62
Traceback (most recent call last):
File "/home/lhcz/sdb/dev/ethnic_face_cls/vpf_test3.py", line 47, in
out = model(img_tensor)
File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
result = self.forward(*input, **kwargs)
File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torchvision/models/detection/generalized_rcnn.py", line 95, in forward
features = self.backbone(images.tensors)
File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
result = self.forward(*input, **kwargs)
File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torchvision/models/detection/backbone_utils.py", line 39, in forward
x = self.body(x)
File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
result = self.forward(*input, **kwargs)
File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torchvision/models/_utils.py", line 63, in forward
x = module(x)
File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
result = self.forward(*input, **kwargs)
File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 419, in forward
return self._conv_forward(input, self.weight)
File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 416, in _conv_forward
self.padding, self.dilation, self.groups)
RuntimeError: cuDNN error: CUDNN_STATUS_MAPPING_ERROR

Process finished with exit code 1

[courtfu]

I met the same error.