• 47、以Orienmask实例分割算法为例子，学习TensorRT的Python和C++开发

    基本思想:一直想学tensorRT开发,没时间,最近有时间,学习一下TensorRT开发,这篇文章的资料大部分来自网络和手册,以学习的目的，促进自己的任务目标实现

    配置环境 50、ubuntu18.04&20.04+CUDA11.1+cudnn11.3+TensorRT7.2+Deepsteam5.1+vulkan环境搭建和YOLO5部署_sxj731533730的博客-CSDN博客

    1. ubuntu@ubuntu:~/glog/build$ python3
    2. Python 3.8.10 (default, Nov 26 2021, 20:14:08) 
    3. [GCC 9.3.0] on linux
    4. Type "help", "copyright", "credits" or "license" for more information.
    5. >>> import tensorrt
    6. >>> tensorrt.__version__
    7. '7.2.3.4

    Documentation Archives :: NVIDIA Deep Learning TensorRT Documentation

    测试环境　显卡GTX1060 笔记本

    第一步:基本的tensorRT结构分析。参考https://www.jianshu.com/p/3c2fb7b45cc7

    1)首先以trt的Logger为参数，使用builder创建计算图类型INetworkDefinition。
2)然后使用Parsers将onnx等网络框架下的结构填充计算图，当然也可以使用tensorrt的API进行构建。
3)由计算图创建cuda环境下的引擎
4)最终进行推理的则是cuda引擎生成的ExecutionContext。engine.create_execution_context()

    补充一个测试onnx测试时间代码

    1. import matplotlib.pyplot as plt
    2. from torch.autograd import Variable
    3. from argparse import ArgumentParser
    4. import torch
    5. import torch.utils.data
    6. import onnxruntime
    7. import cv2
    8. import numpy as np
    9. from onnxruntime.datasets import get_example
    10. import torch.nn.functional as F
    11. import math
    12. from  model.orienmask_yolo_fpnplus import OrienMaskYOLOFPNPlus
    13. from utils.visualizer import InferenceVisualizer
    14. from torch.nn.modules.utils import _pair
    15. from eval.function import batched_nms
    16. from eval.orienmask_yolo_postprocess import OrienMaskYOLOPostProcess
    17. device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    18.  
    19. import os
    20. envpath = '/home/ubuntu/.local/lib/python3.8/site-packages/cv2/qt/plugins/platforms'
    21. os.environ['QT_QPA_PLATFORM_PLUGIN_PATH'] = envpath
    22.  
    23. def pad(image, size_divisor=32, pad_value=0):
    24.     height, width = image.shape[-2:]
    25.     new_height = int(math.ceil(height / size_divisor) * size_divisor)
    26.     new_width = int(math.ceil(width / size_divisor) * size_divisor)
    27.     pad_left, pad_top = (new_width - width) // 2, (new_height - height) // 2
    28.     pad_right, pad_down = new_width - width - pad_left, new_height - height - pad_top
    29.  
    30.     padding = [pad_left, pad_right, pad_top, pad_down]
    31.     image = F.pad(image, padding, value=pad_value)
    32.     pad_info = padding + [new_height, new_width]
    33.  
    34.     return image, pad_info
    35.  
    36.  
    37. def torch2onnx(args, model):
    38.     import datetime
    39.     start_load_data = datetime.datetime.now()
    40.     img_src=cv2.imread(args.img)
    41.     img_color = cv2.cvtColor(img_src, cv2.COLOR_BGR2RGB)
    42.     src_tensor = torch.tensor(img_color, device=device,dtype=torch.float32)
    43.     img_resize = cv2.resize(img_color, (544, 544),cv2.INTER_LINEAR)
    44.     input = np.transpose(img_resize, (2, 0, 1)).astype(np.float32)
    45.     input[0, ...] = (input[0, ...] - 0) / 255  # la
    46.     input[1, ...] = (input[1, ...] - 0) / 255
    47.     input[2, ...] = (input[2, ...] - 0) / 255
    48.     now_image= Variable(torch.from_numpy(input))
    49.     dummy_input = now_image.unsqueeze(0).to(device)
    50.     dummy_input, pad_info = pad(dummy_input)
    51.     end_load_data = datetime.datetime.now()
    52.     print("load data:", (end_load_data - start_load_data).microseconds / 1000, "ms")
    53.     start_convert = datetime.datetime.now()
    54.     torch.onnx.export(model, dummy_input, args.onnx_model_path, input_names=["input"],
    55.                       export_params=True,
    56.                       keep_initializers_as_inputs=True,
    57.                       do_constant_folding=True,
    58.                       verbose=False,
    59.                       opset_version=11)
    60.     end_convert = datetime.datetime.now()
    61.     print("convert model:", (end_convert - start_convert).microseconds / 1000, "ms")
    62.     start_load = datetime.datetime.now()
    63.     example_model = get_example(args.onnx_model_path)
    64.     end_load = datetime.datetime.now()
    65.  
    66.     print("load model:", (end_load - start_load).microseconds / 1000, "ms")
    67.     start_Forward = datetime.datetime.now()
    68.     session = onnxruntime.InferenceSession(example_model)
    69.     input_name = session.get_inputs()[0].name
    70.     result = session.run([], {input_name: dummy_input.data.cpu().numpy()})
    71.  
    72.     result_tuple=((torch.tensor(result[0],device=device),torch.tensor(result[1],device=device)),
    73.                   (torch.tensor(result[2],device=device),torch.tensor(result[3],device=device)),
    74.                   (torch.tensor(result[4],device=device),torch.tensor(result[5],device=device)))
    75.     pred_bbox_batch=[torch.tensor(result[0],device=device),torch.tensor(result[2],device=device),torch.tensor(result[4],device=device)]
    76.     pred_orien_batch=[torch.tensor(result[6],device=device),torch.tensor(result[7],device=device),torch.tensor(result[8],device=device)]
    77.     self_grid_size = [[17, 17], [34, 34], [68, 68]]
    78.     self_image_size = [544, 544]
    79.     self_anchors = [[12, 16], [19, 36], [40, 28], [36, 75], [76, 55], [72, 146], [142, 110], [192, 243], [459, 401]]
    80.     self_anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
    81.     self_num_classes = 80
    82.     self_conf_thresh = 0.05
    83.     self_nms_func = None
    84.     self_nms_pre = 400
    85.     self_nms_post = 100
    86.     self_orien_thresh = 0.3
    87.  
    88.     item_Orien=OrienMaskYOLOPostProcess(self_grid_size, self_image_size, self_anchors, self_anchor_mask, self_num_classes,
    89.                  self_conf_thresh, self_nms_func, self_nms_pre,
    90.                  self_nms_post, self_orien_thresh, device)
    91.     predictions =item_Orien.apply(result_tuple,pred_bbox_batch,pred_orien_batch)
    92.     end_Forward = datetime.datetime.now()
    93.     print("Forward & Postprocess:", (end_Forward - start_Forward).microseconds / 1000, "ms")
    94.     start_visual = datetime.datetime.now()
    95.     dataset='COCO'
    96.     with_mask=True
    97.     conf_thresh=0.3
    98.     alpha=0.6
    99.     line_thickness=1
    100.     ifer_item=InferenceVisualizer(dataset,device, with_mask,conf_thresh,alpha,line_thickness)
    101.     show_image = ifer_item.__call__(predictions[0], src_tensor,pad_info)
    102.     plt.imsave(args.onnxoutput, show_image)
    103.     end_visual =datetime.datetime.now()
    104.     print("Visualize::", (end_visual - start_visual).microseconds/1000, "ms")
    105.  
    106. def main():
    107.     """Test a single image."""
    108.     parser = ArgumentParser()
    109.     parser.add_argument('--img', default="/home/ubuntu/OrienMask/assets/000000163126.jpg",
    110.                         help='Image file')
    111.     parser.add_argument('--weights', default="/home/ubuntu/CLionProjects/model/orienmask_yolo.pth",
    112.                         help='Checkpoint file')
    113.     parser.add_argument('--onnx_model_path',
    114.                         default="/home/ubuntu/CLionProjects/model/orienmask_yolo.onnx",
    115.                         help='onnx_model_path')
    116.     parser.add_argument('--device', default='cuda:0', help='Device used for inference')
    117.     parser.add_argument('--onnxoutput', default=r'onnxsxj731533730.jpg', help='Output image')
    118.     parser.add_argument('--num_anchors', type=int, default=3, help='num_anchors')
    119.     parser.add_argument('--num_classes', type=int, default=80, help='num_classes')
    120.     args = parser.parse_args()
    121.  
    122.  
    123.     model=OrienMaskYOLOFPNPlus(args.num_anchors,args.num_classes).to(device)
    124.     weights = torch.load(args.weights, map_location=device)
    125.     weights = weights['state_dict'] if 'state_dict' in weights else weights
    126.     model.load_state_dict(weights, strict=True)
    127.     torch2onnx(args, model)
    128.  
    129.  
    130. if __name__ == '__main__':
    131.     main()

    第一步:转模型Developer Guide :: NVIDIA Deep Learning TensorRT Documentation

    1. import tensorrt as trt
    2.  
    3. def build_engine(onnx_file_path,engine_file_path,half=False):
    4.     """Takes an ONNX file and creates a TensorRT engine to run inference with"""
    5.     logger = trt.Logger(trt.Logger.INFO)
    6.     builder = trt.Builder(logger)
    7.     config = builder.create_builder_config()
    8.     config.max_workspace_size = 4 * 1 << 30
    9.     flag = (1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))
    10.     network = builder.create_network(flag)
    11.     parser = trt.OnnxParser(network, logger)
    12.     if not parser.parse_from_file(str(onnx_file_path)):
    13.         raise RuntimeError(f'failed to load ONNX file: {onnx_file_path}')
    14.     half &= builder.platform_has_fast_fp16
    15.     if half:
    16.         config.set_flag(trt.BuilderFlag.FP16)
    17.     with builder.build_engine(network, config) as engine, open(engine_file_path, 'wb') as t:
    18.         t.write(engine.serialize())
    19.     return engine_file_path
    20. if __name__ =="__main__":
    21.     onnx_file_path = "/home/ubuntu/CLionProjects/D435_OrienMask/model/orienmask_yolo_sim.onnx"
    22.     engine_file_path = "/home/ubuntu/CLionProjects/D435_OrienMask/model/orienmask_yolo_sim.engine"
    23.     build_engine(onnx_file_path,engine_file_path,True)

    转换结果　注意　如果提示空间不够，需要修改配置项,将30 改小一点

     config.max_workspace_size = 4 * 1 << 30

    转换过程

    1. /usr/bin/python3.8 /home/ubuntu/OrienMask/onnx2trt.py
    2. [TensorRT] WARNING: TensorRT was linked against cuBLAS/cuBLAS LT 11.3.0 but loaded cuBLAS/cuBLAS LT 11.2.1
    3. [TensorRT] INFO: Some tactics do not have sufficient workspace memory to run. Increasing workspace size may increase performance, please check verbose output.
    4. [TensorRT] INFO: Detected 1 inputs and 9 output network tensors.
    5. [TensorRT] WARNING: TensorRT was linked against cuBLAS/cuBLAS LT 11.3.0 but loaded cuBLAS/cuBLAS LT 11.2.1
    6.  
    7. Process finished with exit code 0

    测试结果数据比对

    1. import tensorrt as trt
    2. import pycuda.driver as cuda
    3. import pycuda.autoinit
    4. from pycuda.tools import make_default_context
    5. import torch
    6. import numpy as np
    7. import math
    8. import torch.nn.functional as F
    9. import cv2
    10. from torch.autograd import Variable
    11. device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    12.  
    13. def load_engine(engine_path):
    14.     # TRT_LOGGER = trt.Logger(trt.Logger.WARNING)  # INFO
    15.     TRT_LOGGER = trt.Logger(trt.Logger.ERROR)
    16.     with open(engine_path, 'rb') as f, trt.Runtime(TRT_LOGGER) as runtime:
    17.         return runtime.deserialize_cuda_engine(f.read())
    18.  
    19.  
    20. def pad(image, size_divisor=32, pad_value=0):
    21.     height, width = image.shape[-2:]
    22.     new_height = int(math.ceil(height / size_divisor) * size_divisor)
    23.     new_width = int(math.ceil(width / size_divisor) * size_divisor)
    24.     pad_left, pad_top = (new_width - width) // 2, (new_height - height) // 2
    25.     pad_right, pad_down = new_width - width - pad_left, new_height - height - pad_top
    26.  
    27.     padding = [pad_left, pad_right, pad_top, pad_down]
    28.     image = F.pad(image, padding, value=pad_value)
    29.     pad_info = padding + [new_height, new_width]
    30.  
    31.     return image, pad_info
    32.  
    33.  
    34. img_src = cv2.imread('/home/ubuntu/OrienMask/assets/000000163126.jpg')
    35. img_color = cv2.cvtColor(img_src, cv2.COLOR_BGR2RGB)
    36. img_resize = cv2.resize(img_color, (544, 544), cv2.INTER_LINEAR)
    37. input = np.transpose(img_resize, (2, 0, 1)).astype(np.float32)
    38. input[0, ...] = (input[0, ...] - 0) / 255  # la
    39. input[1, ...] = (input[1, ...] - 0) / 255
    40. input[2, ...] = (input[2, ...] - 0) / 255
    41. now_image = Variable(torch.from_numpy(input))
    42. dummy_input = now_image.unsqueeze(0)
    43. dummy_input, pad_info = pad(dummy_input)
    44. image=np.array(dummy_input.contiguous())
    45. path = "/home/ubuntu/CLionProjects/D435_OrienMask/model/orienmask_yolo_sim.engine"
    46. # 1. 建立模型，构建上下文管理器
    47. engine = load_engine(path)
    48. context = engine.create_execution_context()
    49. context.active_optimization_profile = 0
    50.  
    51. # 3.分配内存空间，并进行数据cpu到gpu的拷贝
    52. # 动态尺寸，每次都要set一下模型输入的shape，0代表的就是输入，输出根据具体的网络结构而定，可以是0,1,2,3...其中的某个头。
    53. context.set_binding_shape(0, image.shape)
    54.  
    55.  
    56. d_input = cuda.mem_alloc(image.nbytes)  # 分配输入的内存。
    57.  
    58.  
    59. output_shape_1 = context.get_binding_shape(1)
    60. output_shape_2 = context.get_binding_shape(2)
    61. output_shape_3 = context.get_binding_shape(3)
    62. output_shape_4 = context.get_binding_shape(4)
    63. output_shape_5 = context.get_binding_shape(5)
    64. output_shape_6 = context.get_binding_shape(6)
    65. output_shape_7 = context.get_binding_shape(7)
    66. output_shape_8 = context.get_binding_shape(8)
    67. output_shape_9 = context.get_binding_shape(9)
    68. buffer_1 = np.empty(output_shape_1, dtype=np.float32)
    69. buffer_2 = np.empty(output_shape_2, dtype=np.float32)
    70. buffer_3 = np.empty(output_shape_3, dtype=np.float32)
    71. buffer_4 = np.empty(output_shape_4, dtype=np.float32)
    72. buffer_5 = np.empty(output_shape_5, dtype=np.float32)
    73. buffer_6 = np.empty(output_shape_6, dtype=np.float32)
    74. buffer_7 = np.empty(output_shape_7, dtype=np.float32)
    75. buffer_8 = np.empty(output_shape_8, dtype=np.float32)
    76. buffer_9 = np.empty(output_shape_9, dtype=np.float32)
    77. d_output_1 = cuda.mem_alloc(buffer_1.nbytes)  # 分配输出内存。
    78. d_output_2 = cuda.mem_alloc(buffer_2.nbytes)  # 分配输出内存
    79. d_output_3 = cuda.mem_alloc(buffer_3.nbytes)  # 分配输出内存
    80. d_output_4 = cuda.mem_alloc(buffer_4.nbytes)  # 分配输出内存
    81. d_output_5 = cuda.mem_alloc(buffer_5.nbytes)  # 分配输出内存
    82. d_output_6 = cuda.mem_alloc(buffer_6.nbytes)  # 分配输出内存
    83. d_output_7 = cuda.mem_alloc(buffer_7.nbytes)  # 分配输出内存
    84. d_output_8 = cuda.mem_alloc(buffer_8.nbytes)  # 分配输出内存
    85. d_output_9 = cuda.mem_alloc(buffer_9.nbytes)  # 分配输出内存
    86.  
    87. cuda.memcpy_htod(d_input, image)
    88. bindings = [d_input, d_output_1,d_output_2,d_output_3,d_output_4,d_output_5,d_output_6,d_output_7,d_output_8,d_output_9]
    89.  
    90. # 4.进行推理，并将结果从gpu拷贝到cpu。
    91. context.execute_v2(bindings)  # 可异步和同步
    92. cuda.memcpy_dtoh(buffer_1, d_output_1)
    93. output_1 = buffer_1.reshape(output_shape_1)
    94. print(output_1.shape)
    95. cuda.memcpy_dtoh(buffer_2, d_output_2)
    96. output_2 = buffer_2.reshape(output_shape_2)
    97. print(output_2.shape)
    98. cuda.memcpy_dtoh(buffer_3, d_output_3)
    99. output_3 = buffer_3.reshape(output_shape_3)
    100. print(output_3.shape)
    101. cuda.memcpy_dtoh(buffer_4, d_output_4)
    102. output_4 = buffer_4.reshape(output_shape_4)
    103. print(output_4.shape)
    104. cuda.memcpy_dtoh(buffer_5, d_output_5)
    105. output_5 = buffer_5.reshape(output_shape_5)
    106. print(output_5.shape)
    107. cuda.memcpy_dtoh(buffer_6, d_output_6)
    108. output_6 = buffer_6.reshape(output_shape_6)
    109. print(output_6.shape)
    110. cuda.memcpy_dtoh(buffer_7, d_output_7)
    111. output_7 = buffer_7.reshape(output_shape_7)
    112. print(output_7.shape)
    113. cuda.memcpy_dtoh(buffer_8, d_output_8)
    114. output_8 = buffer_8.reshape(output_shape_8)
    115. print(output_8.shape)
    116. cuda.memcpy_dtoh(buffer_9, d_output_9)
    117. output_9 = buffer_9.reshape(output_shape_9)
    118. print(output_9.shape)
    119.  
    120.

    数据比对onnx和tengine是一致的,onnx的数据 

    engine的数据

     测试一下时间,只比较推理时间，后处理一致 engine完整的推理代码(含转模型)

    1. import datetime
    2.  
    3. import tensorrt as trt
    4. import matplotlib.pyplot as plt
    5. import pycuda.driver as cuda
    6. import pycuda.autoinit
    7. from pycuda.tools import make_default_context
    8. import torch
    9. import numpy as np
    10. import math
    11. import torch.nn.functional as F
    12. import cv2
    13. from torch.autograd import Variable
    14. from argparse import ArgumentParser
    15. from eval.orienmask_yolo_postprocess import OrienMaskYOLOPostProcess
    16. from utils.visualizer import InferenceVisualizer
    17.  
    18. device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    19.  
    20.  
    21. def build_engine(onnx_file_path, engine_file_path, half=False):
    22.     """Takes an ONNX file and creates a TensorRT engine to run inference with"""
    23.     logger = trt.Logger(trt.Logger.INFO)
    24.     builder = trt.Builder(logger)
    25.     config = builder.create_builder_config()
    26.     config.max_workspace_size = 4 * 1 << 20
    27.     flag = (1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))
    28.     network = builder.create_network(flag)
    29.     parser = trt.OnnxParser(network, logger)
    30.     if not parser.parse_from_file(str(onnx_file_path)):
    31.         raise RuntimeError(f'failed to load ONNX file: {onnx_file_path}')
    32.     half &= builder.platform_has_fast_fp16
    33.     if half:
    34.         config.set_flag(trt.BuilderFlag.FP16)
    35.     with builder.build_engine(network, config) as engine, open(engine_file_path, 'wb') as t:
    36.         t.write(engine.serialize())
    37.  
    38.     return engine_file_path
    39.  
    40.  
    41. def load_engine(engine_path):
    42.     # TRT_LOGGER = trt.Logger(trt.Logger.WARNING)  # INFO
    43.     TRT_LOGGER = trt.Logger(trt.Logger.ERROR)
    44.     with open(engine_path, 'rb') as f, trt.Runtime(TRT_LOGGER) as runtime:
    45.         return runtime.deserialize_cuda_engine(f.read())
    46.  
    47.  
    48. def pad(image, size_divisor=32, pad_value=0):
    49.     height, width = image.shape[-2:]
    50.     new_height = int(math.ceil(height / size_divisor) * size_divisor)
    51.     new_width = int(math.ceil(width / size_divisor) * size_divisor)
    52.     pad_left, pad_top = (new_width - width) // 2, (new_height - height) // 2
    53.     pad_right, pad_down = new_width - width - pad_left, new_height - height - pad_top
    54.  
    55.     padding = [pad_left, pad_right, pad_top, pad_down]
    56.     image = F.pad(image, padding, value=pad_value)
    57.     pad_info = padding + [new_height, new_width]
    58.     return image, pad_info
    59.  
    60.  
    61. def onnx2engine(args):
    62.     start_convert = datetime.datetime.now()
    63.     build_engine(args.onnx_model_path, args.engine_file_path, args.fp16)
    64.     end_convert = datetime.datetime.now()
    65.     print("convert model:", (end_convert - start_convert).microseconds / 1000, "ms")
    66.     start_load_data = datetime.datetime.now()
    67.     img_src = cv2.imread(args.img)
    68.     img_color = cv2.cvtColor(img_src, cv2.COLOR_BGR2RGB)
    69.  
    70.     img_resize = cv2.resize(img_color, (544, 544), cv2.INTER_LINEAR)
    71.     input = np.transpose(img_resize, (2, 0, 1)).astype(np.float32)
    72.     input[0, ...] = (input[0, ...] - 0) / 255  # la
    73.     input[1, ...] = (input[1, ...] - 0) / 255
    74.     input[2, ...] = (input[2, ...] - 0) / 255
    75.     now_image = Variable(torch.from_numpy(input))
    76.     dummy_input = now_image.unsqueeze(0)
    77.     dummy_input, pad_info = pad(dummy_input)
    78.     image = np.array(dummy_input.contiguous())
    79.     end_load_data = datetime.datetime.now()
    80.     print("load data:", (end_load_data - start_load_data).microseconds / 1000, "ms")
    81.     start_load = datetime.datetime.now()
    82.     # 1. 建立模型，构建上下文管理器
    83.     engine = load_engine(args.engine_file_path)
    84.     end_load = datetime.datetime.now()
    85.  
    86.     print("load model:", (end_load - start_load).microseconds / 1000, "ms")
    87.     start_Forward = datetime.datetime.now()
    88.     context = engine.create_execution_context()
    89.     context.active_optimization_profile = 0
    90.  
    91.     # 3.分配内存空间，并进行数据cpu到gpu的拷贝
    92.     # 动态尺寸，每次都要set一下模型输入的shape，0代表的就是输入，输出根据具体的网络结构而定，可以是0,1,2,3...其中的某个头。
    93.     context.set_binding_shape(0, image.shape)
    94.     d_input = cuda.mem_alloc(image.nbytes)  # 分配输入的内存。
    95.     output_shape_1 = context.get_binding_shape(1)
    96.     output_shape_2 = context.get_binding_shape(2)
    97.     output_shape_3 = context.get_binding_shape(3)
    98.     output_shape_4 = context.get_binding_shape(4)
    99.     output_shape_5 = context.get_binding_shape(5)
    100.     output_shape_6 = context.get_binding_shape(6)
    101.     output_shape_7 = context.get_binding_shape(7)
    102.     output_shape_8 = context.get_binding_shape(8)
    103.     output_shape_9 = context.get_binding_shape(9)
    104.     buffer_1 = np.empty(output_shape_1, dtype=np.float32)
    105.     buffer_2 = np.empty(output_shape_2, dtype=np.float32)
    106.     buffer_3 = np.empty(output_shape_3, dtype=np.float32)
    107.     buffer_4 = np.empty(output_shape_4, dtype=np.float32)
    108.     buffer_5 = np.empty(output_shape_5, dtype=np.float32)
    109.     buffer_6 = np.empty(output_shape_6, dtype=np.float32)
    110.     buffer_7 = np.empty(output_shape_7, dtype=np.float32)
    111.     buffer_8 = np.empty(output_shape_8, dtype=np.float32)
    112.     buffer_9 = np.empty(output_shape_9, dtype=np.float32)
    113.     d_output_1 = cuda.mem_alloc(buffer_1.nbytes)  # 分配输出内存。
    114.     d_output_2 = cuda.mem_alloc(buffer_2.nbytes)  # 分配输出内存
    115.     d_output_3 = cuda.mem_alloc(buffer_3.nbytes)  # 分配输出内存
    116.     d_output_4 = cuda.mem_alloc(buffer_4.nbytes)  # 分配输出内存
    117.     d_output_5 = cuda.mem_alloc(buffer_5.nbytes)  # 分配输出内存
    118.     d_output_6 = cuda.mem_alloc(buffer_6.nbytes)  # 分配输出内存
    119.     d_output_7 = cuda.mem_alloc(buffer_7.nbytes)  # 分配输出内存
    120.     d_output_8 = cuda.mem_alloc(buffer_8.nbytes)  # 分配输出内存
    121.     d_output_9 = cuda.mem_alloc(buffer_9.nbytes)  # 分配输出内存
    122.  
    123.     cuda.memcpy_htod(d_input, image)
    124.     bindings = [d_input, d_output_1, d_output_2, d_output_3, d_output_4, d_output_5, d_output_6, d_output_7, d_output_8,
    125.                 d_output_9]
    126.  
    127.     # 4.进行推理，并将结果从gpu拷贝到cpu。
    128.     context.execute_v2(bindings)  # 可异步和同步
    129.     cuda.memcpy_dtoh(buffer_1, d_output_1)
    130.     output_1 = buffer_1.reshape(output_shape_1)
    131.     print(output_1.shape)
    132.     cuda.memcpy_dtoh(buffer_2, d_output_2)
    133.     output_2 = buffer_2.reshape(output_shape_2)
    134.     print(output_2.shape)
    135.     cuda.memcpy_dtoh(buffer_3, d_output_3)
    136.     output_3 = buffer_3.reshape(output_shape_3)
    137.     print(output_3.shape)
    138.     cuda.memcpy_dtoh(buffer_4, d_output_4)
    139.     output_4 = buffer_4.reshape(output_shape_4)
    140.     print(output_4.shape)
    141.     cuda.memcpy_dtoh(buffer_5, d_output_5)
    142.     output_5 = buffer_5.reshape(output_shape_5)
    143.     print(output_5.shape)
    144.     cuda.memcpy_dtoh(buffer_6, d_output_6)
    145.     output_6 = buffer_6.reshape(output_shape_6)
    146.     print(output_6.shape)
    147.     cuda.memcpy_dtoh(buffer_7, d_output_7)
    148.     output_7 = buffer_7.reshape(output_shape_7)
    149.     print(output_7.shape)
    150.     cuda.memcpy_dtoh(buffer_8, d_output_8)
    151.     output_8 = buffer_8.reshape(output_shape_8)
    152.     print(output_8.shape)
    153.     cuda.memcpy_dtoh(buffer_9, d_output_9)
    154.     output_9 = buffer_9.reshape(output_shape_9)
    155.     print(output_9.shape)
    156.  
    157.     result_tuple = ((torch.tensor(output_1, device=device), torch.tensor(output_4, device=device)),
    158.                     (torch.tensor(output_2, device=device), torch.tensor(output_5, device=device)),
    159.                     (torch.tensor(output_3, device=device), torch.tensor(output_6, device=device)))
    160.     pred_bbox_batch = [torch.tensor(output_1, device=device), torch.tensor(output_2, device=device),
    161.                        torch.tensor(output_3, device=device)]
    162.     pred_orien_batch = [torch.tensor(output_7, device=device), torch.tensor(output_8, device=device),
    163.                         torch.tensor(output_9, device=device)]
    164.     self_grid_size = [[17, 17], [34, 34], [68, 68]]
    165.     self_image_size = [544, 544]
    166.     self_anchors = [[12, 16], [19, 36], [40, 28], [36, 75], [76, 55], [72, 146], [142, 110], [192, 243], [459, 401]]
    167.     self_anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
    168.     self_num_classes = 80
    169.     self_conf_thresh = 0.05
    170.     self_nms_func = None
    171.     self_nms_pre = 400
    172.     self_nms_post = 100
    173.     self_orien_thresh = 0.3
    174.     item_Orien = OrienMaskYOLOPostProcess(self_grid_size, self_image_size, self_anchors, self_anchor_mask,
    175.                                           self_num_classes,
    176.                                           self_conf_thresh, self_nms_func, self_nms_pre,
    177.                                           self_nms_post, self_orien_thresh, device)
    178.     predictions = item_Orien.apply(result_tuple, pred_bbox_batch, pred_orien_batch)
    179.     end_Forward = datetime.datetime.now()
    180.     print("Forward & Postprocess:", (end_Forward - start_Forward).microseconds / 1000, "ms")
    181.  
    182.     start_visual = datetime.datetime.now()
    183.  
    184.     dataset = 'COCO'
    185.     with_mask = True
    186.     conf_thresh = 0.3
    187.     alpha = 0.6
    188.     line_thickness = 1
    189.     ifer_item = InferenceVisualizer(dataset, device, with_mask, conf_thresh, alpha, line_thickness)
    190.     show_image = ifer_item.__call__(predictions[0], torch.tensor(img_color, device=device, dtype=torch.float32),
    191.                                     pad_info)
    192.     plt.imsave(args.engineoutput, show_image)
    193.     end_visual = datetime.datetime.now()
    194.     print("Visualize::", (end_visual - start_visual).microseconds / 1000, "ms")
    195.  
    196.  
    197. def main():
    198.     """Test a single image."""
    199.     parser = ArgumentParser()
    200.     parser.add_argument('--img', default="/home/ubuntu/OrienMask/assets/000000163126.jpg",
    201.                         help='Image file')
    202.     parser.add_argument('--onnx_model_path',
    203.                         default="/home/ubuntu/OrienMask/checkpoints/orienmask_yolo.onnx",
    204.                         help='onnx_model_path')
    205.     parser.add_argument('--engine_file_path',
    206.                         default="/home/ubuntu/OrienMask/checkpoints/orienmask_yolo.engine",
    207.                         help='Checkpoint file')
    208.     parser.add_argument('--fp16', default=False, help='Device used for inference')
    209.     parser.add_argument('--device', default='cuda:0', help='Device used for inference')
    210.     parser.add_argument('--engineoutput', default=r'enginesxj731533730.jpg', help='Output image')
    211.     parser.add_argument('--num_anchors', type=int, default=3, help='num_anchors')
    212.     parser.add_argument('--num_classes', type=int, default=80, help='num_classes')
    213.     args = parser.parse_args()
    214.  
    215.     onnx2engine(args)
    216.  
    217.  
    218. if __name__ == '__main__':
    219.     main()

    python 对应infer.py　pt模型时间

    1. [406, 194, 623, 435] 0.9981988668441772 cup
    2. [114, 96, 401, 458] 0.9951307773590088 cup
    3. [399, 292, 450, 343] 0.8919668793678284 baseball-glove
    4. [379, 61, 531, 178] 0.7934483289718628 baseball-bat
    5. The inference takes 1.511667236328125 seconds.
    6. The average inference time is 1511.67 ms (0.66 fps)
    7. Load data: 117.35ms (8.52fps)
    8. Forward & Postprocess: 1329.60ms (0.75fps)
    9. Visualize: 11.64ms (85.92fps)
    10. 100%|██████████| 1/1 [00:01<00:00,  1.46s/it]

    onnx的时间　

    1. /usr/bin/python3.8 /home/ubuntu/OrienMask/pytorch2onnx.py
    2. load data: 27.229 ms
    3. convert model: 472.885 ms
    4. load model: 0.066 ms
    5. Forward & Postprocess: 364.452 ms
    6. [406, 194, 623, 435] 0.9981685876846313 cup
    7. [114, 96, 401, 458] 0.9952691793441772 cup
    8. [399, 292, 450, 343] 0.8922784328460693 baseball-glove
    9. [379, 61, 531, 178] 0.7953709363937378 baseball-bat
    10. Visualize:: 14.279 ms

    tensorRT的时间fp32

    1. /usr/bin/python3.8 /home/ubuntu/OrienMask/onn2trt.py
    2. [TensorRT] WARNING: TensorRT was linked against cuBLAS/cuBLAS LT 11.3.0 but loaded cuBLAS/cuBLAS LT 11.2.1
    3. [TensorRT] INFO: Some tactics do not have sufficient workspace memory to run. Increasing workspace size may increase performance, please check verbose output.
    4. [TensorRT] INFO: Detected 1 inputs and 9 output network tensors.
    5. [TensorRT] WARNING: TensorRT was linked against cuBLAS/cuBLAS LT 11.3.0 but loaded cuBLAS/cuBLAS LT 11.2.1
    6. convert model: 112.855 ms
    7. load data: 11.616 ms
    8. load model: 871.787 ms
    9. (1, 255, 17, 17)
    10. (1, 255, 34, 34)
    11. (1, 255, 68, 68)
    12. (1, 6, 136, 136)
    13. (1, 6, 136, 136)
    14. (1, 6, 136, 136)
    15. (1, 6, 544, 544)
    16. (1, 6, 544, 544)
    17. (1, 6, 544, 544)
    18. excute onnx infer
    19. Forward & Postprocess: 932.863 ms
    20. [406, 194, 623, 435] 0.9981685876846313 cup
    21. [114, 96, 401, 458] 0.9952691793441772 cup
    22. [399, 292, 450, 343] 0.892278790473938 baseball-glove
    23. [379, 61, 531, 178] 0.7953709363937378 baseball-bat
    24. Visualize:: 280.247 ms
    25.  
    26. Process finished with exit code 0

    tensorRT的时间fp16

    1. /usr/bin/python3.8 /home/ubuntu/OrienMask/onn2trt.py
    2. [TensorRT] WARNING: TensorRT was linked against cuBLAS/cuBLAS LT 11.3.0 but loaded cuBLAS/cuBLAS LT 11.2.1
    3. [TensorRT] INFO: Some tactics do not have sufficient workspace memory to run. Increasing workspace size may increase performance, please check verbose output.
    4. [TensorRT] INFO: Detected 1 inputs and 9 output network tensors.
    5. [TensorRT] WARNING: TensorRT was linked against cuBLAS/cuBLAS LT 11.3.0 but loaded cuBLAS/cuBLAS LT 11.2.1
    6. convert model: 21.035 ms
    7. load data: 10.628 ms
    8. load model: 887.876 ms
    9. (1, 255, 17, 17)
    10. (1, 255, 34, 34)
    11. (1, 255, 68, 68)
    12. (1, 6, 136, 136)
    13. (1, 6, 136, 136)
    14. (1, 6, 136, 136)
    15. (1, 6, 544, 544)
    16. (1, 6, 544, 544)
    17. (1, 6, 544, 544)
    18. excute onnx infer
    19. Forward & Postprocess: 919.326 ms
    20. [406, 194, 623, 435] 0.9981685876846313 cup
    21. [114, 96, 401, 458] 0.9952691793441772 cup
    22. [399, 292, 450, 343] 0.892278790473938 baseball-glove
    23. [379, 61, 531, 178] 0.7953709363937378 baseball-bat
    24. Visualize:: 53.748 ms
    25.  
    26. Process finished with exit code 0

    感觉我显卡太拉跨了，还不如onnx快....

    c++代码待学习补充

    参考:

     Documentation Archives :: NVIDIA Deep Learning TensorRT Documentation

  • 相关阅读:
    百度文心一言GPT免费入口也来了！！！
    郑卢高速洛阳至洛宁段路基路面综合设计K14+000-K15+400设计计算书+cad图纸
    Mongodb索引的创建与命名
    真的，Java并发编程基础入门看这个就够了
    1.6 列表（Python）
    对比SQL学习power bi--(1)分组求和后，再平均！
    毕业设计之基于node.js+Vue+Element驾校信息管理系统
    ESB（企业服务总线）
    二十三、生成帮助文档
    电力电子转战数字IC20220729day59——uvm入门实验1
  • 原文地址：https://blog.csdn.net/sxj731533730/article/details/126118443