• 如何使用nuScenes数据集格式的单帧数据推理(以DETR3D为例)

    【请尊重原创！转载和引用文章内容务必注明出处！未经许可上传到某文库或其他收费阅读/下载网站赚钱的必追究责任！】

    无论是mmdetection3D https://github.com/open-mmlab/mmdetection3d还是OpenPCDet  https://github.com/open-mmlab/OpenPCDet都只有使用数据集(使用哪个数据集由配置文件里指定)训练和测试的代码，没有使用某种数据集格式的单帧数据进行推理的代码(也就是一些2D目标检测框里里提供的推理demo代码)，而OpenPCDet尤其是mmdetection3D里面为了支持多种不同的数据集和多种不同模型的训练和测试，把很多实现步骤高度配置化，这是好事也是坏事，如果你只是使用这些框架(尤其是使用它们已支持的公开数据集)进行训练和测试，那么相对简单，如果是要使用这些框架对单帧数据(可以是他们支持的公开数据集里的数据或者遵循这些数据集的格式自己制作的数据集的数据)进行推理，就比较复杂了，框架没有提供工具代码只能自己去写，写之前当然得把框架的相关代码翻看几遍把支持某个数据集和模型的配置文件和分散在多处的代码的逻辑理顺连贯起来，不然没法下手，因为框架高度配置化的话相关的代码就会非常分散在连贯性可读性方面很差，需要花时间先理清楚，然后把那些配置化的相关零散代码都整理并修改和串联起来，完成数据的多步预处理、调用模型推理、对推理结果的解析和后处理的全部流程的实现。

    用nuScense格式数据调用detr3d模型的一个最烦的地方就是，你在调用模型推理时非得提供对应的img_metas数据，模型的这个设计我觉得不是很合理，起码不算友好，对于硬件和相关标定参数定了后，这种参数完全可以通过配置隐式提供给模型，为何每次调用都得提供这些meta参数呢！反正我写下面的推理代码之前是浪费了不少时间在厘清img_metas这个参数的数据该怎么准备上！

    此处以DETR3D最初的官方实现代码GitHub - WangYueFt/detr3d为基础给出了相关数据预处理和模型调用及推理结果后处理等代码，另外，这些代码是用于在ROS2节点中运行，解析出的推理结果还按项目实际要求转换成autoware的ObjectDetection数据格式后发布，所以还包含了ROS2和autoware相关的代码，这些代码不是重点仅供做代码逻辑完整性展示。当然，如果你项目中有类似需要的话，可以直接拷贝使用这些代码。

    1. import rclpy
    2. from rclpy.node import Node
    3. from rclpy.executors import MultiThreadedExecutor
    4. from rclpy.callback_groups import MutuallyExclusiveCallbackGroup, ReentrantCallbackGroup
    5.  
    6. from cv_bridge import CvBridge
    7. from sensor_msgs.msg import CompressedImage, Image
    8. from std_msgs.msg import String
    9. from autoware_auto_perception_msgs.msg import DetectedObjects
    10. from autoware_auto_perception_msgs.msg import DetectedObject
    11. from autoware_auto_perception_msgs.msg import ObjectClassification
    12. from autoware_auto_perception_msgs.msg import DetectedObjectKinematics
    13. from autoware_auto_perception_msgs.msg import Shape
    14.  
    15. import numpy as np
    16. import mmcv
    17. import sys, os
    18. import torch
    19. import warnings
    20. from mmcv import Config, DictAction
    21. from mmcv.cnn import fuse_conv_bn
    22. from mmcv.parallel import MMDataParallel, MMDistributedDataParallel
    23. from mmcv.runner import (get_dist_info, init_dist, load_checkpoint,
    24.                          wrap_fp16_model)
    25.  
    26. from mmdet3d.models import build_model
    27. from mmdet.apis import multi_gpu_test, set_random_seed
    28. from mmdet.datasets import replace_ImageToTensor
    29. from mmdet3d.core.bbox.structures.box_3d_mode import (Box3DMode, CameraInstance3DBoxes,
    30.                               DepthInstance3DBoxes, LiDARInstance3DBoxes)
    31.  
    32. from nuscenes import NuScenes
    33. from pyquaternion import Quaternion
    34. from geometry_msgs.msg import Point
    35. from geometry_msgs.msg import Vector3
    36. from trimesh.transformations import quaternion_from_euler
    37. from geometry_msgs.msg import Quaternion as GeoQuaternion
    38. from geometry_msgs.msg import Twist
    39. import math
    40. import time
    41.  
    42. class DetectionPublisher(Node):
    43.  
    44.     def __init__(self):
    45.         super().__init__('DetectionPublisher_python')
    46.         
    47.         self.publisher_ = self.create_publisher(String, 'example_topic', 10)
    48.         timer_period = 0.5  # seconds
    49.         self.timer = self.create_timer(timer_period, self.timer_callback)
    50.         self.i = 0
    51.         
    52.  
    53.     def timer_callback(self):
    54.         msg = String()
    55.         msg.data = 'Hello World: %d' % self.i
    56.         self.publisher_.publish(msg)
    57.         self.get_logger().info('Publishing: "%s"' % msg.data)
    58.         self.i += 1
    59.  
    60. class PadMultiViewImage(object):
    61.     """Pad the multi-view image.
    62.     There are two padding modes: (1) pad to a fixed size and (2) pad to the
    63.     minimum size that is divisible by some number.
    64.     Added keys are "pad_shape", "pad_fixed_size", "pad_size_divisor",
    65.     Args:
    66.         size (tuple, optional): Fixed padding size.
    67.         size_divisor (int, optional): The divisor of padded size.
    68.         pad_val (float, optional): Padding value, 0 by default.
    69.     """
    70.  
    71.     def __init__(self, size=None, size_divisor=32, pad_val=0):
    72.         self.size = size
    73.         self.size_divisor = size_divisor
    74.         self.pad_val = pad_val
    75.         # only one of size and size_divisor should be valid
    76.         assert size is not None or size_divisor is not None
    77.         assert size is None or size_divisor is None
    78.  
    79.     def _pad_img(self, results):
    80.         """Pad images according to ``self.size``."""
    81.         if self.size is not None:
    82.             padded_img = [mmcv.impad(
    83.                 img, shape=self.size, pad_val=self.pad_val) for img in results['img']]
    84.         elif self.size_divisor is not None:
    85.             padded_img = [mmcv.impad_to_multiple(
    86.                 img, self.size_divisor, pad_val=self.pad_val) for img in results['img']]
    87.         results['img'] = padded_img
    88.         results['img_shape'] = [img.shape for img in padded_img]
    89.         results['pad_shape'] = [img.shape for img in padded_img]
    90.         results['pad_fixed_size'] = self.size
    91.         results['pad_size_divisor'] = self.size_divisor
    92.  
    93.     def __call__(self, results):
    94.         """Call function to pad images, masks, semantic segmentation maps.
    95.         Args:
    96.             results (dict): Result dict from loading pipeline.
    97.         Returns:
    98.             dict: Updated result dict.
    99.         """
    100.         self._pad_img(results)
    101.         return results
    102.  
    103.     def __repr__(self):
    104.         repr_str = self.__class__.__name__
    105.         repr_str += f'(size={self.size}, '
    106.         repr_str += f'size_divisor={self.size_divisor}, '
    107.         repr_str += f'pad_val={self.pad_val})'
    108.         return repr_str
    109.  
    110. class NormalizeMultiviewImage(object):
    111.     """Normalize the image.
    112.     Added key is "img_norm_cfg".
    113.     Args:
    114.         mean (sequence): Mean values of 3 channels.
    115.         std (sequence): Std values of 3 channels.
    116.         to_rgb (bool): Whether to convert the image from BGR to RGB,
    117.             default is true.
    118.     """
    119.  
    120.     def __init__(self, mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], to_rgb=True):
    121.         self.mean = np.array(mean, dtype=np.float32)
    122.         self.std = np.array(std, dtype=np.float32)
    123.         self.to_rgb = to_rgb
    124.  
    125.     def __call__(self, results):
    126.         """Call function to normalize images.
    127.         Args:
    128.             results (dict): Result dict from loading pipeline.
    129.         Returns:
    130.             dict: Normalized results, 'img_norm_cfg' key is added into
    131.                 result dict.
    132.         """
    133.         results['img'] = [mmcv.imnormalize(
    134.             img, self.mean, self.std, self.to_rgb) for img in results['img']]
    135.         results['img_norm_cfg'] = dict(
    136.             mean=self.mean, std=self.std, to_rgb=self.to_rgb)
    137.         return results
    138.  
    139.     def __repr__(self):
    140.         repr_str = self.__class__.__name__
    141.         repr_str += f'(mean={self.mean}, std={self.std}, to_rgb={self.to_rgb})'
    142.         return repr_str
    143.  
    144. class LoadSingleViewImageFromFiles(object):
    145.     """Load multi channel images from a list of separate channel files.
    147.     Expects results['img_filename'] to be a list of filenames.
    149.     Args:
    150.         to_float32 (bool): Whether to convert the img to float32.
    151.             Defaults to False.
    152.         color_type (str): Color type of the file. Defaults to 'unchanged'.
    153.     """
    154.  
    155.     def __init__(self, to_float32=True, color_type='unchanged'):
    156.         self.to_float32 = to_float32
    157.         self.color_type = color_type
    158.  
    159.     def __call__(self, results):
    160.         """Call function to load multi-view image from files.
    162.         Args:
    163.             results (dict): Result dict containing multi-view image filenames.
    165.         Returns:
    166.             dict: The result dict containing the multi-view image data. \
    167.                 Added keys and values are described below.
    169.                 - filename (str): Multi-view image filenames.
    170.                 - img (np.ndarray): Multi-view image arrays.
    171.                 - img_shape (tuple[int]): Shape of multi-view image arrays.
    172.                 - ori_shape (tuple[int]): Shape of original image arrays.
    173.                 - pad_shape (tuple[int]): Shape of padded image arrays.
    174.                 - scale_factor (float): Scale factor.
    175.                 - img_norm_cfg (dict): Normalization configuration of images.
    176.         """
    177.         results['filename'] = 'sample.jpg'
    178.         # h,w,c   => h, w, c, nv 
    179.         img = np.stack([results['img']], axis=-1)
    180.         if self.to_float32:
    181.             img = img.astype(np.float32)
    182.         results['img'] = [img[..., i] for i in range(img.shape[-1])]
    183.         results['img_shape'] = img.shape
    184.         results['ori_shape'] = img.shape
    185.         # Set initial values for default meta_keys
    186.         results['pad_shape'] = img.shape
    187.         results['scale_factor'] = 1.0
    188.         num_channels = 1 if len(img.shape) < 3 else img.shape[2]
    189.         results['img_norm_cfg'] = dict(
    190.             mean=np.zeros(num_channels, dtype=np.float32),
    191.             std=np.ones(num_channels, dtype=np.float32),
    192.             to_rgb=False)
    193.         return results
    194.  
    195.     def __repr__(self):
    196.         """str: Return a string that describes the module."""
    197.         repr_str = self.__class__.__name__
    198.         repr_str += f'(to_float32={self.to_float32}, '
    199.         repr_str += f"color_type='{self.color_type}')"
    200.         return repr_str
    201.  
    202. def obtain_sensor2top(nusc,
    203.                       sensor_token,
    204.                       l2e_t,
    205.                       l2e_r_mat,
    206.                       e2g_t,
    207.                       e2g_r_mat,
    208.                       sensor_type='lidar'):
    209.     """Obtain the info with RT matric from general sensor to Top LiDAR.
    211.     Args:
    212.         nusc (class): Dataset class in the nuScenes dataset.
    213.         sensor_token (str): Sample data token corresponding to the
    214.             specific sensor type.
    215.         l2e_t (np.ndarray): Translation from lidar to ego in shape (1, 3).
    216.         l2e_r_mat (np.ndarray): Rotation matrix from lidar to ego
    217.             in shape (3, 3).
    218.         e2g_t (np.ndarray): Translation from ego to global in shape (1, 3).
    219.         e2g_r_mat (np.ndarray): Rotation matrix from ego to global
    220.             in shape (3, 3).
    221.         sensor_type (str, optional): Sensor to calibrate. Default: 'lidar'.
    223.     Returns:
    224.         sweep (dict): Sweep information after transformation.
    225.     """
    226.     sd_rec = nusc.get('sample_data', sensor_token)
    227.     cs_record = nusc.get('calibrated_sensor',
    228.                          sd_rec['calibrated_sensor_token'])
    229.     pose_record = nusc.get('ego_pose', sd_rec['ego_pose_token'])
    230.     data_path = str(nusc.get_sample_data_path(sd_rec['token']))
    231.     if os.getcwd() in data_path:  # path from lyftdataset is absolute path
    232.         data_path = data_path.split(f'{os.getcwd()}/')[-1]  # relative path
    233.     sweep = {
    234.         'data_path': data_path,
    235.         'type': sensor_type,
    236.         'sample_data_token': sd_rec['token'],
    237.         'sensor2ego_translation': cs_record['translation'],
    238.         'sensor2ego_rotation': cs_record['rotation'],
    239.         'ego2global_translation': pose_record['translation'],
    240.         'ego2global_rotation': pose_record['rotation'],
    241.         'timestamp': sd_rec['timestamp']
    242.     }
    243.     l2e_r_s = sweep['sensor2ego_rotation']
    244.     l2e_t_s = sweep['sensor2ego_translation']
    245.     e2g_r_s = sweep['ego2global_rotation']
    246.     e2g_t_s = sweep['ego2global_translation']
    247.  
    248.     # obtain the RT from sensor to Top LiDAR
    249.     # sweep->ego->global->ego'->lidar
    250.     l2e_r_s_mat = Quaternion(l2e_r_s).rotation_matrix
    251.     e2g_r_s_mat = Quaternion(e2g_r_s).rotation_matrix
    252.     R = (l2e_r_s_mat.T @ e2g_r_s_mat.T) @ (
    253.         np.linalg.inv(e2g_r_mat).T @ np.linalg.inv(l2e_r_mat).T)
    254.     T = (l2e_t_s @ e2g_r_s_mat.T + e2g_t_s) @ (
    255.         np.linalg.inv(e2g_r_mat).T @ np.linalg.inv(l2e_r_mat).T)
    256.     T -= e2g_t @ (np.linalg.inv(e2g_r_mat).T @ np.linalg.inv(l2e_r_mat).T
    257.                   ) + l2e_t @ np.linalg.inv(l2e_r_mat).T
    258.     sweep['sensor2lidar_rotation'] = R.T  # points @ R.T + T
    259.     sweep['sensor2lidar_translation'] = T
    260.     return sweep
    261.  
    262. def getSemanticType(class_name):
    263.     if (class_name == "CAR" or class_name == "Car"):
    264.        return ObjectClassification.CAR
    265.     elif (class_name == "TRUCK" or class_name == "Medium_Truck" or class_name =="Big_Truck"):
    266.        return ObjectClassification.TRUCK
    267.     elif (class_name == "BUS"):
    268.        return ObjectClassification.BUS
    269.     elif (class_name == "TRAILER"):
    270.        return ObjectClassification.TRAILER
    271.     elif (class_name == "BICYCLE"):
    272.        return ObjectClassification.BICYCLE
    273.     elif (class_name == "MOTORBIKE"):
    274.        return ObjectClassification.MOTORCYCLE
    275.     elif (class_name == "PEDESTRIAN" or class_name == "Pedestrian"):
    276.        return ObjectClassification.PEDESTRIAN
    277.     else: 
    278.        return ObjectClassification.UNKNOWN
    279.  
    280.  
    281. class CustomBox3D(object):
    282.   def __init__(self,nid,score,x,y,z,w,l,h,rt,vel_x,vel_y):
    283.       self.id = nid
    284.       self.score = score
    285.       self.x = x
    286.       self.y = y
    287.       self.z = z
    288.       self.w = w
    289.       self.l = l
    290.       self.h = h
    291.       self.rt = rt
    292.       self.vel_x = vel_x
    293.       self.vel_y = vel_y
    294.  
    295. def isCarLikeVehicle(label):
    296.    return label == ObjectClassification.BICYCLE or label == ObjectClassification.BUS or \
    297.          label == ObjectClassification.CAR or label == ObjectClassification.MOTORCYCLE or \
    298.          label == ObjectClassification.TRAILER or label == ObjectClassification.TRUCK 
    299.  
    300. def createPoint(x, y, z):
    301.   p = Point()
    302.   p.x = float(x)
    303.   p.y = float(y)
    304.   p.z = float(z)
    305.   return p
    306.  
    307. def createQuaternionFromYaw(yaw):
    308.   # tf2.Quaternion
    309.   # q.setRPY(0, 0, yaw)
    310.   q = quaternion_from_euler(0, 0, yaw)
    311.   # geometry_msgs.msg.Quaternion
    312.   #return tf2.toMsg(q)
    313.   #return GeoQuaternion(*q)
    314.   return GeoQuaternion(x=q[0],y=q[1],z=q[2],w=q[3])
    315.  
    316. def createTranslation(x, y, z):
    317.   v = Vector3()
    318.   v.x = float(x)
    319.   v.y = float(y)
    320.   v.z = float(z)
    321.   return v
    322.  
    323. def box3DToDetectedObject(box3d, class_names, has_twist, is_sign):
    324.   obj = DetectedObject()
    325.   obj.existence_probability = float(box3d.score)
    326.  
    327.   classification = ObjectClassification()
    328.   classification.probability = 1.0
    329.   if (box3d.id >= 0 and box3d.id < len(class_names)):
    330.     classification.label = getSemanticType(class_names[box3d.id])
    331.   else: 
    332.     if is_sign:
    333.       sign_label = 255
    334.       classification.label = sign_label
    335.     else:
    336.       classification.label = ObjectClassification.UNKNOWN
    337.       print("Unexpected label: UNKNOWN is set.")
    338.   
    339.   if (isCarLikeVehicle(classification.label)):
    340.     obj.kinematics.orientation_availability = DetectedObjectKinematics.SIGN_UNKNOWN
    341.  
    342.   obj.classification.append(classification)
    343.  
    344.   # pose and shape
    345.   # mmdet3d yaw format to ros yaw format
    346.   yaw = -box3d.rt - np.pi / 2
    347.   obj.kinematics.pose_with_covariance.pose.position = createPoint(box3d.x, box3d.y, box3d.z)
    348.   obj.kinematics.pose_with_covariance.pose.orientation = createQuaternionFromYaw(yaw)
    349.   obj.shape.type = Shape.BOUNDING_BOX
    350.   obj.shape.dimensions = createTranslation(box3d.l, box3d.w, box3d.h)
    351.   # twist
    352.   if (has_twist):
    353.     vel_x = float(box3d.vel_x)
    354.     vel_y = float(box3d.vel_y)
    355.     twist = Twist()
    356.     twist.linear.x = math.sqrt(pow(vel_x, 2) + pow(vel_y, 2))
    357.     twist.angular.z = 2 * (math.atan2(vel_y, vel_x) - yaw)
    358.     obj.kinematics.twist_with_covariance.twist = twist
    359.     obj.kinematics.has_twist = has_twist
    360.   return obj  
    361.  
    362.  
    363. class ImageSubscriber(Node):
    364.  
    365.     def __init__(self):
    366.         super().__init__('ImageSubscriber_python')
    367.         cb_group = MutuallyExclusiveCallbackGroup()
    368.         self.img_sub = self.create_subscription(
    369.             CompressedImage,
    370.             'pub_image/compressed',
    371.             self.image_callback,
    372.             10,
    373.             callback_group=cb_group)
    374.         self.img_sub
    375.         self.od_pub = self.create_publisher(DetectedObjects, 'pub_detection', 10)
    376.         self.cvBridge  = CvBridge()
    377.         self.pad = PadMultiViewImage()
    378.         self.norm = NormalizeMultiviewImage()
    379.         self.file_loader = LoadSingleViewImageFromFiles() 
    380.         config_path = "./detr3d_res101_gridmask_wst.py"
    381.         self.cfg = Config.fromfile(config_path)
    382.         if self.cfg.get('custom_imports', None):
    383.            from mmcv.utils import import_modules_from_strings
    384.            import_modules_from_strings(**self.cfg['custom_imports'])
    385.         if hasattr(self.cfg, 'plugin'):
    386.            if self.cfg.plugin:
    387.              import importlib
    388.              if hasattr(self.cfg, 'plugin_dir'):
    389.                 plugin_dir = self.cfg.plugin_dir
    390.                 _module_dir = os.path.dirname(plugin_dir)
    391.                 _module_dir = _module_dir.split('/')
    392.                 _module_path = _module_dir[0]
    393.  
    394.                 for m in _module_dir[1:]:
    395.                     _module_path = _module_path + '.' + m
    396.                 print(_module_path)
    397.                 print(sys.path)
    398.                 plg_lib = importlib.import_module(_module_path)
    399.         if self.cfg.get('cudnn_benchmark', False):
    400.            torch.backends.cudnn.benchmark = True
    401.         self.cfg.model.pretrained = None
    402.         self.cfg.model.train_cfg = None
    403.         self.model = build_model(self.cfg.model, test_cfg=self.cfg.get('test_cfg'))
    404.         fp16_cfg = self.cfg.get('fp16', None)
    405.         if fp16_cfg is not None:
    406.            wrap_fp16_model(self.model)
    407.         checkpoint = load_checkpoint(self.model, "epoch_200.pth", map_location='cpu')
    408.         
    409.         if 'CLASSES' in checkpoint.get('meta', {}):
    410.            self.model.CLASSES = checkpoint['meta']['CLASSES']
    411.         else:
    412.            self.model.CLASSS = ('car', 'truck', 'trailer', 'bus', 'construction_vehicle',
    413.                'bicycle', 'motorcycle', 'pedestrian', 'traffic_cone',
    414.                'barrier')
    415.         # palette for visualization in segmentation tasks
    416.         if 'PALETTE' in checkpoint.get('meta', {}):
    417.            self.model.PALETTE = checkpoint['meta']['PALETTE']
    418.         self.model.cfg  = self.cfg 
    419.  
    420.         self.model.cuda()
    421.         self.model.eval()
    422.         #if torch.cuda.device_count() > 1:  # for server side
    423.         #   self.model = nn.DataParallel(self.model)
    424.         print("model is created!")
    425.  
    426.         nusc = NuScenes(version='v1.0-mini', dataroot='nuscenes_mini', verbose=True)
    427.         scene0 = nusc.scene[0]
    428.         first_sample_token = scene0['first_sample_token']
    429.         first_sample = nusc.get('sample', first_sample_token)
    430.         sd_rec = nusc.get('sample_data', first_sample['data']['LIDAR_TOP'])
    431.         cs_record = nusc.get('calibrated_sensor',
    432.                              sd_rec['calibrated_sensor_token'])
    433.         pose_record = nusc.get('ego_pose', sd_rec['ego_pose_token'])
    434.         lidar_token = first_sample['data']['LIDAR_TOP']
    435.         lidar_path, boxes, _ = nusc.get_sample_data(lidar_token)
    436.         info = {
    437.             'lidar_path': lidar_path,
    438.             'token': first_sample['token'],
    439.             'sweeps': [],
    440.             'cams': dict(),
    441.             'lidar2ego_translation': cs_record['translation'],
    442.             'lidar2ego_rotation': cs_record['rotation'],
    443.             'ego2global_translation': pose_record['translation'],
    444.             'ego2global_rotation': pose_record['rotation'],
    445.             'timestamp': first_sample['timestamp'],
    446.         }
    447.  
    448.         l2e_r = info['lidar2ego_rotation']
    449.         l2e_t = info['lidar2ego_translation']
    450.         e2g_r = info['ego2global_rotation']
    451.         e2g_t = info['ego2global_translation']
    452.         l2e_r_mat = Quaternion(l2e_r).rotation_matrix
    453.         e2g_r_mat = Quaternion(e2g_r).rotation_matrix
    454.         camera_types = [
    455.             'CAM_FRONT',
    456.             'CAM_FRONT_RIGHT',
    457.             'CAM_FRONT_LEFT',
    458.             'CAM_BACK',
    459.             'CAM_BACK_LEFT',
    460.             'CAM_BACK_RIGHT',
    461.         ]
    462.         for cam in camera_types:
    463.             cam_token = first_sample['data'][cam]
    464.             cam_path, _, cam_intrinsic = nusc.get_sample_data(cam_token)
    465.             cam_info = obtain_sensor2top(nusc, cam_token, l2e_t, l2e_r_mat,
    466.                                          e2g_t, e2g_r_mat, cam)
    467.             cam_info.update(cam_intrinsic=cam_intrinsic)
    468.             info['cams'].update({cam: cam_info})
    469.  
    470.         '''
    471.         cam_front_sample_data = nusc.get('sample_data', first_sample['data']['CAM_FRONT'])
    472.         cam_front_sample_path = os.path.join(nusc.dataroot, cam_front_sample_data['filename'])
    473.         print("sample image file:", cam_front_sample_path)
    474.         cam_front_calibrate = nusc.get('calibrated_sensor', cam_front_sample_data['calibrated_sensor_token'])
    475.         sensor2lidar = translation = np.expand_dims(np.array(cam_front_calibrate['translation']), axis=-1)
    476.         sensor2lidar_rotation = np.expand_dims(np.array(cam_front_calibrate['rotation']), axis=-1)
    477.         camera_intrinsic = np.array(cam_front_calibrate['camera_intrinsic'])
    478.         '''
    479.  
    480.         image_paths = []
    481.         lidar2img_rts = []
    482.         lidar2cam_rts = []
    483.         cam_intrinsics = []
    484.         for cam_type, cam_info in info['cams'].items():
    485.             image_paths.append(cam_info['data_path'])
    486.              # obtain lidar to image transformation matrix
    487.             lidar2cam_r = np.linalg.inv(cam_info['sensor2lidar_rotation'])
    488.             lidar2cam_t = cam_info[
    489.                 'sensor2lidar_translation'] @ lidar2cam_r.T
    490.             lidar2cam_rt = np.eye(4)
    491.             lidar2cam_rt[:3, :3] = lidar2cam_r.T
    492.             lidar2cam_rt[3, :3] = -lidar2cam_t
    493.             intrinsic = cam_info['cam_intrinsic']
    494.             viewpad = np.eye(4)
    495.             viewpad[:intrinsic.shape[0], :intrinsic.shape[1]] = intrinsic
    496.             lidar2img_rt = (viewpad @ lidar2cam_rt.T)
    497.             lidar2img_rts.append(lidar2img_rt)
    498.  
    499.             cam_intrinsics.append(viewpad)
    500.             lidar2cam_rts.append(lidar2cam_rt.T)
    501.  
    502.         self.img_metas = {}
    503.         self.img_metas.update(
    504.                 dict(
    505.                     img_filename=image_paths,
    506.                     lidar2img=lidar2img_rts,
    507.                     cam_intrinsic=cam_intrinsics,
    508.                     lidar2cam=lidar2cam_rts,
    509.                 ))
    510.         self.class_names_ = self.cfg.class_names
    511.         print("ImageSubscriber init done")
    512.  
    513.     
    514.     def image_callback(self, msg):
    515.         #image = self.cvBridge.imgmsg_to_cv2(msg, "bgr8")
    516.         image = self.cvBridge.compressed_imgmsg_to_cv2(msg, "bgr8")
    517.         #print("image received, shape:", image.shape)
    518.         results = {'img': image}
    519.         self.file_loader(results)
    520.         self.norm(results)
    521.         self.pad(results)
    522.         image = results['img'][0]
    523.         meta = {'filename': results['filename'], 
    524.                 'img_shape': results['img_shape'],
    525.                 'ori_shape': results['ori_shape'],
    526.                 'pad_shape': results['pad_shape'],
    527.                 'scale_factor': results['scale_factor'],
    528.                 'box_type_3d': LiDARInstance3DBoxes,     #CameraInstance3DBoxes/LiDARInstance3DBoxes
    529.                 'box_mode_3d': Box3DMode.LIDAR}
    530.         meta.update(self.img_metas)
    531.         #print("meta:", meta)
    532.         img_metas = [[meta]]
    533.         inputs = torch.tensor(image).to('cuda')
    534.         # h,w,c => bs,nv,c,h,w
    535.         inputs = inputs.permute(2,0,1)
    536.         inputs = torch.unsqueeze(inputs, 0)
    537.         inputs = torch.unsqueeze(inputs, 0)
    538.         #print("input tensor shape:", inputs.shape)
    539.         with torch.no_grad():
    540.            outputs = self.model(return_loss=False, rescale=True, img=inputs, img_metas=img_metas) 
    541.            torch.cuda.synchronize()
    542.            pts_bbox = outputs[0]['pts_bbox']
    543.            boxes_3d_enc = pts_bbox['boxes_3d']
    544.            scores_3d = pts_bbox['scores_3d']
    545.            labels_3d = pts_bbox['labels_3d']
    546.  
    547.            filter = scores_3d >= 0.5
    548.            boxes_3d_enc.tensor = boxes_3d_enc.tensor[filter]
    549.            boxes_3d = boxes_3d_enc.tensor.numpy() # [[cx, cy, cz, w, l, h, rot, vx, vy]]
    550.            scores_3d = scores_3d[filter].numpy()
    551.            labels_3d = labels_3d[filter].numpy()
    552.            custom_boxes_3d = []
    553.            for i, box_3d in enumerate(boxes_3d):
    554.               box3d = CustomBox3D(labels_3d[i], scores_3d[i],
    555.                                    box_3d[0],box_3d[1],box_3d[2],
    556.                                    box_3d[3],box_3d[4],box_3d[5],
    557.                                    box_3d[6],box_3d[7],box_3d[8])
    558.               custom_boxes_3d.append(box3d)
    559.                
    560.            #print("boxes_3d", boxes_3d)
    561.            #print("scores_3d", scores_3d)
    562.            #print("labels_3d", labels_3d)
    563.            output_msg = DetectedObjects()
    564.            obj_num = len(boxes_3d)
    565.            for i, box3d in enumerate(custom_boxes_3d):
    566.               obj = box3DToDetectedObject(box3d, self.class_names_, True, False);
    567.               output_msg.objects.append(obj)
    568.  
    569.            output_msg.header.stamp = self.get_clock().now().to_msg() #rclpy.time.Time()
    570.            output_msg.header.frame_id = "base_link"
    571.            self.od_pub.publish(output_msg)
    572.            print(obj_num, "Objects published")
    573.  
    574.  
    575. def main(args=None):
    576.     rclpy.init(args=args)
    577.  
    578.     sys.path.append(os.getcwd())
    579.     image_subscriber = ImageSubscriber()
    580.     executor = MultiThreadedExecutor()
    581.     executor.add_node(image_subscriber)
    582.     executor.spin()
    583.  
    584.     image_subscriber.destroy_node()
    585.     detection_publisher.destroy_node()
    586.     rclpy.shutdown()
    587.  
    588.  
    589. if __name__ == '__main__':
    590.     main()

    这里说的单帧数据可以是单张图片或者nuScenes格式的6张环视图片，之所以说一帧，是对应点云的概念，实际做3D目标检测尤其是雷视融合推理时一般都是以点云帧的频率为时间单位进行推理(在获取一帧点云的同时获取一张或多张图片(取决于有几个摄像头)，前融合的话是将点云和图像的数据做数据融合后输入模型或直接输入模型由模型内部抽取特征做特征融合，后融合的话是将点云和图像分别输入不同的模型推理，再对模型各自的结果进行融合处理)。

    上面的代码由于是以DETR3D官方代码为基础的，所以和mmdetection3D后来集成DETR3D的实现代码不同，对比两者代码可以发现，mmdetection3D将DETR3D官方代码里的一些数据预处理环节的实现代码都移走了，在模型内部单独提供了一个Det3DDataPreprocessor类来进行一些公共处理，总体的配置文件和实现思路还是比较类似，毕竟DETR3D官方是在mmdetection3D的早期版本基础上开发的，例如两者的detr3d_r101_gridmask.py配置文件里面有些配置不同，但是思路是相似的：

    mmdetection3d/projects/DETR3D/configs/detr3d_r101_gridmask.py

    1. model = dict(
    2.     type='DETR3D',
    3.     use_grid_mask=True,
    4.     data_preprocessor=dict(
    5.         type='Det3DDataPreprocessor', **img_norm_cfg, pad_size_divisor=32),
    6.     img_backbone=dict(
    7.         type='mmdet.ResNet',
    8.         depth=101,
    9.         num_stages=4,
    10.         out_indices=(0, 1, 2, 3),
    11.         frozen_stages=1,
    12.         norm_cfg=dict(type='BN2d', requires_grad=False),
    13.         norm_eval=True,
    14.         style='caffe',
    15.         dcn=dict(type='DCNv2', deform_groups=1, fallback_on_stride=False),
    16.         stage_with_dcn=(False, False, True, True)),
    17.     img_neck=dict(
    18.         type='mmdet.FPN',
    19.         in_channels=[256, 512, 1024, 2048],
    20.         out_channels=256,
    21.         start_level=1,
    22.         add_extra_convs='on_output',
    23.         num_outs=4,
    24.         relu_before_extra_convs=True),
    25.     pts_bbox_head=dict(
    26.         type='DETR3DHead',
    27.         num_query=900,
    28.         num_classes=10,
    29.         in_channels=256,
    30.         sync_cls_avg_factor=True,
    31.         with_box_refine=True,
    32.         as_two_stage=False,
    33.         transformer=dict(
    34.             type='Detr3DTransformer',
    35.             decoder=dict(
    36.                 type='Detr3DTransformerDecoder',
    37.                 num_layers=6,
    38.                 return_intermediate=True,
    39.                 transformerlayers=dict(
    40.                     type='mmdet.DetrTransformerDecoderLayer',
    41.                     attn_cfgs=[
    42.                         dict(
    43.                             type='MultiheadAttention',  # mmcv.
    44.                             embed_dims=256,
    45.                             num_heads=8,
    46.                             dropout=0.1),
    47.                         dict(
    48.                             type='Detr3DCrossAtten',
    49.                             pc_range=point_cloud_range,
    50.                             num_points=1,
    51.                             embed_dims=256)
    52.                     ],
    53.                     feedforward_channels=512,
    54.                     ffn_dropout=0.1,
    55.                     operation_order=('self_attn', 'norm', 'cross_attn', 'norm',
    56.                                      'ffn', 'norm')))),
    57.         bbox_coder=dict(
    58.             type='NMSFreeCoder',
    59.             post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0],
    60.             pc_range=point_cloud_range,
    61.             max_num=300,
    62.             voxel_size=voxel_size,
    63.             num_classes=10),
    64.         positional_encoding=dict(
    65.             type='mmdet.SinePositionalEncoding',
    66.             num_feats=128,
    67.             normalize=True,
    68.             offset=-0.5),
    69.         loss_cls=dict(
    70.             type='mmdet.FocalLoss',
    71.             use_sigmoid=True,
    72.             gamma=2.0,
    73.             alpha=0.25,
    74.             loss_weight=2.0),
    75.         loss_bbox=dict(type='mmdet.L1Loss', loss_weight=0.25),
    76.         loss_iou=dict(type='mmdet.GIoULoss', loss_weight=0.0)),
    77.     # model training and testing settings
    78.     train_cfg=dict(
    79.         pts=dict(
    80.             grid_size=[512, 512, 1],
    81.             voxel_size=voxel_size,
    82.             point_cloud_range=point_cloud_range,
    83.             out_size_factor=4,
    84.             assigner=dict(
    85.                 type='HungarianAssigner3D',
    86.                 cls_cost=dict(type='mmdet.FocalLossCost', weight=2.0),
    87.                 reg_cost=dict(type='BBox3DL1Cost', weight=0.25),
    88.                 # ↓ Fake cost. This is just to get compatible with DETR head
    89.                 iou_cost=dict(type='mmdet.IoUCost', weight=0.0),
    90.                 pc_range=point_cloud_range))))
    91.  
    92. dataset_type = 'NuScenesDataset'
    93. data_root = 'data/nuscenes/'
    94.  
    95. test_transforms = [
    96.     dict(
    97.         type='RandomResize3D',
    98.         scale=(1600, 900),
    99.         ratio_range=(1., 1.),
    100.         keep_ratio=True)
    101. ]
    102. train_transforms = [dict(type='PhotoMetricDistortion3D')] + test_transforms
    103.  
    104. backend_args = None
    105. train_pipeline = [
    106.     dict(
    107.         type='LoadMultiViewImageFromFiles',
    108.         to_float32=True,
    109.         num_views=6,
    110.         backend_args=backend_args),
    111.     dict(
    112.         type='LoadAnnotations3D',
    113.         with_bbox_3d=True,
    114.         with_label_3d=True,
    115.         with_attr_label=False),
    116.     dict(type='MultiViewWrapper', transforms=train_transforms),
    117.     dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
    118.     dict(type='ObjectNameFilter', classes=class_names),
    119.     dict(type='Pack3DDetInputs', keys=['img', 'gt_bboxes_3d', 'gt_labels_3d'])
    120. ]
    121.  
    122. test_pipeline = [
    123.     dict(
    124.         type='LoadMultiViewImageFromFiles',
    125.         to_float32=True,
    126.         num_views=6,
    127.         backend_args=backend_args),
    128.     dict(type='MultiViewWrapper', transforms=test_transforms),
    129.     dict(type='Pack3DDetInputs', keys=['img'])
    130. ]

    detr3d/projects/configs/detr3d/detr3d_res101_gridmask.py

    1. model = dict(
    2.     type='Detr3D',
    3.     use_grid_mask=True,
    4.     img_backbone=dict(
    5.         type='ResNet',
    6.         depth=101,
    7.         num_stages=4,
    8.         out_indices=(0, 1, 2, 3),
    9.         frozen_stages=1,
    10.         norm_cfg=dict(type='BN2d', requires_grad=False),
    11.         norm_eval=True,
    12.         style='caffe',
    13.         dcn=dict(type='DCNv2', deform_groups=1, fallback_on_stride=False),
    14.         stage_with_dcn=(False, False, True, True)),
    15.     img_neck=dict(
    16.         type='FPN',
    17.         in_channels=[256, 512, 1024, 2048],
    18.         out_channels=256,
    19.         start_level=1,
    20.         add_extra_convs='on_output',
    21.         num_outs=4,
    22.         relu_before_extra_convs=True),
    23.     pts_bbox_head=dict(
    24.         type='Detr3DHead',
    25.         num_query=900,
    26.         num_classes=10,
    27.         in_channels=256,
    28.         sync_cls_avg_factor=True,
    29.         with_box_refine=True,
    30.         as_two_stage=False,
    31.         transformer=dict(
    32.             type='Detr3DTransformer',
    33.             decoder=dict(
    34.                 type='Detr3DTransformerDecoder',
    35.                 num_layers=6,
    36.                 return_intermediate=True,
    37.                 transformerlayers=dict(
    38.                     type='DetrTransformerDecoderLayer',
    39.                     attn_cfgs=[
    40.                         dict(
    41.                             type='MultiheadAttention',
    42.                             embed_dims=256,
    43.                             num_heads=8,
    44.                             dropout=0.1),
    45.                         dict(
    46.                             type='Detr3DCrossAtten',
    47.                             pc_range=point_cloud_range,
    48.                             num_points=1,
    49.                             embed_dims=256)
    50.                     ],
    51.                     feedforward_channels=512,
    52.                     ffn_dropout=0.1,
    53.                     operation_order=('self_attn', 'norm', 'cross_attn', 'norm',
    54.                                      'ffn', 'norm')))),
    55.         bbox_coder=dict(
    56.             type='NMSFreeCoder',
    57.             post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0],
    58.             pc_range=point_cloud_range,
    59.             max_num=300,
    60.             voxel_size=voxel_size,
    61.             num_classes=10), 
    62.         positional_encoding=dict(
    63.             type='SinePositionalEncoding',
    64.             num_feats=128,
    65.             normalize=True,
    66.             offset=-0.5),
    67.         loss_cls=dict(
    68.             type='FocalLoss',
    69.             use_sigmoid=True,
    70.             gamma=2.0,
    71.             alpha=0.25,
    72.             loss_weight=2.0),
    73.         loss_bbox=dict(type='L1Loss', loss_weight=0.25),
    74.         loss_iou=dict(type='GIoULoss', loss_weight=0.0)),
    75.     # model training and testing settings
    76.     train_cfg=dict(pts=dict(
    77.         grid_size=[512, 512, 1],
    78.         voxel_size=voxel_size,
    79.         point_cloud_range=point_cloud_range,
    80.         out_size_factor=4,
    81.         assigner=dict(
    82.             type='HungarianAssigner3D',
    83.             cls_cost=dict(type='FocalLossCost', weight=2.0),
    84.             reg_cost=dict(type='BBox3DL1Cost', weight=0.25),
    85.             iou_cost=dict(type='IoUCost', weight=0.0), # Fake cost. This is just to make it compatible with DETR head. 
    86.             pc_range=point_cloud_range))))
    87.  
    88. dataset_type = 'NuScenesDataset'
    89. data_root = 'data/nuscenes/'
    90.  
    91. ...
    92.  
    93. train_pipeline = [
    94.     dict(type='LoadMultiViewImageFromFiles', to_float32=True),
    95.     dict(type='PhotoMetricDistortionMultiViewImage'),
    96.     dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_attr_label=False),
    97.     dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
    98.     dict(type='ObjectNameFilter', classes=class_names),
    99.     dict(type='NormalizeMultiviewImage', **img_norm_cfg),
    100.     dict(type='PadMultiViewImage', size_divisor=32),
    101.     dict(type='DefaultFormatBundle3D', class_names=class_names),
    102.     dict(type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img'])
    103. ]
    104. test_pipeline = [
    105.     dict(type='LoadMultiViewImageFromFiles', to_float32=True),
    106.     dict(type='NormalizeMultiviewImage', **img_norm_cfg),
    107.     dict(type='PadMultiViewImage', size_divisor=32),
    108.     dict(
    109.         type='MultiScaleFlipAug3D',
    110.         img_scale=(1333, 800),
    111.         pts_scale_ratio=1,
    112.         flip=False,
    113.         transforms=[
    114.             dict(
    115.                 type='DefaultFormatBundle3D',
    116.                 class_names=class_names,
    117.                 with_label=False),
    118.             dict(type='Collect3D', keys=['img'])
    119.         ])
    120. ]

    所以，如果是使用mmmdetection3D里集成的DETR3D推理的话，上面的程序需要做修改，可以先对比配置文件和相关代码实现的理清楚差异然后修改代码，例如PadMultiViewImage在mmdetection3D里是不存在的。

  • 相关阅读:
    C //例5.11 译密码。为使电文保密，往往按一定规律将其转换成密码，收报人再按约定的规律将其译回原文。
    《PyTorch深度学习实践》第十二课（循环神经网络RNN）
    丹尼尔·拉瑞莫(BM)：EOS的主要开发者
    文字的选择与排版
    分布式事务详解
    车路协同 智能路侧决策系统总体架构及应用
    实战讲解Spring配置中心Config同步更新配置信息（图+文）
    PROFINET通信介绍
    Mybatis框架的缓存
    软件测试测试常见分类有哪些？
  • 原文地址：https://blog.csdn.net/XCCCCZ/article/details/133501513