• deepstream 检测结果截图

    目录

    1. 如何访问GstBuffer数据

    2.2 Makefile文件的修改

    pgie_pad_buffer_probe获取图片，转opencv mat

     sgie获取帧数据，转opencv mat图：

    以下内容转自：

    deepstream实现视频推理结果截图_hello_dear_you的博客-CSDN博客

    这个是deepstream论坛中相关问题的回答，具体可以阅读下面上个相关问题的内容

    1. Access frame pointer in deepstream-app
    2. Frame data extraction and pipeline halts/hangs after 7 minutes of run​​​​​​
    3. Semantic Segmentation in DS 4
    同样，在CSDN中也找到相关博客的介绍

    1. DeepStream结合OpenCV4实现视频的分析和截图（一）

    2. DeepStream结合OpenCV4实现视频的分析和截图（二）

    3. Ubuntu18.04.3 安装 Nvidia DeepStream 并在 P4 显卡上解码取帧存图

    此外，还参考了OpenCV对于RGBA图像操作的相关内容

    1. OpenCV入门（4）：透明图像（RGBA）的处理

    2. Opencv 图像读取与保存问题

    1. 如何访问GstBuffer数据


    通过上述分析，我们知道可以在probe回调函数中来保存图像信息。此时我们应该根据自己的需求，选择相应element的pad来设置probe回调函。根据OSD之前和之后获取数据，可以分为两大类：

    OSD element之前保存数据：可以选择nvinfer的src pad（buffer type为NV12）或者nvvidconv的src pad（buffer type为RGBA）来访问数据
    OSD element之后保存数据：也即选择osd 的src pad（buffer type为RGBA）
    上述两种类型之前的区别，应该在于检测信息是否已经渲染到frame_meta数据中。

    下面以osd_sink_pad_buffer_probe回调函数举例，如何访问GstBuffer数据？

      /* osd_sink_pad_buffer_probe  will extract metadata received on OSD sink pad
     * and update params for drawing rectangle, object information etc. */
     
    static GstPadProbeReturn
    osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info,
        gpointer u_data)
    {
        GstBuffer *buf = (GstBuffer *) info->data;  
     
        char* src_data = NULL;
        if (!gst_buffer_map (buf, &in_map_info, GST_MAP_READ)) 
        {
              g_print ("Error: Failed to map gst buffer\n");
              gst_buffer_unmap (buf, &in_map_info);
              return GST_PAD_PROBE_OK;
        }
        NvBufSurface *surface = (NvBufSurface *)in_map_info.data;
     
        gst_buffer_unmap (buf, &in_map_info);
    }

    2.2 Makefile文件的修改


    1. 添加opencv的头文件

    CFLAGS+= -I/opt/nvidia/deepstream/deepstream-5.1/sources/includes \
             -I /usr/local/cuda-$(CUDA_VER)/include \
             -fPIC -std=c++17 -g    \
             -I/usr/local/opencv-4.5.1/include/opencv4
    2. 添加相对应的库文件

    -lnvbufsurface -lnvbufsurftransform
    -L/usr/local/opencv-4.5.1/lib -lopencv_core -lopencv_imgproc -lopencv_imgcodecs 
    LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper -lm  -lnvbufsurface -lnvbufsurftransform \
           -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart \
           -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR)    \
           -L/usr/local/opencv-4.5.1/lib -lopencv_core -lopencv_imgproc -lopencv_imgcodecs
    通过上述的过程，我们就能保存带检测结果的图像帧数据。
     

    原文链接：https://blog.csdn.net/hello_dear_you/article/details/121687664

    pgie_pad_buffer_probe获取图片，转opencv mat

    static GstPadProbeReturn pgie_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data)

    1. #include 
    2. #include 
    3. #include 
    4. #include 
    5. #include 
    6. #include 
    7. #include 
    8. #include 
    9. #include 
    10. #include "gst-nvmessage.h"
    11. #include "gstnvdsmeta.h"
    12. #include "nvdsmeta_schema.h"
    13. #include "nvdsinfer_custom_impl.h"
    14. #include "nvbufsurface.h"
    15. #include 
    16. #include "cuda_runtime_api.h"
    17. #include "gstnvdsinfer.h"
    18. #include 
    19. #include "nvds_parse_bbox_wpod.h"
    20. #include "gst-nvdssr.h"
    21.  
    22. #define OUTPUT_FILE "output/out.mp4"
    23.  
    24. #define INTERVAL 2
    25. #define OSD_PROCESS_MODE 0
    26. #define SGIE_INPUT_H 32
    27. #define SGIE_INPUT_W 100
    28. #define SGIE_OUTPUT_SIZE 26 * 37
    29. #define SGIE_PAD_OUT 8
    30.  
    31. /* By default, OSD will not display text. To display text, change this to 1 */
    32. #define OSD_DISPLAY_TEXT 1
    33. #define PGIE_NET_WIDTH 256
    34. #define PGIE_NET_HEIGHT 256
    35. #define MUXER_OUTPUT_WIDTH 1920
    36. #define MUXER_OUTPUT_HEIGHT 1080
    37. #define TILED_OUTPUT_WIDTH 1920
    38. #define TILED_OUTPUT_HEIGHT 1080
    39. #define MUXER_BATCH_TIMEOUT_USEC 25000
    40. #define MAX_DISPLAY_LEN 64
    41. #define MEMORY_FEATURES "memory:NVMM"
    42. #define PGIE_CONFIG_FILE "config_pgie.txt"
    43. #define SGIE_CONFIG_FILE "config_sgie.txt"
    44. #define START_TIME 0
    45. #define SHOW_CRNN 0
    46. const std::string alphabet = "-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
    47.  
    48. // ./deepstream-ALPR file:///opt/nvidia/deepstream/deepstream-5.0/sources/alpr_ds/wpod/deepstream_app_wpod/test1.mp4
    49.  
    50. /* Duration of recording
    51.  */
    52. bool display = FALSE;
    53.  
    54. extern "C" bool NvDsInferParseCustomWpod(
    55.     std::vector const &outputLayersInfo,
    56.     NvDsInferNetworkInfo const &networkInfo,
    57.     NvDsInferParseDetectionParams const &detectionParams,
    58.     std::vector &objectList);
    59.  
    60. std::string strDecode(std::vector<int> &preds, bool raw)
    61. {
    62.     std::string str;
    63.     if (raw)
    64.     {
    65.         for (auto v : preds)
    66.         {
    67.             str.push_back(alphabet[v]);
    68.         }
    69.     }
    70.     else
    71.     {
    72.         for (size_t i = 0; i < preds.size(); i++)
    73.         {
    74.             if (preds[i] == 0 || (i > 0 && preds[i - 1] == preds[i]))
    75.                 continue;
    76.             str.push_back(alphabet[preds[i]]);
    77.         }
    78.     }
    79.     return str;
    80. }
    81. static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer data)
    82. {
    83.     GMainLoop *loop = (GMainLoop *)data;
    84.     switch (GST_MESSAGE_TYPE(msg))
    85.     {
    86.     case GST_MESSAGE_EOS:
    87.     {
    88.         g_print("End of stream\n");
    89.         g_main_loop_quit(loop);
    90.         break;
    91.     }
    92.  
    93.     case GST_MESSAGE_WARNING:
    94.     {
    95.         gchar *debug;
    96.         GError *error;
    97.         gst_message_parse_warning(msg, &error, &debug);
    98.         g_printerr("WARNING from element %s: %s\n", GST_OBJECT_NAME(msg->src), error->message);
    99.         g_free(debug);
    100.         g_printerr("Warning: %s\n", error->message);
    101.         g_error_free(error);
    102.         break;
    103.     }
    104.     case GST_MESSAGE_ERROR:
    105.     {
    106.         gchar *debug;
    107.         GError *error;
    108.         gst_message_parse_error(msg, &error, &debug);
    109.         g_printerr("ERROR from element %s: %s\n",
    110.                    GST_OBJECT_NAME(msg->src), error->message);
    111.         if (debug)
    112.             g_printerr("Error details: %s\n", debug);
    113.         g_free(debug);
    114.         g_error_free(error);
    115.         g_main_loop_quit(loop);
    116.         break;
    117.     }
    118. #ifdef PLATFORM_TEGRA
    119.     case GST_MESSAGE_ELEMENT:
    120.     {
    121.         if (gst_nvmessage_is_stream_eos(msg))
    122.         {
    123.             guint stream_id;
    124.             if (gst_nvmessage_parse_stream_eos(msg, &stream_id))
    125.             {
    126.                 g_print("Got EOS from stream %d\n", stream_id);
    127.             }
    128.         }
    129.         break;
    130.     }
    131. #endif
    132.     default:
    133.         break;
    134.     }
    135.     return TRUE;
    136. }
    137.  
    138. static GstPadProbeReturn pgie_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data)
    139. {
    140.     NvBufSurface *surface = NULL;
    141.     GstBuffer *inbuf = GST_PAD_PROBE_INFO_BUFFER(info);
    142.     GstMapInfo in_map_info;
    143.     NvDsMetaList *l_obj = NULL;
    144.     NvDsObjectMeta *obj_meta = NULL;
    145.     memset(&in_map_info, 0, sizeof(in_map_info));
    146.     if (!gst_buffer_map(inbuf, &in_map_info, GST_MAP_READ))
    147.     {
    148.         g_error("Error: Failed to map gst buffer\n");
    149.     }
    150.     surface = (NvBufSurface *)in_map_info.data;
    151.     NvDsBatchMeta *batch_meta =
    152.         gst_buffer_get_nvds_batch_meta(inbuf);
    153.     static guint use_device_mem = 0;
    154.     static NvDsInferNetworkInfo networkInfo{3, PGIE_NET_WIDTH, PGIE_NET_HEIGHT};
    155.     NvDsInferParseDetectionParams detectionParams;
    156.  
    157. #ifndef PLATFORM_TEGRA
    158.     // if (surface->memType != NVBUF_MEM_CUDA_UNIFIED)
    159.     // {
    160.     //     g_error("need NVBUF_MEM_CUDA_UNIFIED memory for opencv\n");
    161.     // }
    162. #endif
    163.     /* Iterate each frame metadata in batch */
    164.     for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next)
    165.     {
    166.         NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)l_frame->data;
    167.         cv::Mat in_mat;
    168.         if (surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0] == NULL)
    169.         {
    170.             if (NvBufSurfaceMap(surface, frame_meta->batch_id, 0, NVBUF_MAP_READ_WRITE) != 0)
    171.             {
    172.                 g_error("buffer map to be accessed by CPU failed\n");
    173.             }
    174.         }
    175.         /* Cache the mapped data for CPU access */
    176.         NvBufSurfaceSyncForCpu(surface, frame_meta->batch_id, 0);
    177.         in_mat =
    178.             cv::Mat(surface->surfaceList[frame_meta->batch_id].planeParams.height[0],
    179.                     surface->surfaceList[frame_meta->batch_id].planeParams.width[0], CV_8UC4,
    180.                     surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0],
    181.                     surface->surfaceList[frame_meta->batch_id].planeParams.pitch[0]);
    182.         // cv::imwrite("output/in_im.jpg", in_mat);
    183.         // cv::Mat bgr_frame = cv::Mat(cv::Size(surface->surfaceList[frame_meta->batch_id].planeParams.height[0], surface->surfaceList[frame_meta->batch_id].planeParams.width[0]), CV_8UC3);
    184.  
    185.         // cv::cvtColor(in_mat, bgr_frame, cv::COLOR_RGBA2BGR);
    186.  
    187.         // cv::imwrite("output/a.jpg", bgr_frame);
    188.         /* Iterate user metadata in frames to search PGIE's tensor metadata */
    189.         for (NvDsMetaList *l_user = frame_meta->frame_user_meta_list; l_user != NULL; l_user = l_user->next)
    190.         {
    191.             NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data;
    192.             if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META)
    193.                 continue;
    194.  
    195.             /* convert to tensor metadata */
    196.             NvDsInferTensorMeta *meta = (NvDsInferTensorMeta *)user_meta->user_meta_data;
    197.             for (unsigned int i = 0; i < meta->num_output_layers; i++)
    198.             {
    199.                 NvDsInferLayerInfo *info = &meta->output_layers_info[i];
    200.                 info->buffer = meta->out_buf_ptrs_host[i];
    201.                 if (use_device_mem && meta->out_buf_ptrs_dev[i])
    202.                 {
    203.                     cudaMemcpy(meta->out_buf_ptrs_host[i], meta->out_buf_ptrs_dev[i],
    204.                                info->inferDims.numElements * 4, cudaMemcpyDeviceToHost);
    205.                 }
    206.             }
    207.             // g_print("\n1 %d\n", meta->num_output_layers);
    208.  
    209.             /* Parse output tensor and fill detection results into objectList. */
    210.             std::vector outputLayersInfo(meta->output_layers_info,
    211.                                                              meta->output_layers_info + meta->num_output_layers);
    212.             std::vector objectList;
    213. #if NVDS_VERSION_MAJOR >= 5
    214.             if (nvds_lib_major_version >= 5)
    215.             {
    216.                 if (meta->network_info.width != networkInfo.width ||
    217.                     meta->network_info.height != networkInfo.height ||
    218.                     meta->network_info.channels != networkInfo.channels)
    219.                 {
    220.                     g_error("failed to check pgie network info\n");
    221.                 }
    222.             }
    223. #endif
    224.             NvDsInferParseCustomWpod(outputLayersInfo, networkInfo,
    225.                                      detectionParams, objectList);
    226.             for (auto &obj : objectList)
    227.             {
    228.  
    229.                 NvDsObjectMeta *obj_meta = nvds_acquire_obj_meta_from_pool(batch_meta);
    230.                 obj_meta->unique_component_id = meta->unique_id;
    231.                 obj_meta->confidence = 0.0;
    232.                 /* This is an untracked object. Set tracking_id to -1. */
    233.                 obj_meta->object_id = UNTRACKED_OBJECT_ID;
    234.                 obj_meta->class_id = 0;
    235.                 // g_print("\nleft:%f\ttop:%f\twidth:%f\theight:%f", obj.left, obj.top, obj.width, obj.height);
    236.                 cv::Rect plate_rect = cv::Rect(obj.left * MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH, obj.top * MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT, obj.width * MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH, obj.height * MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT);
    237.                 // cv::Mat bgr = in_mat(plate_rect);
    238.                 cv::Mat cropref = cv::Mat(in_mat, plate_rect);
    239.                 cv::Mat crop;
    240.                 cropref.copyTo(crop);
    241.                 //cv::imwrite("output/" + std::to_string(idx) + "raw.jpg", crop);
    242.                 cv::Size out_size;
    243.                 // bool type;
    244.                 if (obj.width / obj.height < 1.7)
    245.                 {
    246.                     out_size.width = 280;
    247.                     out_size.height = 200;
    248.                     // type = 0;
    249.                 }
    250.                 else
    251.                 {
    252.                     out_size.width = 470;
    253.                     out_size.height = 110;
    254.                     // type = 1;
    255.                 }
    256.  
    257.                 std::vectorfloat>> H = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}};
    258.                 float t_pts[3][4];
    259.                 getRectPts(t_pts, 0, 0, out_size.width, out_size.height);
    260.                 float ptsh[3][4];
    261.                 for (int j = 0; j < 4; j++)
    262.                 {
    263.                     ptsh[0][j] = obj.pts[0][j] * MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH;
    264.                 }
    265.                 for (int j = 0; j < 4; j++)
    266.                 {
    267.                     ptsh[1][j] = obj.pts[1][j] * MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT;
    268.                 }
    269.                 for (int i = 0; i < 4; i++)
    270.                 {
    271.                     ptsh[2][i] = 1.0;
    272.                 }
    273.                 find_T_matrix(t_pts, ptsh, H);
    274.                 float Hf[3][3];
    275.                 for (int i = 0; i < 3; i++)
    276.                 {
    277.                     for (int j = 0; j < 3; j++)
    278.                     {
    279.                         Hf[i][j] = H[i][j];
    280.                     }
    281.                 }
    282.                 cv::Mat H1(3, 3, CV_32F);
    283.                 std::memcpy(H1.data, Hf, 3 * 3 * sizeof(float));
    284.                 cv::Mat tmp_img;
    285.                 cv::warpPerspective(in_mat, tmp_img, H1, out_size, 1, 0);
    286.                 //cv::imwrite("output/im" + std::to_string(idx) + ".jpg", tmp_img);
    287.                 cv::resize(tmp_img, tmp_img, cv::Size(SGIE_INPUT_W, SGIE_INPUT_H));
    288.                 tmp_img.copyTo(in_mat(cv::Rect(SGIE_PAD_OUT, SGIE_PAD_OUT, SGIE_INPUT_W, SGIE_INPUT_H)));
    289.  
    290.                 /* Assign bounding box coordinates. */
    291.                 NvOSD_RectParams &rect_params = obj_meta->rect_params;
    292.                 NvOSD_TextParams &text_params = obj_meta->text_params;
    293.                 NvOSD_RectParams &sgie_rect_params = obj_meta->sgie_rect_params;
    294.                 rect_params.left = obj.left * MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH;
    295.                 rect_params.top = obj.top * MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT;
    296.                 rect_params.width = obj.width * MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH;
    297.                 rect_params.height = obj.height * MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT;
    298.                 rect_params.border_width = 3;
    299.                 rect_params.has_bg_color = 0;
    300.                 rect_params.border_color = (NvOSD_ColorParams){0, 1, 0, 1};
    301.                 /* SGIE bounding box*/
    302.                 sgie_rect_params.left = SGIE_PAD_OUT;
    303.                 sgie_rect_params.top = SGIE_PAD_OUT;
    304.                 sgie_rect_params.width = SGIE_INPUT_W;
    305.                 sgie_rect_params.height = SGIE_INPUT_H;
    306.                 /* display_text requires heap allocated memory. */
    307.                 // text_params.display_text = g_strdup(type ? "Bien 1 hang" : "Bien 2 hang");
    308.                 /* Display text above the left top corner of the object. */
    309.                 text_params.x_offset = rect_params.left;
    310.                 text_params.y_offset = rect_params.top - 10;
    311.                 /* Set black background for the text. */
    312.                 text_params.set_bg_clr = 1;
    313.                 text_params.text_bg_clr = (NvOSD_ColorParams){0, 0, 0, 1};
    314.                 /* Font face, size and color. */
    315.                 text_params.font_params.font_name = (gchar *)"Serif";
    316.                 text_params.font_params.font_size = 20;
    317.                 text_params.font_params.font_color = (NvOSD_ColorParams){1, 1, 1, 1};
    318.                 nvds_add_obj_meta_to_frame(frame_meta, obj_meta, NULL);
    319.                 NvBufSurfaceSyncForDevice(surface, frame_meta->batch_id, 0);
    320.             }
    321.         }
    322.         NvBufSurfaceUnMap(surface, frame_meta->batch_id, 0);
    323.     }
    324.     use_device_mem = 1 - use_device_mem;
    325.     gst_buffer_unmap(inbuf, &in_map_info);
    326.     return GST_PAD_PROBE_OK;
    327. }
    328.  
    329. static GstPadProbeReturn
    330. sgie_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data)
    331. {
    332.     NvDsMetaList *l_obj = NULL;
    333.     NvDsObjectMeta *obj_meta = NULL;
    334.     NvDsBatchMeta *batch_meta =
    335.         gst_buffer_get_nvds_batch_meta(GST_BUFFER(info->data));
    336.     static guint use_device_mem = 0;
    337.     for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next)
    338.     {
    339.         NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)l_frame->data;
    340.         for (NvDsMetaList *l_obj = frame_meta->obj_meta_list; l_obj != NULL;
    341.              l_obj = l_obj->next)
    342.         {
    343.             NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data;
    344.             for (NvDsMetaList *l_user = obj_meta->obj_user_meta_list; l_user != NULL;
    345.                  l_user = l_user->next)
    346.             {
    347.                 NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data;
    348.                 if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META)
    349.                     continue;
    350.                 /* convert to tensor metadata */
    351.                 NvDsInferTensorMeta *meta = (NvDsInferTensorMeta *)user_meta->user_meta_data;
    352.                 for (unsigned int i = 0; i < meta->num_output_layers; i++)
    353.                 {
    354.                     NvDsInferLayerInfo *info = &meta->output_layers_info[i];
    355.                     info->buffer = meta->out_buf_ptrs_host[i];
    356.                     if (use_device_mem && meta->out_buf_ptrs_dev[i])
    357.                     {
    358.                         cudaMemcpy(meta->out_buf_ptrs_host[i], meta->out_buf_ptrs_dev[i],
    359.                                    info->inferDims.numElements * 4, cudaMemcpyDeviceToHost);
    360.                     }
    361.                 }
    362.                 std::vector outputLayersInfo(meta->output_layers_info,
    363.                                                                  meta->output_layers_info + meta->num_output_layers);
    364.                 float *prob = (float *)outputLayersInfo[0].buffer;
    365.                 std::vector<int> preds;
    366.                 for (int i = 0; i < 26; i++)
    367.                 {
    368.                     int maxj = 0;
    369.                     for (int j = 1; j < 37; j++)
    370.                     {
    371.                         if (prob[37 * i + j] > prob[37 * i + maxj])
    372.                             maxj = j;
    373.                     }
    374.                     preds.push_back(maxj);
    375.                 }
    376.                 std::string raw = strDecode(preds, true);
    377.                 std::string sim = strDecode(preds, false);
    378.                 if (SHOW_CRNN)
    379.                 {
    380.                     g_print("\nIdx = \nRaw : %s", raw.c_str());
    381.                     g_print("\nSim : %s", sim.c_str());
    382.                 }
    383.                 else
    384.                 {
    385.                     obj_meta->text_params.display_text = g_strdup(sim.c_str());
    386.                 }
    387.             }
    388.         }
    389.     }
    390.  
    391.     return GST_PAD_PROBE_OK;
    392. }
    393.  
    394. static void
    395. cb_newpad(GstElement *decodebin, GstPad *decoder_src_pad, gpointer data)
    396. {
    397.     GstCaps *caps = gst_pad_get_current_caps(decoder_src_pad);
    398.     const GstStructure *str = gst_caps_get_structure(caps, 0);
    399.     const gchar *name = gst_structure_get_name(str);
    400.     GstElement *source_bin = (GstElement *)data;
    401.     GstCapsFeatures *features = gst_caps_get_features(caps, 0); //
    402.     if (!strncmp(name, "video", 5))
    403.     {
    404.  
    405.         if (gst_caps_features_contains(features, MEMORY_FEATURES))
    406.         {
    407.             GstPad *bin_ghost_pad = gst_element_get_static_pad(source_bin, "src");
    408.             if (!gst_ghost_pad_set_target(GST_GHOST_PAD(bin_ghost_pad),
    409.                                           decoder_src_pad))
    410.             {
    411.                 g_printerr("Failed to link decoder src pad to source bin ghost pad\n");
    412.             }
    413.             gst_object_unref(bin_ghost_pad);
    414.         }
    415.         else
    416.         {
    417.             g_printerr("Error: Decodebin did not pick nvidia decoder plugin.\n");
    418.         }
    419.     }
    420. }
    421. static void decodebin_child_added(GstChildProxy *child_proxy, GObject *object,
    422.                                   gchar *name, gpointer user_data)
    423. {
    424.     g_print("Decodebin child added: %s\n", name);
    425.     if (g_strrstr(name, "decodebin") == name)
    426.     {
    427.         g_signal_connect(G_OBJECT(object), "child-added",
    428.                          G_CALLBACK(decodebin_child_added), user_data);
    429.     }
    430. }
    431.  
    432. static GstElement *
    433. create_source_bin(guint index, gchar *uri)
    434. {
    435.     GstElement *bin = NULL, *uri_decode_bin = NULL;
    436.  
    437.     gchar bin_name[16] = {};
    438.  
    439.     g_snprintf(bin_name, 15, "source-bin-%02d", index);
    440.     /* Create a source GstBin to abstract this bin's content from the rest of the
    441.    * pipeline */
    442.     bin = gst_bin_new(bin_name);
    443.  
    444.     /* Source element for reading from the uri.
    445.    * We will use decodebin and let it figure out the container format of the
    446.    * stream and the codec and plug the appropriate demux and decode plugins. */
    447.     uri_decode_bin = gst_element_factory_make("uridecodebin", "uri-decode-bin");
    448.  
    449.     /* We set the input uri to the source element */
    450.     g_object_set(G_OBJECT(uri_decode_bin), "uri", uri, NULL);
    451.     /* Connect to the "pad-added" signal of the decodebin which generates a
    452.    * callback once a new pad for raw data has beed created by the decodebin */
    453.  
    454.     g_signal_connect(G_OBJECT(uri_decode_bin), "pad-added", G_CALLBACK(cb_newpad), bin);
    455.  
    456.     g_signal_connect(G_OBJECT(uri_decode_bin), "child-added",
    457.                      G_CALLBACK(decodebin_child_added), bin);
    458.  
    459.     gst_bin_add(GST_BIN(bin), uri_decode_bin);
    460.  
    461.     /* We need to create a ghost pad for the source bin which will act as a proxy
    462.    * for the video decoder src pad. The ghost pad will not have a target right
    463.    * now. Once the decode bin creates the video decoder and generates the
    464.    * cb_newpad callback, we will set the ghost pad target to the video decoder
    465.    * src pad. */
    466.     if (!gst_element_add_pad(bin, gst_ghost_pad_new_no_target("src",
    467.                                                               GST_PAD_SRC)))
    468.     {
    469.         g_printerr("Failed to add ghost pad in source bin\n");
    470.         return NULL;
    471.     }
    472.  
    473.     return bin;
    474. }
    475.  
    476. int main(int argc, char *argv[])
    477. {
    478.     GMainLoop *loop = NULL;
    479.     GstElement *pipeline = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL,
    480.                *queue1, *queue2, *queue3, *queue4, *queue5, *nvvidconv = NULL, *qtmux = NULL, *videoconvert = NULL,
    481.                *nvosd = NULL, *tiler = NULL, *sgie = NULL, *queue_sgie = NULL, *nvvidconv1 = NULL,
    482.                *h264parser = NULL, *decoder = NULL;
    483.     GstElement *filter1 = NULL, *filter2 = NULL, *filter3 = NULL, *filter4 = NULL, *x264enc = NULL, *converter = NULL;
    484.     GstCaps *caps1 = NULL, *caps2 = NULL, *caps3 = NULL, *caps4 = NULL;
    485. #ifdef PLATFORM_TEGRA
    486.     GstElement *transform = NULL;
    487. #endif
    488.     GstBus *bus = NULL;
    489.     guint bus_watch_id;
    490.     GstPad *pgie_src_pad = NULL, *sgie_src_pad = NULL;
    491.     guint i, num_sources;
    492.     guint tiler_rows, tiler_columns;
    493.     guint pgie_batch_size;
    494.  
    495.     /* Check input arguments */
    496.     if (argc < 2)
    497.     {
    498.         g_printerr("Usage: %s  [uri2] ... [uriN] \n", argv[0]);
    499.         return -1;
    500.     }
    501.     num_sources = argc - 1;
    502.  
    503.     /* Standard GStreamer initialization */
    504.     gst_init(&argc, &argv);
    505.     loop = g_main_loop_new(NULL, FALSE);
    506.  
    507.     /* Create gstreamer elements */
    508.     /* Create Pipeline element that will form a connection of other elements */
    509.     pipeline = gst_pipeline_new("alpr-pipeline");
    510.  
    511.     /* Create nvstreammux instance to form batches from one or more sources. */
    512.     streammux = gst_element_factory_make("nvstreammux", "stream-muxer");
    513.     queue_sgie = gst_element_factory_make("queue", "queue_sgie");
    514.  
    515.     if (!pipeline || !streammux)
    516.     {
    517.         g_printerr("One element could not be created. Exiting.\n");
    518.         return -1;
    519.     }
    520.     gst_bin_add(GST_BIN(pipeline), streammux);
    521.  
    522.     for (i = 0; i < num_sources; i++)
    523.     {
    524.         GstPad *sinkpad, *srcpad;
    525.         gchar pad_name[16] = {};
    526.         GstElement *source_bin = create_source_bin(i, argv[i + 1]);
    527.  
    528.         if (!source_bin)
    529.         {
    530.             g_printerr("Failed to create source bin. Exiting.\n");
    531.             return -1;
    532.         }
    533.         gst_bin_add(GST_BIN(pipeline), source_bin);
    534.  
    535.         g_snprintf(pad_name, 15, "sink_%u", i);
    536.         sinkpad = gst_element_get_request_pad(streammux, pad_name);
    537.         if (!sinkpad)
    538.         {
    539.             g_printerr("Streammux request sink pad failed. Exiting.\n");
    540.             return -1;
    541.         }
    542.  
    543.         srcpad = gst_element_get_static_pad(source_bin, "src");
    544.         if (!srcpad)
    545.         {
    546.             g_printerr("Failed to get src pad of source bin. Exiting.\n");
    547.             return -1;
    548.         }
    549.  
    550.         if (gst_pad_link(srcpad, sinkpad) != GST_PAD_LINK_OK)
    551.         {
    552.             g_printerr("Failed to link source bin to stream muxer. Exiting.\n");
    553.             return -1;
    554.         }
    555.  
    556.         gst_object_unref(srcpad);
    557.         gst_object_unref(sinkpad);
    558.     }
    559.  
    560.     /* Since the data format in the input file is elementary h264 stream,
    561.    * we need a h264parser */
    562.     h264parser = gst_element_factory_make("h264parse", "h264-parser");
    563.  
    564.     /* Use nvdec_h264 for hardware accelerated decode on GPU */
    565.     decoder = gst_element_factory_make("nvv4l2decoder", "nvv4l2-decoder");
    566.     /* Use nvinfer to infer on batched frame. */
    567.     pgie = gst_element_factory_make("nvinfer", "primary-nvinference-engine");
    568.     sgie = gst_element_factory_make("nvinfer", "secondary-nvinference-engine");
    569.     /* Add queue elements between every two elements */
    570.     queue1 = gst_element_factory_make("queue", "queue1");
    571.     queue2 = gst_element_factory_make("queue", "queue2");
    572.     queue3 = gst_element_factory_make("queue", "queue3");
    573.     queue4 = gst_element_factory_make("queue", "queue4");
    574.     queue5 = gst_element_factory_make("queue", "queue5");
    575.     filter1 = gst_element_factory_make("capsfilter", "filter1");
    576.     filter2 = gst_element_factory_make("capsfilter", "filter2");
    577.     filter3 = gst_element_factory_make("capsfilter", "filter3");
    578.     filter4 = gst_element_factory_make("capsfilter", "filter4");
    579.     x264enc = gst_element_factory_make("x264enc", "h264 encoder");
    580.     converter = gst_element_factory_make("videoconvert", "converter");
    581.     videoconvert = gst_element_factory_make("videoconvert", "converter");
    582.     qtmux = gst_element_factory_make("qtmux", "muxer");
    583.     nvvidconv1 = gst_element_factory_make("nvvideoconvert", "nvvideo-converter1");
    584.     /* Use nvtiler to composite the batched frames into a 2D tiled array based
    585.    * on the source of the frames. */
    586.     tiler = gst_element_factory_make("nvmultistreamtiler", "nvtiler");
    587.  
    588.     /* Use convertor to convert from NV12 to RGBA as required by nvosd */
    589.     nvvidconv = gst_element_factory_make("nvvideoconvert", "nvvideo-converter");
    590.  
    591.     /* Create OSD to draw on the converted RGBA buffer */
    592.     nvosd = gst_element_factory_make("nvdsosd", "nv-onscreendisplay");
    593.  
    594.     sink = gst_element_factory_make("filesink", "nvvideo-renderer");
    595.  
    596.     if (!pgie || !tiler || !nvvidconv || !nvosd || !sink)
    597.     {
    598.         g_printerr("One element could not be created. Exiting.\n");
    599.         return -1;
    600.     }
    601.     if (!converter || !x264enc || !qtmux || !filter3 || !filter4)
    602.     {
    603.         g_printerr("One element could not be created. Exiting.\n");
    604.         return -1;
    605.     }
    606. #ifdef PLATFORM_TEGRA
    607.     transform = gst_element_factory_make("queue", "nvegl-transform");
    608.     if (!transform)
    609.     {
    610.         g_printerr("One tegra element could not be created. Exiting.\n");
    611.         return -1;
    612.     }
    613. #endif
    614.  
    615.     // #ifndef PLATFORM_TEGRA
    616.     //     /* Set properties of the nvvideoconvert element
    617.     //    * requires unified cuda memory for opencv blurring on CPU
    618.     //    */
    619.     //     g_object_set(G_OBJECT(nvvidconv), "nvbuf-memory-type", 0, NULL);
    620.     // #else
    621.     //     g_object_set(G_OBJECT(nvvidconv), "nvbuf-memory-type", 4, NULL);
    622.     // #endif
    623.     g_object_set(G_OBJECT(streammux), "batch-size", num_sources, NULL);
    624.  
    625.     g_object_set(G_OBJECT(streammux), "width", MUXER_OUTPUT_WIDTH, "height",
    626.                  MUXER_OUTPUT_HEIGHT,
    627.                  "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL);
    628.  
    629.     /* Configure the nvinfer element using the nvinfer config file. */
    630.     g_object_set(G_OBJECT(pgie),
    631.                  "config-file-path", PGIE_CONFIG_FILE, "output-tensor-meta", TRUE, NULL);
    632.     g_object_set(G_OBJECT(sgie), "config-file-path", SGIE_CONFIG_FILE, "output-tensor-meta", TRUE, NULL);
    633.  
    634.     /* Override the batch-size set in the config file with the number of sources. */
    635.     g_object_get(G_OBJECT(pgie), "batch-size", &pgie_batch_size, NULL);
    636.     if (pgie_batch_size != num_sources)
    637.     {
    638.         g_printerr("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n",
    639.                    pgie_batch_size, num_sources);
    640.         g_object_set(G_OBJECT(pgie), "batch-size", num_sources, NULL);
    641.     }
    642.  
    643.     tiler_rows = (guint)sqrt(num_sources);
    644.     tiler_columns = (guint)ceil(1.0 * num_sources / tiler_rows);
    645.     /* we set the tiler properties here */
    646.     g_object_set(G_OBJECT(tiler), "rows", tiler_rows, "columns", tiler_columns,
    647.                  "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL);
    648.  
    649.     g_object_set(G_OBJECT(nvosd), "process-mode", OSD_PROCESS_MODE,
    650.                  "display-text", OSD_DISPLAY_TEXT, NULL);
    651.  
    652.     g_object_set(G_OBJECT(sink), "sync", FALSE, NULL);
    653.     g_object_set(G_OBJECT(sink), "location", OUTPUT_FILE, NULL);
    654.  
    655.     /* we add a message handler */
    656.     bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline));
    657.     bus_watch_id = gst_bus_add_watch(bus, bus_call, loop);
    658.     gst_object_unref(bus);
    659.  
    660.     /* Set up the pipeline */
    661.     /* we add all elements into the pipeline */
    662. #ifdef PLATFORM_TEGRA
    663.     // , queue_sgie, sgie
    664.     // h264parser, decoder,
    665.     gst_bin_add_many(GST_BIN(pipeline), queue1, pgie, queue_sgie, sgie, queue2, tiler, queue3,
    666.                      filter1, nvvidconv, filter2, nvosd, nvvidconv1, filter3, converter, filter4,
    667.                      x264enc, qtmux, transform, sink, NULL);
    668.     /* we link the elements together */
    669.     if (!gst_element_link_many(streammux, queue1, pgie, queue_sgie, sgie, queue2, tiler, queue3,
    670.                                filter1, nvvidconv, filter2, nvosd, nvvidconv1, filter3, converter, filter4,
    671.                                x264enc, qtmux, transform, sink, NULL))
    672.     {
    673.         g_printerr("Elements could not be linked. Exiting.\n");
    674.         return -1;
    675.     }
    676. #else
    677.     gst_bin_add_many(GST_BIN(pipeline), queue1, pgie, queue_sgie, sgie, queue2, tiler, queue3,
    678.                      filter1, nvvidconv, filter2, nvosd, nvvidconv1, filter3, converter, filter4,
    679.                      x264enc, qtmux, sink, NULL);
    680.     /* we link the elements together */
    681.     if (!gst_element_link_many(streammux, queue1, pgie, queue_sgie, sgie, queue2, tiler, queue3,
    682.                                filter1, nvvidconv, filter2, nvosd, nvvidconv1, filter3, converter, filter4,
    683.                                x264enc, qtmux, sink, NULL))
    684.     {
    685.         g_printerr("Elements could not be linked. Exiting.\n");
    686.         return -1;
    687.     }
    688. #endif
    689.     caps1 = gst_caps_from_string("video/x-raw(memory:NVMM), format=NV12");
    690.     g_object_set(G_OBJECT(filter1), "caps", caps1, NULL);
    691.     gst_caps_unref(caps1);
    692.     caps2 = gst_caps_from_string("video/x-raw(memory:NVMM), format=RGBA");
    693.     g_object_set(G_OBJECT(filter2), "caps", caps2, NULL);
    694.     gst_caps_unref(caps2);
    695.     caps3 = gst_caps_from_string("video/x-raw, format=RGBA");
    696.     g_object_set(G_OBJECT(filter3), "caps", caps3, NULL);
    697.     gst_caps_unref(caps3);
    698.     caps4 = gst_caps_from_string("video/x-raw, format=NV12");
    699.     g_object_set(G_OBJECT(filter4), "caps", caps4, NULL);
    700.     gst_caps_unref(caps4);
    701.  
    702.     /* Lets add probe to get informed of the meta data generated, we add probe to
    703.    * the sink pad of the osd element, since by that time, the buffer would have
    704.    * had got all the metadata. */
    705.     pgie_src_pad = gst_element_get_static_pad(pgie, "src");
    706.     gst_pad_add_probe(pgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER,
    707.                       pgie_pad_buffer_probe, NULL, NULL);
    708.  
    709.     gst_object_unref(pgie_src_pad);
    710.     sgie_src_pad = gst_element_get_static_pad(sgie, "src");
    711.     gst_pad_add_probe(sgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER,
    712.                       sgie_pad_buffer_probe, (gpointer)sink, NULL);
    713.     gst_object_unref(sgie_src_pad);
    714.  
    715.     /* Set the pipeline to "playing" state */
    716.     GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline");
    717.     g_print("Now playing:");
    718.     for (i = 0; i < num_sources; i++)
    719.     {
    720.         g_print(" %s", argv[i + 1]);
    721.     }
    722.     g_print("\n");
    723.     gst_element_set_state(pipeline, GST_STATE_PLAYING);
    724.  
    725.     /* Wait till pipeline encounters an error or EOS */
    726.     g_print("Running...\n");
    727.     g_main_loop_run(loop);
    728.  
    729.     /* Out of the main loop, clean up nicely */
    730.     g_print("Returned, stopping playback\n");
    731.     gst_element_set_state(pipeline, GST_STATE_NULL);
    732.     g_print("Deleting pipeline\n");
    733.     gst_object_unref(GST_OBJECT(pipeline));
    734.     g_source_remove(bus_watch_id);
    735.     g_main_loop_unref(loop);
    736.     return 0;
    737. }

     sgie获取帧数据，转opencv mat图：

    1.  
    2.  
    3. void Xstreamer::get_product_size(GstBuffer *buf, gint batch_id, gint source_id, std::vector &boxes)
    4. {
    5.  
    6.     if (boxes.size()==0){
    7.         return;    
    8.     }
    9.     NvDsMetaList * l_frame = NULL;
    10.     NvDsMetaList * l_user_meta = NULL;
    11.     // Get original raw data
    12.     GstMapInfo in_map_info;
    13.  
    14.     memset(&in_map_info, 0, sizeof(in_map_info));
    15.     if (!gst_buffer_map(buf, &in_map_info, GST_MAP_READ)){
    16.         g_print("Error: Failed to map gst buffer\n");
    17.     }
    18.  
    19.     NvBufSurface *surface = (NvBufSurface *)in_map_info.data;
    20.     char *src_data = NULL;
    21. 		// cout<<"surfaceList[batch_id].dataSize: "<surfaceList[batch_id].dataSize<
    22.     src_data = (char *)malloc(surface->surfaceList[batch_id].dataSize);
    23.     if (src_data == NULL)
    24.     {
    25.         g_print("Error: failed to malloc src_data \n");
    26.     }
    27.  
    28.     NvBufSurfaceMap(surface, -1, -1, NVBUF_MAP_READ_WRITE);
    29.     NvBufSurfacePlaneParams *pParams = &surface->surfaceList[batch_id].planeParams;
    30. 		// cout<< "planeParams: "<< pParams->width[0] <<" "<height[0] <<" " << pParams->bytesPerPix[0]<< endl;
    31.    
    32.     gint frame_width = (gint)surface->surfaceList[batch_id].width;
    33.     gint frame_height = (gint)surface->surfaceList[batch_id].height;
    34.     gint frame_step = surface->surfaceList[batch_id].pitch;
    35.  
    36.   uint32_t current_frame_width = surface->surfaceList[batch_id].width;  
    37.     uint32_t current_frame_height = surface->surfaceList[batch_id].height;
    38.     // save image
    39.     cv::Mat current_frame_data = cv::Mat((gint)current_frame_height,
    40.                                           (gint)current_frame_width,
    41.                                           CV_8UC4,
    42.                                           surface->surfaceList[batch_id].mappedAddr.addr[0],
    43.                                           surface->surfaceList[batch_id].pitch
    44.                                           );
    45.     cv::Mat image_data((gint)current_frame_height,
    46.                         (gint)current_frame_width,
    47.                         CV_8UC4);
    48.     current_frame_data.copyTo(image_data);
    49.     // RGBA convert BGR 
    50.     cv::Mat in_mat = cv::Mat ((gint)current_frame_height,
    51.                                 (gint)current_frame_width, 
    52.                                 CV_8UC3);
    53.     cv::cvtColor(image_data, in_mat, cv::COLOR_RGBA2BGR);    // opencv4

  • 相关阅读:
    postgrest API CURD数据库
    操作系统实验四 进程间通信
    C and C++ 在线参考手册
    项目实战第二十五讲：复杂业务流程编排规则引擎
    nacos
    量化交易之One Piece篇 - linux - 定时任务（重启服务器、执行程序、验证）
    基础 | 并发编程 - [锁]
    【基础篇】一、什么是Flink
    泛型内容总结
    【C++初阶】C++入门(一)
  • 原文地址：https://blog.csdn.net/jacke121/article/details/126024941