• 使用Gstreamer+OpenCV实现两路图像数据混合拉流推流

    本示例看完后，可基本掌握以下内容
    1、利用opencv+gstreamer拉流推流的基本步骤
    2、可学习gstreamer，图像混合的一些插件用法
    3、TX2 NX上的视频编解码与硬件加速，H264编码参数调整
    4、linux下如何提高线程优先级

    我需要实现的功能是在TX2 NX上，拉取摄像头数据，并在摄像头数据上与另一张图片混合
    混合后推流到rtsp服务器。

    由于混合的时候需要保留透明度，但是OpenCV不支持四通道的数据写入。硬件加速混合插件无法剔除透明度。当然可以用内存（不是显存，无法硬件加速）中剔除指定颜色的插件实现，但效率太低

    所以只能利用VideoCapture先拉到摄像头数据，利用要混合的图片，手动计算叠加。

    叠加后利用VideoWriter推流到rtsp 服务器

    为提高推流效率，提高了推流线程优先级。

    具体流程：
    首先搭建rtsp服务端，可利用rtsp-simple-serve这个项目在局域网部署一套
     
     安装gstreamer后
     由于用到了rtspclientsink插件，所以需要安装rtsp插件
    // sudo apt-get update
    // sudo apt-get upgrade
    // sudo apt install gstreamer1.0-rtsp

    下面是利用VideoWriter推流的过程

    1. //cpp 实现
    2. #include "rtsp_push_stream.h"
    3.  
    4.  
    5. static int get_thread_policy(pthread_attr_t& attr) {
    6.     int policy;
    7.     int rs = pthread_attr_getschedpolicy(&attr, &policy);
    8.     assert(rs == 0);
    9.     switch (policy) {
    10.         case SCHED_FIFO:
    11.             cout << "policy = SCHED_FIFO" << endl;
    12.             break;
    13.  
    14.         case SCHED_RR:
    15.             cout << "policy = SCHED_RR" << endl;
    16.             break;
    17.  
    18.         case SCHED_OTHER:
    19.             cout << "policy = SCHED_OTHER" << endl;
    20.             break;
    21.  
    22.         default:
    23.             cout << "policy = UNKNOWN" << endl;
    24.             break;
    25.     }
    26.  
    27.     return policy;
    28. }
    29.  
    30. static void show_thread_priority(pthread_attr_t& attr, int policy) {
    31.     int priority = sched_get_priority_max(policy);
    32.     assert(priority != -1);
    33.     cout << "max_priority = " << priority << endl;
    34.  
    35.     priority = sched_get_priority_min(policy);
    36.     assert(priority != -1);
    37.     cout << "min_priority = " << priority << endl;
    38. }
    39.  
    40. static int get_thread_priority(pthread_attr_t& attr) {
    41.     struct sched_param param;
    42.  
    43.     int rs = pthread_attr_getschedparam(&attr, ¶m);
    44.     assert(rs == 0);
    45.     cout << "priority = " << param.__sched_priority << endl;
    46.  
    47.     return param.__sched_priority;
    48. }
    49.  
    50. static void set_thread_policy(pthread_attr_t& attr, int policy) {
    51.     int rs = pthread_attr_setschedpolicy(&attr, policy);
    52.     assert(rs == 0);
    53.     get_thread_policy(attr);
    54. }
    55.  
    56. RtspPushStream::RtspPushStream() : active_(true) {}
    57.  
    58. RtspPushStream::~RtspPushStream() {}
    59. void RtspPushStream::start() {
    60.  
    61.     string appsrcpipeline =
    62.         "appsrc ! video/x-raw, format=BGR ! queue ! videoconvert ! video/x-raw,format=RGBA ! nvvidconv ! nvv4l2h264enc "
    63.         "! h264parse ! qtmux ! filesink location={filename}  sync=false";
    64.  
    65.     //使用 rtspclientsink 需要安装插件
    66.     // sudo apt-get update
    67.     // sudo apt-get upgrade
    68.     // sudo apt install gstreamer1.0-rtsp
    69.  
    70.  
    71.     std::string pipeline_useglvideomixer =
    72.         "appsrc "
    73.         "! video/x-raw, format=BGR "
    74.         "! videoconvert "
    75.         "! video/x-raw,format=(string)RGBA , width=(int)1024, height=(int)600"
    76.         "! queue2"
    77.         "! alpha method=blue "
    78.         "! glvideomixer name = compos sink_0::zorder=1 sink_0::alpha=0.85 sink_1::alpha=1 sink_1::zorder=0 "
    79.         "sink_1::width=1024 sink_1::height=600 "
    80.         "! nvvidconv"
    81.         "! video/x-raw(memory:NVMM), format=(string)I420, width=(int)1024, height=(int)600"
    82.         "! nvv4l2h264enc "
    83.         "! rtspclientsink location=rtsp://192.168.20.99:8554/my_pipeline"
    84.         " nvarguscamerasrc"
    85.         "! video/x-raw(memory:NVMM),format=(string)NV12, width=(int)1640, height=(int)1232, framerate=(fraction)25/1"
    86.         "! queue2"
    87.         "! nvvidconv left=0 right=1640 top=136 bottom=1096 "
    88.         "! compos. ";
    89.  
    90.     // nvcompsositor 的两个输入必须是一样的图片格式和内存形式,需要将云图透明部分用白色(255,255,255)填充
    91.     std::string pipeline_nvcompsositor =
    92.         "appsrc "
    93.         "! video/x-raw, format=BGR "
    94.         "! videoconvert "
    95.         "! video/x-raw,format=(string)RGBA, width=(int)1024, height=(int)600"
    96.         "! nvvidconv "
    97.         "! queue2"
    98.         "! nvcompositor  name = compos sink_0::zorder=1 sink_0::alpha=0.5 "
    99.         "sink_1::alpha=1 "
    100.         "sink_1::zorder=0 sink_1::width=1024 sink_1::height=600 "
    101.         "! nvvidconv "
    102.         "! nvv4l2h264enc "
    103.         "! rtspclientsink location=rtsp://192.168.20.99:8554/my_pipeline"
    104.         " nvarguscamerasrc "
    105.         "! video/x-raw(memory:NVMM), format=(string)NV12, width=(int)1640, height=(int)1232,framerate=(fraction)25/1 "
    106.         "! nvvidconv  left=0 right=1640 top=136 bottom=1096 "
    107.         "! video/x-raw,format=(string)RGBA, width=(int)1024, height=(int)600 "
    108.         "! videobalance brightness=0.3 "
    109.         "! nvvidconv "
    110.         "! queue2"
    111.         "! compos. ";
    112.  
    113.     video_writer_.open(pipeline_nvcompsositor, cv::CAP_GSTREAMER, 0, 25, cv::Size(1024, 600));
    114.     mat_          = cv::imread("test.jpg");
    115.     write_thread_ = make_shared(&RtspPushStream::run, this);
    116.     write_thread_->join();
    117. }
    118.  
    119. void RtspPushStream::run() {
    120.     int id = 0;
    121.     while (active_) {
    122.         space_mat_ = cv::Mat(600, 1024, CV_8UC3, cv::Scalar(0, 0, 0));
    123.  
    124.         
    125.         video_writer_.write(space_mat_);
    126.  
    127.         std::this_thread::sleep_for(std::chrono::milliseconds(3));
    128.     }
    129. }
    130.  
    131. void RtspPushStream::end() {
    132.     active_ = false;
    133.     //LOG::info("RtspPushStream::end()!");
    134. }
    135.  
    136. void RtspPushStream::start_capture() {
    137.  
    138.     std::string pipeline_camear_capture =
    139.         " nvarguscamerasrc "
    140.         "! video/x-raw(memory:NVMM), format=(string)NV12, width=(int)1640, "
    141.         "height=(int)1232,framerate=(fraction)30/1 "
    142.         "! nvvidconv  left=0 right=1640 top=136 bottom=1096 "
    143.         "! video/x-raw,format=(string)I420, width=(int)1024, height=(int)600 "
    144.         "! videoconvert "
    145.         "! video/x-raw,format=(string)BGR "
    146.         "! appsink";
    147.  
    148.     video_capture_.open(pipeline_camear_capture, cv::CAP_GSTREAMER);
    149.     if (!video_capture_.isOpened()) {
    150.         //LOG::error("Failed to open VideoCapture");
    151.         return;
    152.     }
    153.     mat_ = cv::imread("test.jpg");
    154.     std::string pipeline_video_writer =
    155.         "appsrc "
    156.         "! video/x-raw, format=BGR "
    157.         "! videoconvert "
    158.         "! nvvidconv "
    159.         "! nvv4l2h264enc iframeinterval=10 idrinterval=32 "  //这里增加H264 i帧与idr帧次数，降低webrtc拉流时卡顿的频率和时间
    160.         "! rtspclientsink location=rtsp://192.168.20.99:8554/my_pipeline";
    161.  
    162.     video_writer_.open(pipeline_video_writer, cv::CAP_GSTREAMER, 0, 20, cv::Size(1024, 600));
    163.     cap_thread_ = make_shared(&RtspPushStream::run_capture, this);
    164.     cap_thread_->detach();
    165. }
    166.  
    167. void RtspPushStream::run_capture() {
    168.    pthread_attr_t     attr;
    169.     struct sched_param sched;
    170.     int                rs;
    171.  
    172.     rs = pthread_attr_init(&attr);
    173.     assert(rs == 0);
    174.  
    175.     int policy = get_thread_policy(attr);
    176.  
    177.     cout << "Show current configuration of priority" << endl;
    178.     show_thread_priority(attr, policy);
    179.  
    180.     cout << "Show priority of current thread" << endl;
    181.     int priority = get_thread_priority(attr);
    182.  
    183.     set_thread_policy(attr, SCHED_FIFO); // SCHED_RR
    184.     cout << "Restore current policy" << endl;
    185.     // set_thread_policy(attr, policy);
    186.     struct sched_param param_;
    187.     param_.sched_priority = sched_get_priority_max(SCHED_FIFO); // SCHED_RR
    188.     //这里设置线程优先级
    189.     cout << "param_.sched_priority  = " << param_.sched_priority << endl;
    190.     pthread_setschedparam(pthread_self(), SCHED_FIFO, ¶m_);
    191.     cout << "pthread_setschedparam success" << endl;
    192.  
    193.     rs = pthread_attr_destroy(&attr);
    194.     assert(rs == 0);
    195.     cout << "start capture!!!!!!!!!" << endl;
    196.  
    197.     int     id = 0;
    198.     cv::Mat src;
    199.     while (active_) {
    200.         //读取摄像头内容，也就是从摄像头拉流
    201.         video_capture_ >> src;
    202.         //在这里自定义融合算法就行了
    203.         //blender
    204.         // 两张图的透明度融合算法，取rgb像素进行融合
    205.         // data1[add_col]     = data2[add_col] * 0.4 + data1[add_col] * 0.6;
    206.         // data1[add_col + 1] = data2[add_col + 1] * 0.4 + data1[add_col + 1] * 0.6;
    207.         // data1[add_col + 2] = data2[add_col + 2] * 0.4 + data1[add_col + 2] * 0.6;
    208.  
    209.         
    210.  
    211.         //融合后，可以利用opencv给src写入当前时间用来观察
    212. auto              cur_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
    213.     std::stringstream ss;
    214.     ss << std::put_time(std::localtime(&cur_time), "%Y-%m-%d %H:%M:%S");
    215.     std::string str_time = ss.str();
    216.     cv::putText(src, str_time, cv::Point(830, 20), cv::HersheyFonts::FONT_ITALIC, 0.5, cv::Scalar(0, 0, 0), 1,
    217.                 4);
    218.         video_writer_.write(src);
    219.         src.release();
    220.     }
    221.  
    222.     //LOG::info("RtspPushStream::run_capture() end!");
    223. }
    1.  
    2. //头文件
    3. #ifndef RTSHPUSHSTREAM_H
    4. #define RTSHPUSHSTREAM_H
    5. #include 
    6. #include 
    7. #include 
    8. #include 
    9. #include 
    10. #include 
    11. #include 
    12. #include 
    13. #include 
    14. #include 
    15. #include 
    16.  
    17.  
    18. using namespace std;
    19. class RtspPushStream {
    20. public:
    21.     RtspPushStream();
    22.     ~RtspPushStream();
    23.  
    24.     void start();
    25.     void write_image(cv::Mat image);
    26.  
    27.     void run();
    28.     void start_capture();
    29.     void run_capture();
    30.     void end();
    31.  
    32. private:
    33.     mutex                mutex_;
    34.     cv::VideoWriter      video_writer_;
    35.     list        img_mats_;
    36.     bool                 active_;
    37.     cv::Mat              mat_;
    38.     cv::Mat              space_mat_;
    39.     shared_ptr   cap_thread_;
    40.     shared_ptr   write_thread_;
    41.     cv::VideoCapture     video_capture_;
    42.     std::vector push_frames_;
    43.     std::mutex           push_frames_lock_;
    44. };
    45. #endif


    推流后用相关视频软件拉流就能看到效果，如PotPlayer 64 bit，VLC等等

  • 相关阅读:
    基于springboot高考填报志愿综合参考系统设计与实现-计算机毕业设计源码+LW文档
    SOME/IP 协议介绍（三）参数和数据结构的序列化
    CSS 展开收起 样式
    用户考试分数大于单科科目平均分的查询
    工业信息物理系统攻击检测增强模型
    Vue——整理一些开源管理平台项目
    【C语言】模拟实现内存函数
    图像风格转换易语言代码
    前端设计模式和设计原则之设计模式
    SpringBoot-黑马程序员-学习笔记（六）
  • 原文地址：https://blog.csdn.net/zanglengyu/article/details/128130290