• Windows平台如何实现RTSP流二次编码并添加动态水印后推送RTMP或轻量级RTSP服务

    技术背景

    我们在对接RTSP播放器相关的技术诉求的时候，遇到这样的需求，客户做特种设备巡检的，需要把摄像头拍到的RTSP流拉下来，然后添加动态水印后，再生成新的RTSP URL，供平台调用。真个流程需要延迟尽可能的低，分辨率要支持到1080p，并需要把添加过动态水印的数据，保存到本地。

    技术实现

    在此之前，大牛直播SDK有非常成熟的RTSP播放、轻量级RTSP服务和录像模块，要做的就是，拉取到RTSP流后，把解码后的YUV或RGB回调给上层，上层通过图层的形式，添加动态文字水印（图片水印亦可），然后，投递给轻量级RTSP服务，RTSP服务对外提供个拉流的RTSP URL，无图无真相：

    左侧就是我们基于Windows平台C#的播放器的demo，二次开发的，添加了软、硬编码设置（考虑到分辨率比较高，添加支持了硬编码选项设置）、动态水印设置、轻量级RTSP服务、实时录像和RTMP推送。

    先说数据回调，本文以回调yuv数据为例：

    1. video_frame_call_back_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
    2. NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FORMAT_RGB32, IntPtr.Zero, video_frame_call_back_);

    回调后的数据，投递到轻量级RTSP服务模块。

    1. public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame)
    2.         {
    3.             if (frame == IntPtr.Zero)
    4.             {
    5.                 return;
    6.             }
    7.  
    8.             NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));
    9.  
    10.             if (publisher_wrapper_ != null)
    11.             {
    12.                 if (publisher_wrapper_.IsPublisherHandleAvailable())
    13.                 {
    14.                     if (publisher_wrapper_.IsPublishing() || publisher_wrapper_.IsRecording() || publisher_wrapper_.IsRTSPPublisherRunning())
    15.                     {
    16.                         //publisher_wrapper_.OnPostRGB32Data(0, video_frame.plane0_, video_frame.width_ * 4 * video_frame.height_, video_frame.stride0_,
    17.                         //            video_frame.width_, video_frame.height_);
    18.  
    19.                         publisher_wrapper_.OnPostYUVData(0, video_frame.plane0_, video_frame.stride0_, video_frame.plane1_, video_frame.stride1_,
    20.                                     video_frame.plane2_, video_frame.stride2_,
    21.                                     video_frame.width_, video_frame.height_);
    22.                     }
    23.                 }
    24.             }
    25.         }

    音频由于暂时不要二次处理，直接投递过去，如果需要处理的话，处理后再投递给publisher wrapper：

    1. public void SetAudioPCMFrameCallBack(IntPtr handle, IntPtr user_data,
    2.              UInt32 status, IntPtr data, UInt32 size,
    3.              Int32 sample_rate, Int32 channel, Int32 per_channel_sample_number)
    4.         {
    5.             if (data == IntPtr.Zero || size == 0)
    6.             {
    7.                 return;
    8.             }
    9.  
    10.             if (publisher_wrapper_ != null)
    11.             {
    12.                 if (publisher_wrapper_.IsPublisherHandleAvailable())
    13.                 {
    14.                     if (publisher_wrapper_.IsPublishing() || publisher_wrapper_.IsRecording() || publisher_wrapper_.IsRTSPPublisherRunning())
    15.                     {
    16.                         publisher_wrapper_.OnPostAudioPCMData(data, size, 0, sample_rate, channel, per_channel_sample_number);
    17.                     }
    18.                 }
    19.             }
    20.  
    21.         }

    设置文字水印字体：

    1.         private void btn_set_font_Click(object sender, EventArgs e)
    2.         {
    3.             FontDialog font_dlg = new FontDialog();
    4.             DialogResult result = font_dlg.ShowDialog();
    5.  
    6.             if (result == DialogResult.OK)
    7.             {
    8.                 // 获取用户所选字体
    9.                 Font selectedFont = font_dlg.Font;
    10.                 btn_set_font.Text = "" + selectedFont.Name + ", " + selectedFont.Size + "pt";
    11.  
    12.                 selected_osd_font_ = new Font(selectedFont.Name, selectedFont.Size, FontStyle.Regular, GraphicsUnit.Point);
    13.             }
    14.         }

    动态设置文字水印：

    1.         private async void btn_text_osd_Click(object sender, EventArgs e)
    2.         {
    3.             string format = "yyyy-MM-dd HH:mm:ss.fff";
    4.  
    5.             StringBuilder sb = new StringBuilder();
    6.             sb.Append("施工单位：上海视沃信息科技有限公司(daniusdk.com)");
    7.             sb.Append("\r\n");
    8.             sb.Append("施工时间：");
    9.             sb.Append(DateTime.Now.DayOfWeek.ToString());
    10.             sb.Append(" ");
    11.             sb.Append(DateTime.Now.ToString(format));
    12.             sb.Append("\r\n");
    13.             sb.Append("当前位置：上海市");
    14.             string str = sb.ToString();
    15.  
    16.             Bitmap bmp = GenerateBitmap(str);
    17.  
    18.             int index = 1;
    19.             int x = 0;
    20.             int y = 200;
    21.             UpdateLayerRegion(index, x, y, bmp);
    22.  
    23.             await Task.Delay(30);
    24.             UpdateARGBBitmap(index, bmp);
    25.         }

    如果需要硬编码：

    1.             if (btn_check_video_hardware_encoder_.Checked)
    2.             {
    3.                 is_hw_encoder = true;
    4.             }
    5.  
    6.             Int32 cur_sel_encoder_id = 0;
    7.             Int32 cur_sel_gpu = 0;
    8.  
    9.             if (is_hw_encoder)
    10.             {
    11.                 int cur_sel_hw = combobox_video_encoders_.SelectedIndex;
    12.                 if (cur_sel_hw >= 0)
    13.                 {
    14.                     cur_sel_encoder_id = Convert.ToInt32(combobox_video_encoders_.SelectedValue);
    15.                     cur_sel_gpu = -1;
    16.  
    17.                     int cur_sel_hw_dev = combobox_video_hardware_encoder_devices_.SelectedIndex;
    18.                     if (cur_sel_hw_dev >= 0)
    19.                     {
    20.                         cur_sel_gpu = Convert.ToInt32(combobox_video_hardware_encoder_devices_.SelectedValue);
    21.                     }
    22.                 }
    23.                 else
    24.                 {
    25.                     is_hw_encoder = false;
    26.                 }
    27.             }
    28.  
    29.             if (!is_hw_encoder)
    30.             {
    31.                 if ((int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264 == cur_video_codec_id)
    32.                 {
    33.                     cur_sel_encoder_id = btn_check_openh264_encoder_.Checked ? 1 : 0;
    34.                 }
    35.             }
    36.  
    37.             publisher_wrapper_.SetVideoEncoder((int)(is_hw_encoder ? 1 : 0), (int)cur_sel_encoder_id, (uint)cur_video_codec_id, (int)cur_sel_gpu);
    38.  
    39.             publisher_wrapper_.SetVideoQualityV2(publisher_wrapper_.CalVideoQuality(width_, height_, is_h264_encoder));
    40.  
    41.             publisher_wrapper_.SetVideoBitRate(publisher_wrapper_.CalBitRate(video_fps_, width_, height_));
    42.  
    43.             publisher_wrapper_.SetVideoMaxBitRate(publisher_wrapper_.CalMaxKBitRate(video_fps_, width_, height_, false));
    44.  
    45.             publisher_wrapper_.SetVideoKeyFrameInterval(key_frame_interval_);
    46.  
    47.             if (is_h264_encoder)
    48.             {
    49.                 publisher_wrapper_.SetVideoEncoderProfile(1);
    50.             }
    51.  
    52.             publisher_wrapper_.SetVideoEncoderSpeed(publisher_wrapper_.CalVideoEncoderSpeed(width_, height_, is_h264_encoder));

    启动停止RTSP服务：

    1.         private void btn_rtsp_service_Click(object sender, EventArgs e)
    2.         {
    3.             if(publisher_wrapper_.IsRTSPSerivceRunning())
    4.             {
    5.                 publisher_wrapper_.StopRtspService();
    6.                 btn_rtsp_service.Text = "启动RTSP服务";
    7.                 btn_rtsp_stream.Enabled = false;
    8.             }
    9.             else
    10.             {
    11.                 if(publisher_wrapper_.StartRtspService())
    12.                 {
    13.                     btn_rtsp_service.Text = "停止RTSP服务";
    14.                     btn_rtsp_stream.Enabled = true;
    15.                 }
    16.             }
    17.         }

    发布RTSP流：

    1.         private void btn_rtsp_stream_Click(object sender, EventArgs e)
    2.         {
    3.             if (publisher_wrapper_.IsRTSPPublisherRunning())
    4.             {
    5.                 publisher_wrapper_.StopRtspStream();
    6.                 btn_rtsp_stream.Text = "发布RTSP流";
    7.                 btn_get_rtsp_session_numbers.Enabled = false;
    8.                 btn_rtsp_service.Enabled = true;
    9.             }
    10.             else
    11.             {
    12.                 if (!publisher_wrapper_.IsPublisherHandleAvailable())
    13.                 {
    14.                     if (!OpenPublisherHandle())
    15.                     {
    16.                         return;
    17.                     }
    18.                 }
    19.  
    20.                 if (publisher_wrapper_.GetPublisherHandleCount() < 1)
    21.                 {
    22.                     SetCommonOptionToPublisherSDK();
    23.                 }
    24.  
    25.                 if (!publisher_wrapper_.StartRtspStream())
    26.                 {
    27.                     MessageBox.Show("调用StartRtspStream失败..");
    28.                     return;
    29.                 }
    30.  
    31.                 btn_rtsp_stream.Text = "停止RTSP流";
    32.                 btn_get_rtsp_session_numbers.Enabled = true;
    33.                 btn_rtsp_service.Enabled = false;
    34.             }
    35.         }

    获取RTSP会话数：

    1.         private void btn_get_rtsp_session_numbers_Click(object sender, EventArgs e)
    2.         {
    3.             if (publisher_wrapper_.IsRTSPPublisherRunning())
    4.             {
    5.                int session_numbers = publisher_wrapper_.GetRtspSessionNumbers();
    6.  
    7.                MessageBox.Show(session_numbers.ToString(), "当前RTSP连接会话数");
    8.             }
    9.         }

    本地录像：

    1.         private void btn_start_recorder_Click(object sender, EventArgs e)
    2.         {
    3.             if (!publisher_wrapper_.IsPublisherHandleAvailable())
    4.             {
    5.                 if (!OpenPublisherHandle())
    6.                 {
    7.                     return;
    8.                 }
    9.             }
    10.  
    11.             if (publisher_wrapper_.GetPublisherHandleCount() < 1)
    12.             {
    13.                 SetCommonOptionToPublisherSDK();
    14.             }
    15.  
    16.             if (!publisher_wrapper_.StartRecorder())
    17.             {
    18.                 MessageBox.Show("调用StartRecorder失败..");
    19.                 return;
    20.             }
    21.  
    22.             btn_start_recorder.Enabled = false;
    23.             btn_stop_recorder.Enabled = true;
    24.         }
    25.  
    26.         private void btn_stop_recorder_Click(object sender, EventArgs e)
    27.         {
    28.             if (!publisher_wrapper_.IsPublisherHandleAvailable())
    29.                 return;
    30.  
    31.             if (publisher_wrapper_.IsRecording())
    32.             {
    33.                 publisher_wrapper_.StopRecorder();
    34.  
    35.                 btn_start_recorder.Enabled = true;
    36.                 btn_stop_recorder.Enabled = false;
    37.             }
    38.         }

    暂停录像、恢复录像：

    1.         private void btn_pause_recorder_Click(object sender, EventArgs e)
    2.         {
    3.             if (!publisher_wrapper_.IsPublisherHandleAvailable())
    4.             {
    5.                 return;
    6.             }
    7.  
    8.             String btn_pause_rec_text = btn_pause_recorder.Text;
    9.  
    10.             if ("暂停录像" == btn_pause_rec_text)
    11.             {
    12.                 UInt32 ret = publisher_wrapper_.PauseRecorder(true);
    13.  
    14.                 if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
    15.                 {
    16.                     MessageBox.Show("暂停录像失败, 请重新尝试!");
    17.                     return;
    18.                 }
    19.                 else if (NTBaseCodeDefine.NT_ERC_OK == ret)
    20.                 {
    21.                     btn_pause_recorder.Text = "恢复录像";
    22.                 }
    23.             }
    24.             else
    25.             {
    26.                 UInt32 ret = publisher_wrapper_.PauseRecorder(false);
    27.                 if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
    28.                 {
    29.                     MessageBox.Show("恢复录像失败, 请重新尝试!");
    30.                     return;
    31.                 }
    32.                 else if (NTBaseCodeDefine.NT_ERC_OK == ret)
    33.                 {
    34.                     btn_pause_recorder.Text = "暂停录像";
    35.                 }
    36.             }
    37.         }

    推送RTMP：

    1.         private void btn_publish_rtmp_Click(object sender, EventArgs e)
    2.         {
    3.             if (!publisher_wrapper_.IsPublisherHandleAvailable())
    4.             {
    5.                 if (!OpenPublisherHandle())
    6.                 {
    7.                     return;
    8.                 }
    9.             }
    10.  
    11.             if (publisher_wrapper_.GetPublisherHandleCount() < 1)
    12.             {
    13.                 SetCommonOptionToPublisherSDK();
    14.             }
    15.  
    16.             String url = "rtmp://192.168.0.108:1935/hls/stream1";
    17.  
    18.             if (url.Length < 8)
    19.             {
    20.                 publisher_wrapper_.Close();
    21.  
    22.                 MessageBox.Show("请输入推送地址");
    23.                 return;
    24.             }
    25.  
    26.             if (!publisher_wrapper_.StartPublisher(url))
    27.             {
    28.                 MessageBox.Show("调用StartPublisher失败..");
    29.                 return;
    30.             }
    31.  
    32.             btn_publish_rtmp.Enabled = false;
    33.             btn_stop_publish_rtmp.Enabled = true;
    34.         }
    35.  
    36.         private void btn_stop_publish_rtmp_Click(object sender, EventArgs e)
    37.         {
    38.             if (!publisher_wrapper_.IsPublisherHandleAvailable())
    39.                 return;
    40.  
    41.             if (publisher_wrapper_.IsPublishing())
    42.             {
    43.                 publisher_wrapper_.StopPublisher();
    44.  
    45.                 btn_publish_rtmp.Enabled = true;
    46.                 btn_stop_publish_rtmp.Enabled = false;
    47.             }
    48.         }

    图层设计，目前设计两个图层，一个是原始YUV底层，另外一个是文字水印图层，如果需要动态去除文字水印，只要index为1的图层，enable设置为0即可。

    1.             NTSmartPublisherSDK.NT_PB_ClearLayersConfig(publisher_handle_, 0,
    2.                             0, IntPtr.Zero);
    3.  
    4.             if (video_option_ == (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
    5.             {
    6.                 NT_PB_ExternalVideoFrameLayerConfig external_layer_c0 = new NT_PB_ExternalVideoFrameLayerConfig();
    7.  
    8.                 external_layer_c0.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
    9.                 external_layer_c0.base_.index_ = 0;
    10.                 external_layer_c0.base_.enable_ = 1;
    11.                 external_layer_c0.base_.region_.x_ = 0;
    12.                 external_layer_c0.base_.region_.y_ = 0;
    13.                 external_layer_c0.base_.region_.width_ = video_width_;
    14.                 external_layer_c0.base_.region_.height_ = video_height_;
    15.  
    16.                 external_layer_c0.base_.offset_ = Marshal.OffsetOf(external_layer_c0.GetType(), "base_").ToInt32();
    17.                 external_layer_c0.base_.cb_size_ = (uint)Marshal.SizeOf(external_layer_c0);
    18.  
    19.                 IntPtr external_layer_conf0 = Marshal.AllocHGlobal(Marshal.SizeOf(external_layer_c0));
    20.  
    21.                 Marshal.StructureToPtr(external_layer_c0, external_layer_conf0, true);
    22.  
    23.                 UInt32 external_r0 = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,
    24.                                 external_layer_conf0, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME,
    25.                                 0, IntPtr.Zero);
    26.  
    27.                 Marshal.FreeHGlobal(external_layer_conf0);
    28.  
    29.                 //OSD水印层
    30.                 NT_PB_ExternalVideoFrameLayerConfig external_layer_c1 = new NT_PB_ExternalVideoFrameLayerConfig();
    31.  
    32.                 external_layer_c1.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
    33.                 external_layer_c1.base_.index_ = 1;
    34.                 external_layer_c1.base_.enable_ = 1;
    35.                 external_layer_c1.base_.region_.x_ = 0;
    36.                 external_layer_c1.base_.region_.y_ = 200;
    37.                 external_layer_c1.base_.region_.width_ = 200;
    38.                 external_layer_c1.base_.region_.height_ = 200;
    39.  
    40.                 external_layer_c1.base_.offset_ = Marshal.OffsetOf(external_layer_c1.GetType(), "base_").ToInt32();
    41.                 external_layer_c1.base_.cb_size_ = (uint)Marshal.SizeOf(external_layer_c1);
    42.  
    43.                 IntPtr external_layer_conf = Marshal.AllocHGlobal(Marshal.SizeOf(external_layer_c1));
    44.  
    45.                 Marshal.StructureToPtr(external_layer_c1, external_layer_conf, true);
    46.  
    47.                 UInt32 external_r1 = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,
    48.                                 external_layer_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME,
    49.                                 0, IntPtr.Zero);
    50.  
    51.                 Marshal.FreeHGlobal(external_layer_conf);
    52.                 //end
    53.             }

    总结

    RTSP拉流二次编码，整体逻辑不复杂，就是把数据回调后，二次处理，我们推送端设计的是图层的形式，所以，回调后的数据，直接作为第0层，文字水印作为第一层，如果需要图片水印，图片水印作为第三层即可。RTSP拉流二次编码，如果做到客户端尽量无感知，需要尽可能的压缩整体处理的延迟，确保从数据采集，到二次处理，到再次播放出来毫秒级，满足绝大多数场景下的技术需求。

  • 相关阅读:
    创建.NET程序Dump的几种姿势
    从0到0.01入门React | 008.精选 React 面试题
    MySQL事务
    深度学习第二章
    Xylan-MAL|木聚糖-马来酰亚胺|木聚糖-聚乙二醇-马来酰亚胺|马来酰亚胺-PEG-木聚糖
    代码运行出现了堆栈溢出错误及解决方法
    13 使用Vue + FormData + axios实现图片上传功能实战
    手撸一个springsecurity，了解一下security原理
    Linux 查找指令
    Java面试题
  • 原文地址：https://blog.csdn.net/renhui1112/article/details/134558618