• Windows平台Unity下实现camera场景推送RTMP|轻量级RTSP服务|实时录像

    技术背景

    我们在对接Unity平台camera场景采集的时候，除了常规的RTMP推送、录像外，还有一些开发者，需要能实现轻量级RTSP服务，对外提供个拉流的RTSP URL。

    目前我们在Windows平台Unity下数据源可采集到以下部分：

    • 采集Unity camera场景；
    • 采集摄像头；
    • 采集屏幕；
    • 采集Unity声音；
    • 采集麦克风；
    • 采集扬声器；
    • Unity PCM混音；

    对外提供的技术能力有：

    • RTMP直播推送；
    • 轻量级RTSP服务；
    • 实时录像、暂停|恢复录像；
    • 实时预览。

    以下录制下来的MP4文件是采集Unity camera场景，音频是unity声音。

    Unity平台实现camera场景实时录像

    技术实现

    实际上，在实现Unity平台音视频能力之前，我们原生模块已经有非常成熟的技术积累，Unity下还是调用的原生的推送模块，不同的是，数据源需要采集Unity的audio、video，然后高效的投递到底层模块，底层模块负责编码打包，并投递到RTMP或RTSP服务。

    先说支持的音视频类型：

    1.     public void SelVideoPushType(int type)
    2.     {
    3.         switch (type)
    4.         {
    5.             case 0:
    6.                 video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER;    //采集Unity窗体
    7.                 break;
    8.             case 1:
    9.                 video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA;   //采集摄像头
    10.                 break;
    11.             case 2:
    12.                 video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_SCREEN;   //采集屏幕
    13.                 break;
    14.             case 3:
    15.                 video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_NO_VIDEO; //不采集视频
    16.                 break;
    17.         }
    18.  
    19.         Debug.Log("SelVideoPushType type: " + type + " video_push_type: " + video_push_type_);
    20.     }
    21.  
    22.     public void SelAudioPushType(int type)
    23.     {
    24.         switch (type)
    25.         {
    26.             case 0:
    27.                 audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA;    //采集Unity声音
    28.                 break;
    29.             case 1:
    30.                 audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_CAPTURE_MIC;  //采集麦克风
    31.                 break;
    32.             case 2:
    33.                 audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_CAPTURE_SPEAKER;  //采集扬声器
    34.                 break;
    35.             case 3:
    36.                 audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER;  //两路Unity AudioClip混音测试
    37.                 break;
    38.             case 4:
    39.                 audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_NO_AUDIO;   //不采集音频
    40.                 break;
    41.         }
    42.  
    43.         Debug.Log("SelAudioPushType type: " + type + " audio_push_type: " + audio_push_type_);
    44.     }

    采集音视频数据：

    1.     private void StartCaptureAvData()
    2.     {
    3.         if (audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA
    4.             || audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER)
    5.         {
    6.             PostUnityAudioClipData();
    7.         }
    8.  
    9.         textures_poll_ = new TexturesPool();
    10.         post_image_worker_ = new PostImageWorker(textures_poll_, publisher_wrapper_);
    11.         post_worker_thread_ = new Thread(post_image_worker_.run);
    12.         post_worker_thread_.Start();
    13.     }
    14.  
    15.     private void StopCaptureAvData()
    16.     {
    17.         if (post_image_worker_ != null)
    18.         {
    19.             post_image_worker_.requestStop();
    20.             post_image_worker_ = null;
    21.         }
    22.  
    23.         if (post_worker_thread_ != null)
    24.         {
    25.             post_worker_thread_.Join();
    26.             post_worker_thread_ = null;
    27.         }
    28.  
    29.         if (textures_poll_ != null)
    30.         {
    31.             textures_poll_.clear();
    32.             textures_poll_ = null;
    33.         }
    34.  
    35.         StopAudioSource();
    36.     }

    RTMP推送：

    1.     public void btn_start_rtmp_pusher_Click()
    2.     {
    3.         if (publisher_wrapper_.IsPushingRtmp())
    4.         {
    5.             StopPushRTMP();
    6.             btn_rtmp_pusher_.GetComponentInChildren().text = "推送RTMP";
    7.  
    8.             return;
    9.         }
    10.  
    11.         String url = rtmp_pusher_url_.text;
    12.  
    13.         if (url.Length < 8)
    14.         {
    15.             publisher_wrapper_.Close();
    16.  
    17.             Debug.LogError("请输入RTMP推送地址");
    18.             return;
    19.         }
    20.  
    21.         if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
    22.         {
    23.             publisher_wrapper_.SetVideoPushType(video_push_type_);
    24.             publisher_wrapper_.SetAudioPushType(audio_push_type_);
    25.         }
    26.  
    27.         if (!publisher_wrapper_.StartRtmpPusher(url))
    28.         {
    29.             Debug.LogError("调用StartPublisher失败..");
    30.             return;
    31.         }
    32.  
    33.         btn_rtmp_pusher_.GetComponentInChildren().text = "停止推送";
    34.  
    35.         if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
    36.         {
    37.             StartCaptureAvData();
    38.             coroutine_ = StartCoroutine(OnPostVideo());
    39.         }
    40.     }

    轻量级RTSP服务相关调用：

    1.     public void btn_rtsp_service_Click()
    2.     {
    3.         if (publisher_wrapper_.IsRtspServiceRunning())
    4.         {
    5.             publisher_wrapper_.StopRtspService();
    6.             btn_rtsp_service_.GetComponentInChildren().text = "启动RTSP服务";
    7.  
    8.             btn_rtsp_publisher_.interactable = false;
    9.             return;
    10.         }
    11.  
    12.         if (!publisher_wrapper_.StartRtspService())
    13.         {
    14.             Debug.LogError("调用StartRtspService失败..");
    15.             return;
    16.         }
    17.  
    18.         btn_rtsp_publisher_.interactable = true;
    19.  
    20.         btn_rtsp_service_.GetComponentInChildren().text = "停止RTSP服务";
    21.     }
    22.  
    23.     public void btn_rtsp_publisher_Click()
    24.     {
    25.         if (publisher_wrapper_.IsRtspPublisherRunning())
    26.         {
    27.             publisher_wrapper_.StopRtspStream();
    28.  
    29.             if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
    30.             {
    31.                 StopCaptureAvData();
    32.  
    33.                 if (coroutine_ != null)
    34.                 {
    35.                     StopCoroutine(coroutine_);
    36.                     coroutine_ = null;
    37.                 }
    38.             }
    39.  
    40.             btn_rtsp_service_.interactable = true;
    41.  
    42.             btn_rtsp_publisher_.GetComponentInChildren().text = "发布RTSP";
    43.         }
    44.         else
    45.         {
    46.             if (!publisher_wrapper_.IsRtspServiceRunning())
    47.             {
    48.                 Debug.LogError("RTSP service is not running..");
    49.                 return;
    50.             }
    51.  
    52.  
    53.             if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
    54.             {
    55.                 publisher_wrapper_.SetVideoPushType(video_push_type_);
    56.                 publisher_wrapper_.SetAudioPushType(audio_push_type_);
    57.             }
    58.  
    59.             publisher_wrapper_.StartRtspStream();
    60.  
    61.             if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
    62.             {
    63.                 StartCaptureAvData();
    64.                 coroutine_ = StartCoroutine(OnPostVideo());
    65.             }
    66.  
    67.             btn_rtsp_publisher_.GetComponentInChildren().text = "停止RTSP";
    68.  
    69.             btn_rtsp_service_.interactable = false;
    70.         }
    71.     }
    72.  
    73.     public void btn_get_rtsp_session_numbers_Click()
    74.     {
    75.         if (publisher_wrapper_.IsRtspServiceRunning())
    76.         {
    77.             btn_get_rtsp_session_numbers_.GetComponentInChildren().text = "RTSP会话数:" + publisher_wrapper_.GetRtspSessionNumbers();
    78.         }
    79.     }

    实时录像调用：

    1.     public void btn_record_Click()
    2.     {
    3.         if (publisher_wrapper_.IsRecording())
    4.         {
    5.             StopRecord();
    6.             btn_record_.GetComponentInChildren().text = "开始录像";
    7.  
    8.             return;
    9.         }
    10.  
    11.         if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
    12.         {
    13.             publisher_wrapper_.SetVideoPushType(video_push_type_);
    14.             publisher_wrapper_.SetAudioPushType(audio_push_type_);
    15.         }
    16.  
    17.         if (!publisher_wrapper_.StartRecorder())
    18.         {
    19.             Debug.LogError("调用StartRecorder失败..");
    20.             return;
    21.         }
    22.  
    23.         btn_record_.GetComponentInChildren().text = "停止录像";
    24.  
    25.         if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
    26.         {
    27.             StartCaptureAvData();
    28.             coroutine_ = StartCoroutine(OnPostVideo());
    29.         }
    30.     }
    31.  
    32.     public void StopRecord()
    33.     {
    34.         if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
    35.         {
    36.             StopCaptureAvData();
    37.  
    38.             if (coroutine_ != null)
    39.             {
    40.                 StopCoroutine(coroutine_);
    41.                 coroutine_ = null;
    42.             }
    43.         }
    44.  
    45.         publisher_wrapper_.StopRecorder();
    46.     }
    47.  
    48.     private void btn_pause_record_Click()
    49.     {
    50.         if (!publisher_wrapper_.IsPublisherHandleAvailable())
    51.         {
    52.             return;
    53.         }
    54.  
    55.         String btn_pause_rec_text = btn_pause_record_.GetComponentInChildren().text;
    56.  
    57.         if ("暂停录像" == btn_pause_rec_text)
    58.         {
    59.             UInt32 ret = publisher_wrapper_.PauseRecorder(true);
    60.  
    61.             if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
    62.             {
    63.                 Debug.LogError("暂停录像失败, 请重新尝试!");
    64.                 return;
    65.             }
    66.             else if (NTBaseCodeDefine.NT_ERC_OK == ret)
    67.             {
    68.                 btn_pause_record_.GetComponentInChildren().text = "恢复录像";
    69.             }
    70.         }
    71.         else
    72.         {
    73.             UInt32 ret = publisher_wrapper_.PauseRecorder(false);
    74.             if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
    75.             {
    76.                 Debug.LogError("恢复录像失败, 请重新尝试!");
    77.                 return;
    78.             }
    79.             else if (NTBaseCodeDefine.NT_ERC_OK == ret)
    80.             {
    81.                 btn_pause_record_.GetComponentInChildren().text = "暂停录像";
    82.             }
    83.         }
    84.     }

    实时预览相关：

    1.     public void btn_preview_Click()
    2.     {
    3.         if (btn_preview_.GetComponentInChildren().text.Equals("本地预览"))
    4.         {
    5.             if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
    6.             {
    7.                 publisher_wrapper_.SetVideoPushType(video_push_type_);
    8.             }
    9.  
    10.             if (publisher_wrapper_.StartPreview())
    11.             {
    12.                 btn_preview_.GetComponentInChildren().text = "停止预览";
    13.             }
    14.  
    15.             if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
    16.             {
    17.                 StartCaptureAvData();
    18.                 coroutine_ = StartCoroutine(OnPostVideo());
    19.             }
    20.         }
    21.         else
    22.         {
    23.             if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
    24.             {
    25.                 StopCaptureAvData();
    26.  
    27.                 if (coroutine_ != null)
    28.                 {
    29.                     StopCoroutine(coroutine_);
    30.                     coroutine_ = null;
    31.                 }
    32.             }
    33.  
    34.             publisher_wrapper_.StopPreview();
    35.             btn_preview_.GetComponentInChildren().text = "本地预览";
    36.         }
    37.     }

    总结

    Unity平台下RTMP推送、录像、轻量级RTSP服务，在虚拟仿真、医疗、教育等场景下，应用非常广泛。要实现低延迟，除了需要高效率的音视频数据采集，编码和数据投递外，还需要好的直播播放器支持。配合我们的SmartPlayer，可轻松实现毫秒级体验，满足绝大多数应用场景技术诉求。

  • 相关阅读:
    MySQL索引优化，设计原则 及 trace 详解（思维导图）
    C++中的一些困惑（长期更新中）
    Ps：变换
    亚商投资顾问 早餐FM/1121 2022年卡塔尔世界杯开幕
    MySQL(三)：表的增删改查(进阶)
    打造类ChatGPT服务，本地部署大语言模型（LLM），如何远程访问？
    【转】数据库索引详细介绍
    MPLS --- 多协议标签交换技术
    SpringBoot项目创建及其核心设计思想
    【LLM教程】为什么做大语言模型fine tuning时，要将 drop_last_batch设置为True？
  • 原文地址：https://blog.csdn.net/renhui1112/article/details/134554456