• Android端如何实现拉取RTSP/RTMP流并回调YUV/RGB数据然后注入轻量级RTSP服务?


    技术背景

    我们在对接开发Android平台音视频模块的时候,遇到过这样的问题,厂商希望拉取到海康、大华等摄像机的RTSP流,然后解码后的YUV或RGB数据回给他们,他们做视频分析或处理后,再投递给轻量级RTSP服务模块或RTMP推送模块,实现处理后的数据,二次转发,本文以拉取RTSP流,解析后再注入轻量级RTSP服务为例,介绍下大概的技术实现。

    技术实现

    废话不多说,无图无真相,下图是测试的时候,Android终端拉取RTSP流,然后把YUV数据回调上来,又通过推送接口,注入到轻量级RTSP服务,然后Windows平台拉取轻量级RTSP的URL,整体下来,毫秒级延迟:

    先说拉取RTSP流,需要注意的是,如果不要播放的话,可以SetSurface()的时候,第二个参数设置null,如果不需要audio的话,直接SetMute设置1即可,因为需要回调YUV上来,那么设置下I420回调,如果需要RGB的,只要开RGB的回调即可。

    1. private boolean StartPlay()
    2. {
    3. if (!OpenPullHandle())
    4. return false;
    5. // 如果第二个参数设置为null,则播放纯音频
    6. libPlayer.SmartPlayerSetSurface(playerHandle, sSurfaceView);
    7. libPlayer.SmartPlayerSetRenderScaleMode(playerHandle, 1);
    8. // libPlayer.SmartPlayerSetExternalRender(playerHandle, new
    9. // RGBAExternalRender());
    10. libPlayer.SmartPlayerSetExternalRender(playerHandle, new
    11. I420ExternalRender());
    12. libPlayer.SmartPlayerSetFastStartup(playerHandle, isFastStartup ? 1 : 0);
    13. libPlayer.SmartPlayerSetAudioOutputType(playerHandle, 1);
    14. if (isMute) {
    15. libPlayer.SmartPlayerSetMute(playerHandle, isMute ? 1
    16. : 0);
    17. }
    18. if (isHardwareDecoder)
    19. {
    20. int isSupportH264HwDecoder = libPlayer
    21. .SetSmartPlayerVideoHWDecoder(playerHandle, 1);
    22. int isSupportHevcHwDecoder = libPlayer.SetSmartPlayerVideoHevcHWDecoder(playerHandle, 1);
    23. Log.i(TAG, "isSupportH264HwDecoder: " + isSupportH264HwDecoder + ", isSupportHevcHwDecoder: " + isSupportHevcHwDecoder);
    24. }
    25. libPlayer.SmartPlayerSetLowLatencyMode(playerHandle, isLowLatency ? 1
    26. : 0);
    27. libPlayer.SmartPlayerSetRotation(playerHandle, rotate_degrees);
    28. int iPlaybackRet = libPlayer
    29. .SmartPlayerStartPlay(playerHandle);
    30. if (iPlaybackRet != 0) {
    31. Log.e(TAG, "StartPlay failed!");
    32. if ( !isPulling && !isRecording && !isPushing && !isRTSPPublisherRunning)
    33. {
    34. releasePlayerHandle();
    35. }
    36. return false;
    37. }
    38. isPlaying = true;
    39. return true;
    40. }

    OpenPullHandle()对应的实现如下:

    1. /*
    2. * SmartRelayDemo.java
    3. * Created: daniusdk.com
    4. */
    5. private boolean OpenPullHandle()
    6. {
    7. //if (playerHandle != 0) {
    8. // return true;
    9. //}
    10. if(isPulling || isPlaying || isRecording)
    11. return true;
    12. //playbackUrl = "rtsp://xxxx";
    13. if (playbackUrl == null) {
    14. Log.e(TAG, "playback URL is null...");
    15. return false;
    16. }
    17. playerHandle = libPlayer.SmartPlayerOpen(myContext);
    18. if (playerHandle == 0) {
    19. Log.e(TAG, "playerHandle is nil..");
    20. return false;
    21. }
    22. libPlayer.SetSmartPlayerEventCallbackV2(playerHandle,
    23. new EventHandlePlayerV2());
    24. libPlayer.SmartPlayerSetBuffer(playerHandle, playBuffer);
    25. // set report download speed
    26. libPlayer.SmartPlayerSetReportDownloadSpeed(playerHandle, 1, 5);
    27. //设置RTSP超时时间
    28. int rtsp_timeout = 12;
    29. libPlayer.SmartPlayerSetRTSPTimeout(playerHandle, rtsp_timeout);
    30. //设置RTSP TCP/UDP模式自动切换
    31. int is_auto_switch_tcp_udp = 1;
    32. libPlayer.SmartPlayerSetRTSPAutoSwitchTcpUdp(playerHandle, is_auto_switch_tcp_udp);
    33. libPlayer.SmartPlayerSaveImageFlag(playerHandle, 1);
    34. // It only used when playback RTSP stream..
    35. //libPlayer.SmartPlayerSetRTSPTcpMode(playerHandle, 1);
    36. libPlayer.SmartPlayerSetUrl(playerHandle, playbackUrl);
    37. return true;
    38. }

    拉流端的Event回调状态如下,拉流端主要关注的是链接状态,还有实时下载速度:

    1. class EventHandlePlayerV2 implements NTSmartEventCallbackV2 {
    2. @Override
    3. public void onNTSmartEventCallbackV2(long handle, int id, long param1,
    4. long param2, String param3, String param4, Object param5) {
    5. //Log.i(TAG, "EventHandleV2: handle=" + handle + " id:" + id);
    6. String player_event = "";
    7. switch (id) {
    8. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STARTED:
    9. player_event = "开始..";
    10. break;
    11. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTING:
    12. player_event = "连接中..";
    13. break;
    14. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTION_FAILED:
    15. player_event = "连接失败..";
    16. break;
    17. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTED:
    18. player_event = "连接成功..";
    19. break;
    20. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_DISCONNECTED:
    21. player_event = "连接断开..";
    22. break;
    23. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STOP:
    24. player_event = "停止播放..";
    25. break;
    26. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RESOLUTION_INFO:
    27. player_event = "分辨率信息: width: " + param1 + ", height: " + param2;
    28. break;
    29. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_NO_MEDIADATA_RECEIVED:
    30. player_event = "收不到媒体数据,可能是url错误..";
    31. break;
    32. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_SWITCH_URL:
    33. player_event = "切换播放URL..";
    34. break;
    35. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CAPTURE_IMAGE:
    36. player_event = "快照: " + param1 + " 路径:" + param3;
    37. if (param1 == 0) {
    38. player_event = player_event + ", 截取快照成功";
    39. } else {
    40. player_event = player_event + ", 截取快照失败";
    41. }
    42. break;
    43. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RECORDER_START_NEW_FILE:
    44. player_event = "[record]开始一个新的录像文件 : " + param3;
    45. break;
    46. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_ONE_RECORDER_FILE_FINISHED:
    47. player_event = "[record]已生成一个录像文件 : " + param3;
    48. break;
    49. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_START_BUFFERING:
    50. Log.i(TAG, "Start Buffering");
    51. break;
    52. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_BUFFERING:
    53. Log.i(TAG, "Buffering:" + param1 + "%");
    54. break;
    55. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STOP_BUFFERING:
    56. Log.i(TAG, "Stop Buffering");
    57. break;
    58. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_DOWNLOAD_SPEED:
    59. player_event = "download_speed:" + param1 + "Byte/s" + ", "
    60. + (param1 * 8 / 1000) + "kbps" + ", " + (param1 / 1024)
    61. + "KB/s";
    62. break;
    63. case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RTSP_STATUS_CODE:
    64. Log.e(TAG, "RTSP error code received, please make sure username/password is correct, error code:" + param1);
    65. player_event = "RTSP error code:" + param1;
    66. break;
    67. }
    68. }
    69. }

    下一步,是启动RTSP服务:

    1. //启动/停止RTSP服务
    2. class ButtonRtspServiceListener implements OnClickListener {
    3. public void onClick(View v) {
    4. if (isRTSPServiceRunning) {
    5. stopRtspService();
    6. btnRtspService.setText("启动RTSP服务");
    7. btnRtspPublisher.setEnabled(false);
    8. isRTSPServiceRunning = false;
    9. return;
    10. }
    11. if(!OpenPushHandle())
    12. {
    13. return;
    14. }
    15. Log.i(TAG, "onClick start rtsp service..");
    16. rtsp_handle_ = libPublisher.OpenRtspServer(0);
    17. if (rtsp_handle_ == 0) {
    18. Log.e(TAG, "创建rtsp server实例失败! 请检查SDK有效性");
    19. } else {
    20. int port = 8554;
    21. if (libPublisher.SetRtspServerPort(rtsp_handle_, port) != 0) {
    22. libPublisher.CloseRtspServer(rtsp_handle_);
    23. rtsp_handle_ = 0;
    24. Log.e(TAG, "创建rtsp server端口失败! 请检查端口是否重复或者端口不在范围内!");
    25. }
    26. //String user_name = "admin";
    27. //String password = "12345";
    28. //libPublisher.SetRtspServerUserNamePassword(rtsp_handle_, user_name, password);
    29. if (libPublisher.StartRtspServer(rtsp_handle_, 0) == 0) {
    30. Log.i(TAG, "启动rtsp server 成功!");
    31. } else {
    32. libPublisher.CloseRtspServer(rtsp_handle_);
    33. rtsp_handle_ = 0;
    34. Log.e(TAG, "启动rtsp server失败! 请检查设置的端口是否被占用!");
    35. }
    36. btnRtspService.setText("停止RTSP服务");
    37. btnRtspPublisher.setEnabled(true);
    38. isRTSPServiceRunning = true;
    39. }
    40. }
    41. }

    如果需要停止服务,对应实现如下:

    1. //停止RTSP服务
    2. private void stopRtspService() {
    3. if(!isRTSPServiceRunning)
    4. return;
    5. if (libPublisher != null && rtsp_handle_ != 0) {
    6. libPublisher.StopRtspServer(rtsp_handle_);
    7. libPublisher.CloseRtspServer(rtsp_handle_);
    8. rtsp_handle_ = 0;
    9. }
    10. if(!isPushing)
    11. {
    12. releasePublisherHandle();
    13. }
    14. isRTSPServiceRunning = false;
    15. }

    发布、停止发布RTSP流:

    1. private boolean StartRtspStream()
    2. {
    3. if (isRTSPPublisherRunning)
    4. return false;
    5. String rtsp_stream_name = "stream1";
    6. libPublisher.SetRtspStreamName(publisherHandle, rtsp_stream_name);
    7. libPublisher.ClearRtspStreamServer(publisherHandle);
    8. libPublisher.AddRtspStreamServer(publisherHandle, rtsp_handle_, 0);
    9. if (libPublisher.StartRtspStream(publisherHandle, 0) != 0)
    10. {
    11. Log.e(TAG, "调用发布rtsp流接口失败!");
    12. if (!isPushing)
    13. {
    14. libPublisher.SmartPublisherClose(publisherHandle);
    15. publisherHandle = 0;
    16. }
    17. return false;
    18. }
    19. isRTSPPublisherRunning = true;
    20. return true;
    21. }
    22. //停止发布RTSP流
    23. private void stopRtspPublisher()
    24. {
    25. if(!isRTSPPublisherRunning)
    26. return;
    27. isRTSPPublisherRunning = false;
    28. if (null == libPublisher || 0 == publisherHandle)
    29. return;
    30. libPublisher.StopRtspStream(publisherHandle);
    31. if (!isPushing && !isRTSPServiceRunning)
    32. {
    33. releasePublisherHandle();
    34. }
    35. }

    因为处理后YUV或RGB数据需要重新编码,这时候需要推送端,设置下编码参数:

    1. private boolean OpenPushHandle() {
    2. if(publisherHandle != 0)
    3. {
    4. return true;
    5. }
    6. publisherHandle = libPublisher.SmartPublisherOpen(myContext, audio_opt, video_opt,
    7. videoWidth, videoHeight);
    8. if (publisherHandle == 0) {
    9. Log.e(TAG, "sdk open failed!");
    10. return false;
    11. }
    12. Log.i(TAG, "publisherHandle=" + publisherHandle);
    13. int fps = 20;
    14. int gop = fps * 1;
    15. int videoEncodeType = 1; //1: h.264硬编码 2: H.265硬编码
    16. if(videoEncodeType == 1) {
    17. int h264HWKbps = setHardwareEncoderKbps(true, videoWidth, videoHeight);
    18. h264HWKbps = h264HWKbps*fps/25;
    19. Log.i(TAG, "h264HWKbps: " + h264HWKbps);
    20. int isSupportH264HWEncoder = libPublisher
    21. .SetSmartPublisherVideoHWEncoder(publisherHandle, h264HWKbps);
    22. if (isSupportH264HWEncoder == 0) {
    23. libPublisher.SetNativeMediaNDK(publisherHandle, 0);
    24. libPublisher.SetVideoHWEncoderBitrateMode(publisherHandle, 1); // 0:CQ, 1:VBR, 2:CBR
    25. libPublisher.SetVideoHWEncoderQuality(publisherHandle, 39);
    26. libPublisher.SetAVCHWEncoderProfile(publisherHandle, 0x08); // 0x01: Baseline, 0x02: Main, 0x08: High
    27. // libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x200); // Level 3.1
    28. // libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x400); // Level 3.2
    29. // libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x800); // Level 4
    30. libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x1000); // Level 4.1 多数情况下,这个够用了
    31. //libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x2000); // Level 4.2
    32. // libPublisher.SetVideoHWEncoderMaxBitrate(publisherHandle, ((long)h264HWKbps)*1300);
    33. Log.i(TAG, "Great, it supports h.264 hardware encoder!");
    34. }
    35. }
    36. else if (videoEncodeType == 2) {
    37. int hevcHWKbps = setHardwareEncoderKbps(false, videoWidth, videoHeight);
    38. hevcHWKbps = hevcHWKbps*fps/25;
    39. Log.i(TAG, "hevcHWKbps: " + hevcHWKbps);
    40. int isSupportHevcHWEncoder = libPublisher
    41. .SetSmartPublisherVideoHevcHWEncoder(publisherHandle, hevcHWKbps);
    42. if (isSupportHevcHWEncoder == 0) {
    43. libPublisher.SetNativeMediaNDK(publisherHandle, 0);
    44. libPublisher.SetVideoHWEncoderBitrateMode(publisherHandle, 0); // 0:CQ, 1:VBR, 2:CBR
    45. libPublisher.SetVideoHWEncoderQuality(publisherHandle, 39);
    46. // libPublisher.SetVideoHWEncoderMaxBitrate(publisherHandle, ((long)hevcHWKbps)*1200);
    47. Log.i(TAG, "Great, it supports hevc hardware encoder!");
    48. }
    49. }
    50. libPublisher.SetSmartPublisherEventCallbackV2(publisherHandle, new EventHandlePublisherV2());
    51. libPublisher.SmartPublisherSetGopInterval(publisherHandle, gop);
    52. libPublisher.SmartPublisherSetFPS(publisherHandle, fps);
    53. return true;
    54. }

    I420ExternalRender实现如下,这里可以拿到拉流的RTSP的YUV数据,然后处理后,可以调用推送端的PostLayerImageI420ByteBuffer()投递到轻量级RTSP服务或RTMP推送端编码发送出去。

    1. class I420ExternalRender implements NTExternalRender {
    2. // public static final int NT_FRAME_FORMAT_RGBA = 1;
    3. // public static final int NT_FRAME_FORMAT_ABGR = 2;
    4. // public static final int NT_FRAME_FORMAT_I420 = 3;
    5. private int width_ = 0;
    6. private int height_ = 0;
    7. private int y_row_bytes_ = 0;
    8. private int u_row_bytes_ = 0;
    9. private int v_row_bytes_ = 0;
    10. private ByteBuffer y_buffer_ = null;
    11. private ByteBuffer u_buffer_ = null;
    12. private ByteBuffer v_buffer_ = null;
    13. @Override
    14. public int getNTFrameFormat() {
    15. Log.i(TAG, "I420ExternalRender::getNTFrameFormat return "
    16. + NT_FRAME_FORMAT_I420);
    17. return NT_FRAME_FORMAT_I420;
    18. }
    19. @Override
    20. public void onNTFrameSizeChanged(int width, int height) {
    21. width_ = width;
    22. height_ = height;
    23. y_row_bytes_ = (width_ + 15) & (~15);
    24. u_row_bytes_ = ((width_ + 1) / 2 + 15) & (~15);
    25. v_row_bytes_ = ((width_ + 1) / 2 + 15) & (~15);
    26. y_buffer_ = ByteBuffer.allocateDirect(y_row_bytes_ * height_);
    27. u_buffer_ = ByteBuffer.allocateDirect(u_row_bytes_
    28. * ((height_ + 1) / 2));
    29. v_buffer_ = ByteBuffer.allocateDirect(v_row_bytes_
    30. * ((height_ + 1) / 2));
    31. Log.i(TAG, "I420ExternalRender::onNTFrameSizeChanged width_="
    32. + width_ + " height_=" + height_ + " y_row_bytes_="
    33. + y_row_bytes_ + " u_row_bytes_=" + u_row_bytes_
    34. + " v_row_bytes_=" + v_row_bytes_);
    35. }
    36. @Override
    37. public ByteBuffer getNTPlaneByteBuffer(int index) {
    38. if (index == 0) {
    39. return y_buffer_;
    40. } else if (index == 1) {
    41. return u_buffer_;
    42. } else if (index == 2) {
    43. return v_buffer_;
    44. } else {
    45. Log.e(TAG, "I420ExternalRender::getNTPlaneByteBuffer index error:" + index);
    46. return null;
    47. }
    48. }
    49. @Override
    50. public int getNTPlanePerRowBytes(int index) {
    51. if (index == 0) {
    52. return y_row_bytes_;
    53. } else if (index == 1) {
    54. return u_row_bytes_;
    55. } else if (index == 2) {
    56. return v_row_bytes_;
    57. } else {
    58. Log.e(TAG, "I420ExternalRender::getNTPlanePerRowBytes index error:" + index);
    59. return 0;
    60. }
    61. }
    62. public void onNTRenderFrame(int width, int height, long timestamp)
    63. {
    64. if ( y_buffer_ == null )
    65. return;
    66. if ( u_buffer_ == null )
    67. return;
    68. if ( v_buffer_ == null )
    69. return;
    70. y_buffer_.rewind();
    71. u_buffer_.rewind();
    72. v_buffer_.rewind();
    73. if( isPushing || isRTSPPublisherRunning )
    74. {
    75. libPublisher.PostLayerImageI420ByteBuffer(publisherHandle, 0, 0, 0,
    76. y_buffer_, 0, y_row_bytes_,
    77. u_buffer_, 0, u_row_bytes_,
    78. v_buffer_, 0, v_row_bytes_,
    79. width_, height_, 0, 0,
    80. 960, 540, 0,0);
    81. }
    82. }
    83. }

    如果轻量级服务正常启动,会把rtsp的url回调上来:

    1. class EventHandlePublisherV2 implements NTSmartEventCallbackV2 {
    2. @Override
    3. public void onNTSmartEventCallbackV2(long handle, int id, long param1, long param2, String param3, String param4, Object param5) {
    4. Log.i(TAG, "EventHandlePublisherV2: handle=" + handle + " id:" + id);
    5. String publisher_event = "";
    6. switch (id) {
    7. ....
    8. case NTSmartEventID.EVENT_DANIULIVE_ERC_PUBLISHER_RTSP_URL:
    9. publisher_event = "RTSP服务URL: " + param3;
    10. break;
    11. }
    12. }
    13. }

    技术总结

    以上是大概的流程,从RTSP拉流到数据处理后,重新塞给轻量级RTSP服务,然后播放端再从轻量级RTSP服务端拉流,如果针对YUV或RGB算法处理延迟不大的话,整体延迟可轻松达到毫秒级,满足大多数场景的技术诉求。

  • 相关阅读:
    【LeetCode】每日一题 2023_11_14 阈值距离内邻居最少的城市(Floyd 最短路算法)
    JavaScript 中 toString 的奇妙使用
    2022学习进阶之路:高并发+性能优化+Spring boot等大型项目实战
    C++核心编程--类篇
    10款实用的市场分析工具,你知道几个?
    Spring 源码分析-简单示例带你了解BeanFactoryPostProcessor
    Spring面试题(一)
    【taro react】---- 微信小程序 通过 Jenkins 实现自动化部署
    vue-devTools Chrome安装配置
    C Primer Plus(6) 中文版 第13章 文件输入/输出 13.2 标准I/O
  • 原文地址:https://blog.csdn.net/renhui1112/article/details/133135936