我们在对接开发Android平台音视频模块的时候,遇到过这样的问题,厂商希望拉取到海康、大华等摄像机的RTSP流,然后解码后的YUV或RGB数据回给他们,他们做视频分析或处理后,再投递给轻量级RTSP服务模块或RTMP推送模块,实现处理后的数据,二次转发,本文以拉取RTSP流,解析后再注入轻量级RTSP服务为例,介绍下大概的技术实现。
废话不多说,无图无真相,下图是测试的时候,Android终端拉取RTSP流,然后把YUV数据回调上来,又通过推送接口,注入到轻量级RTSP服务,然后Windows平台拉取轻量级RTSP的URL,整体下来,毫秒级延迟:
先说拉取RTSP流,需要注意的是,如果不要播放的话,可以SetSurface()的时候,第二个参数设置null,如果不需要audio的话,直接SetMute设置1即可,因为需要回调YUV上来,那么设置下I420回调,如果需要RGB的,只要开RGB的回调即可。
- private boolean StartPlay()
- {
- if (!OpenPullHandle())
- return false;
-
- // 如果第二个参数设置为null,则播放纯音频
- libPlayer.SmartPlayerSetSurface(playerHandle, sSurfaceView);
-
- libPlayer.SmartPlayerSetRenderScaleMode(playerHandle, 1);
-
- // libPlayer.SmartPlayerSetExternalRender(playerHandle, new
- // RGBAExternalRender());
- libPlayer.SmartPlayerSetExternalRender(playerHandle, new
- I420ExternalRender());
-
- libPlayer.SmartPlayerSetFastStartup(playerHandle, isFastStartup ? 1 : 0);
-
- libPlayer.SmartPlayerSetAudioOutputType(playerHandle, 1);
-
- if (isMute) {
- libPlayer.SmartPlayerSetMute(playerHandle, isMute ? 1
- : 0);
- }
-
- if (isHardwareDecoder)
- {
- int isSupportH264HwDecoder = libPlayer
- .SetSmartPlayerVideoHWDecoder(playerHandle, 1);
-
- int isSupportHevcHwDecoder = libPlayer.SetSmartPlayerVideoHevcHWDecoder(playerHandle, 1);
-
- Log.i(TAG, "isSupportH264HwDecoder: " + isSupportH264HwDecoder + ", isSupportHevcHwDecoder: " + isSupportHevcHwDecoder);
- }
-
- libPlayer.SmartPlayerSetLowLatencyMode(playerHandle, isLowLatency ? 1
- : 0);
-
- libPlayer.SmartPlayerSetRotation(playerHandle, rotate_degrees);
-
- int iPlaybackRet = libPlayer
- .SmartPlayerStartPlay(playerHandle);
-
- if (iPlaybackRet != 0) {
- Log.e(TAG, "StartPlay failed!");
-
- if ( !isPulling && !isRecording && !isPushing && !isRTSPPublisherRunning)
- {
- releasePlayerHandle();
- }
-
- return false;
- }
-
- isPlaying = true;
- return true;
- }
OpenPullHandle()对应的实现如下:
- /*
- * SmartRelayDemo.java
- * Created: daniusdk.com
- */
- private boolean OpenPullHandle()
- {
- //if (playerHandle != 0) {
- // return true;
- //}
-
- if(isPulling || isPlaying || isRecording)
- return true;
-
- //playbackUrl = "rtsp://xxxx";
-
- if (playbackUrl == null) {
- Log.e(TAG, "playback URL is null...");
- return false;
- }
-
- playerHandle = libPlayer.SmartPlayerOpen(myContext);
-
- if (playerHandle == 0) {
- Log.e(TAG, "playerHandle is nil..");
- return false;
- }
-
- libPlayer.SetSmartPlayerEventCallbackV2(playerHandle,
- new EventHandlePlayerV2());
-
- libPlayer.SmartPlayerSetBuffer(playerHandle, playBuffer);
-
- // set report download speed
- libPlayer.SmartPlayerSetReportDownloadSpeed(playerHandle, 1, 5);
-
- //设置RTSP超时时间
- int rtsp_timeout = 12;
- libPlayer.SmartPlayerSetRTSPTimeout(playerHandle, rtsp_timeout);
-
- //设置RTSP TCP/UDP模式自动切换
- int is_auto_switch_tcp_udp = 1;
- libPlayer.SmartPlayerSetRTSPAutoSwitchTcpUdp(playerHandle, is_auto_switch_tcp_udp);
-
- libPlayer.SmartPlayerSaveImageFlag(playerHandle, 1);
-
- // It only used when playback RTSP stream..
- //libPlayer.SmartPlayerSetRTSPTcpMode(playerHandle, 1);
-
- libPlayer.SmartPlayerSetUrl(playerHandle, playbackUrl);
-
- return true;
- }
拉流端的Event回调状态如下,拉流端主要关注的是链接状态,还有实时下载速度:
- class EventHandlePlayerV2 implements NTSmartEventCallbackV2 {
- @Override
- public void onNTSmartEventCallbackV2(long handle, int id, long param1,
- long param2, String param3, String param4, Object param5) {
-
- //Log.i(TAG, "EventHandleV2: handle=" + handle + " id:" + id);
-
- String player_event = "";
-
- switch (id) {
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STARTED:
- player_event = "开始..";
- break;
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTING:
- player_event = "连接中..";
- break;
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTION_FAILED:
- player_event = "连接失败..";
- break;
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTED:
- player_event = "连接成功..";
- break;
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_DISCONNECTED:
- player_event = "连接断开..";
- break;
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STOP:
- player_event = "停止播放..";
- break;
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RESOLUTION_INFO:
- player_event = "分辨率信息: width: " + param1 + ", height: " + param2;
- break;
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_NO_MEDIADATA_RECEIVED:
- player_event = "收不到媒体数据,可能是url错误..";
- break;
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_SWITCH_URL:
- player_event = "切换播放URL..";
- break;
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CAPTURE_IMAGE:
- player_event = "快照: " + param1 + " 路径:" + param3;
-
- if (param1 == 0) {
- player_event = player_event + ", 截取快照成功";
- } else {
- player_event = player_event + ", 截取快照失败";
- }
- break;
-
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RECORDER_START_NEW_FILE:
- player_event = "[record]开始一个新的录像文件 : " + param3;
- break;
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_ONE_RECORDER_FILE_FINISHED:
- player_event = "[record]已生成一个录像文件 : " + param3;
- break;
-
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_START_BUFFERING:
- Log.i(TAG, "Start Buffering");
- break;
-
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_BUFFERING:
- Log.i(TAG, "Buffering:" + param1 + "%");
- break;
-
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STOP_BUFFERING:
- Log.i(TAG, "Stop Buffering");
- break;
-
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_DOWNLOAD_SPEED:
- player_event = "download_speed:" + param1 + "Byte/s" + ", "
- + (param1 * 8 / 1000) + "kbps" + ", " + (param1 / 1024)
- + "KB/s";
- break;
-
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RTSP_STATUS_CODE:
- Log.e(TAG, "RTSP error code received, please make sure username/password is correct, error code:" + param1);
- player_event = "RTSP error code:" + param1;
- break;
- }
- }
- }
下一步,是启动RTSP服务:
- //启动/停止RTSP服务
- class ButtonRtspServiceListener implements OnClickListener {
- public void onClick(View v) {
- if (isRTSPServiceRunning) {
- stopRtspService();
-
- btnRtspService.setText("启动RTSP服务");
- btnRtspPublisher.setEnabled(false);
-
- isRTSPServiceRunning = false;
- return;
- }
-
- if(!OpenPushHandle())
- {
- return;
- }
-
- Log.i(TAG, "onClick start rtsp service..");
-
- rtsp_handle_ = libPublisher.OpenRtspServer(0);
-
- if (rtsp_handle_ == 0) {
- Log.e(TAG, "创建rtsp server实例失败! 请检查SDK有效性");
- } else {
- int port = 8554;
- if (libPublisher.SetRtspServerPort(rtsp_handle_, port) != 0) {
- libPublisher.CloseRtspServer(rtsp_handle_);
- rtsp_handle_ = 0;
- Log.e(TAG, "创建rtsp server端口失败! 请检查端口是否重复或者端口不在范围内!");
- }
-
- //String user_name = "admin";
- //String password = "12345";
- //libPublisher.SetRtspServerUserNamePassword(rtsp_handle_, user_name, password);
-
- if (libPublisher.StartRtspServer(rtsp_handle_, 0) == 0) {
- Log.i(TAG, "启动rtsp server 成功!");
- } else {
- libPublisher.CloseRtspServer(rtsp_handle_);
- rtsp_handle_ = 0;
- Log.e(TAG, "启动rtsp server失败! 请检查设置的端口是否被占用!");
- }
-
- btnRtspService.setText("停止RTSP服务");
- btnRtspPublisher.setEnabled(true);
-
- isRTSPServiceRunning = true;
- }
- }
- }
如果需要停止服务,对应实现如下:
- //停止RTSP服务
- private void stopRtspService() {
- if(!isRTSPServiceRunning)
- return;
-
- if (libPublisher != null && rtsp_handle_ != 0) {
- libPublisher.StopRtspServer(rtsp_handle_);
- libPublisher.CloseRtspServer(rtsp_handle_);
- rtsp_handle_ = 0;
- }
-
- if(!isPushing)
- {
- releasePublisherHandle();
- }
-
- isRTSPServiceRunning = false;
- }
发布、停止发布RTSP流:
- private boolean StartRtspStream()
- {
- if (isRTSPPublisherRunning)
- return false;
-
- String rtsp_stream_name = "stream1";
- libPublisher.SetRtspStreamName(publisherHandle, rtsp_stream_name);
- libPublisher.ClearRtspStreamServer(publisherHandle);
-
- libPublisher.AddRtspStreamServer(publisherHandle, rtsp_handle_, 0);
-
- if (libPublisher.StartRtspStream(publisherHandle, 0) != 0)
- {
- Log.e(TAG, "调用发布rtsp流接口失败!");
-
- if (!isPushing)
- {
- libPublisher.SmartPublisherClose(publisherHandle);
- publisherHandle = 0;
- }
-
- return false;
- }
-
- isRTSPPublisherRunning = true;
- return true;
- }
-
- //停止发布RTSP流
- private void stopRtspPublisher()
- {
- if(!isRTSPPublisherRunning)
- return;
-
- isRTSPPublisherRunning = false;
-
- if (null == libPublisher || 0 == publisherHandle)
- return;
-
- libPublisher.StopRtspStream(publisherHandle);
-
- if (!isPushing && !isRTSPServiceRunning)
- {
- releasePublisherHandle();
- }
- }
因为处理后YUV或RGB数据需要重新编码,这时候需要推送端,设置下编码参数:
- private boolean OpenPushHandle() {
-
- if(publisherHandle != 0)
- {
- return true;
- }
-
- publisherHandle = libPublisher.SmartPublisherOpen(myContext, audio_opt, video_opt,
- videoWidth, videoHeight);
-
- if (publisherHandle == 0) {
- Log.e(TAG, "sdk open failed!");
- return false;
- }
-
- Log.i(TAG, "publisherHandle=" + publisherHandle);
-
- int fps = 20;
- int gop = fps * 1;
-
- int videoEncodeType = 1; //1: h.264硬编码 2: H.265硬编码
-
- if(videoEncodeType == 1) {
- int h264HWKbps = setHardwareEncoderKbps(true, videoWidth, videoHeight);
- h264HWKbps = h264HWKbps*fps/25;
-
- Log.i(TAG, "h264HWKbps: " + h264HWKbps);
-
- int isSupportH264HWEncoder = libPublisher
- .SetSmartPublisherVideoHWEncoder(publisherHandle, h264HWKbps);
-
- if (isSupportH264HWEncoder == 0) {
- libPublisher.SetNativeMediaNDK(publisherHandle, 0);
- libPublisher.SetVideoHWEncoderBitrateMode(publisherHandle, 1); // 0:CQ, 1:VBR, 2:CBR
- libPublisher.SetVideoHWEncoderQuality(publisherHandle, 39);
- libPublisher.SetAVCHWEncoderProfile(publisherHandle, 0x08); // 0x01: Baseline, 0x02: Main, 0x08: High
-
- // libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x200); // Level 3.1
- // libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x400); // Level 3.2
- // libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x800); // Level 4
- libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x1000); // Level 4.1 多数情况下,这个够用了
- //libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x2000); // Level 4.2
-
- // libPublisher.SetVideoHWEncoderMaxBitrate(publisherHandle, ((long)h264HWKbps)*1300);
-
- Log.i(TAG, "Great, it supports h.264 hardware encoder!");
- }
- }
- else if (videoEncodeType == 2) {
- int hevcHWKbps = setHardwareEncoderKbps(false, videoWidth, videoHeight);
- hevcHWKbps = hevcHWKbps*fps/25;
-
- Log.i(TAG, "hevcHWKbps: " + hevcHWKbps);
-
- int isSupportHevcHWEncoder = libPublisher
- .SetSmartPublisherVideoHevcHWEncoder(publisherHandle, hevcHWKbps);
-
- if (isSupportHevcHWEncoder == 0) {
- libPublisher.SetNativeMediaNDK(publisherHandle, 0);
- libPublisher.SetVideoHWEncoderBitrateMode(publisherHandle, 0); // 0:CQ, 1:VBR, 2:CBR
- libPublisher.SetVideoHWEncoderQuality(publisherHandle, 39);
-
- // libPublisher.SetVideoHWEncoderMaxBitrate(publisherHandle, ((long)hevcHWKbps)*1200);
-
- Log.i(TAG, "Great, it supports hevc hardware encoder!");
- }
- }
-
- libPublisher.SetSmartPublisherEventCallbackV2(publisherHandle, new EventHandlePublisherV2());
-
- libPublisher.SmartPublisherSetGopInterval(publisherHandle, gop);
-
- libPublisher.SmartPublisherSetFPS(publisherHandle, fps);
-
- return true;
- }
I420ExternalRender实现如下,这里可以拿到拉流的RTSP的YUV数据,然后处理后,可以调用推送端的PostLayerImageI420ByteBuffer()投递到轻量级RTSP服务或RTMP推送端编码发送出去。
- class I420ExternalRender implements NTExternalRender {
- // public static final int NT_FRAME_FORMAT_RGBA = 1;
- // public static final int NT_FRAME_FORMAT_ABGR = 2;
- // public static final int NT_FRAME_FORMAT_I420 = 3;
-
- private int width_ = 0;
- private int height_ = 0;
-
- private int y_row_bytes_ = 0;
- private int u_row_bytes_ = 0;
- private int v_row_bytes_ = 0;
-
- private ByteBuffer y_buffer_ = null;
- private ByteBuffer u_buffer_ = null;
- private ByteBuffer v_buffer_ = null;
-
- @Override
- public int getNTFrameFormat() {
- Log.i(TAG, "I420ExternalRender::getNTFrameFormat return "
- + NT_FRAME_FORMAT_I420);
- return NT_FRAME_FORMAT_I420;
- }
-
- @Override
- public void onNTFrameSizeChanged(int width, int height) {
- width_ = width;
- height_ = height;
-
- y_row_bytes_ = (width_ + 15) & (~15);
- u_row_bytes_ = ((width_ + 1) / 2 + 15) & (~15);
- v_row_bytes_ = ((width_ + 1) / 2 + 15) & (~15);
-
- y_buffer_ = ByteBuffer.allocateDirect(y_row_bytes_ * height_);
- u_buffer_ = ByteBuffer.allocateDirect(u_row_bytes_
- * ((height_ + 1) / 2));
- v_buffer_ = ByteBuffer.allocateDirect(v_row_bytes_
- * ((height_ + 1) / 2));
-
- Log.i(TAG, "I420ExternalRender::onNTFrameSizeChanged width_="
- + width_ + " height_=" + height_ + " y_row_bytes_="
- + y_row_bytes_ + " u_row_bytes_=" + u_row_bytes_
- + " v_row_bytes_=" + v_row_bytes_);
- }
-
- @Override
- public ByteBuffer getNTPlaneByteBuffer(int index) {
- if (index == 0) {
- return y_buffer_;
- } else if (index == 1) {
- return u_buffer_;
- } else if (index == 2) {
- return v_buffer_;
- } else {
- Log.e(TAG, "I420ExternalRender::getNTPlaneByteBuffer index error:" + index);
- return null;
- }
- }
-
- @Override
- public int getNTPlanePerRowBytes(int index) {
- if (index == 0) {
- return y_row_bytes_;
- } else if (index == 1) {
- return u_row_bytes_;
- } else if (index == 2) {
- return v_row_bytes_;
- } else {
- Log.e(TAG, "I420ExternalRender::getNTPlanePerRowBytes index error:" + index);
- return 0;
- }
- }
-
- public void onNTRenderFrame(int width, int height, long timestamp)
- {
- if ( y_buffer_ == null )
- return;
-
- if ( u_buffer_ == null )
- return;
-
- if ( v_buffer_ == null )
- return;
-
- y_buffer_.rewind();
- u_buffer_.rewind();
- v_buffer_.rewind();
-
- if( isPushing || isRTSPPublisherRunning )
- {
- libPublisher.PostLayerImageI420ByteBuffer(publisherHandle, 0, 0, 0,
- y_buffer_, 0, y_row_bytes_,
- u_buffer_, 0, u_row_bytes_,
- v_buffer_, 0, v_row_bytes_,
- width_, height_, 0, 0,
- 960, 540, 0,0);
- }
- }
- }
如果轻量级服务正常启动,会把rtsp的url回调上来:
- class EventHandlePublisherV2 implements NTSmartEventCallbackV2 {
- @Override
- public void onNTSmartEventCallbackV2(long handle, int id, long param1, long param2, String param3, String param4, Object param5) {
-
- Log.i(TAG, "EventHandlePublisherV2: handle=" + handle + " id:" + id);
-
- String publisher_event = "";
-
- switch (id) {
- ....
- case NTSmartEventID.EVENT_DANIULIVE_ERC_PUBLISHER_RTSP_URL:
- publisher_event = "RTSP服务URL: " + param3;
- break;
- }
- }
- }
以上是大概的流程,从RTSP拉流到数据处理后,重新塞给轻量级RTSP服务,然后播放端再从轻量级RTSP服务端拉流,如果针对YUV或RGB算法处理延迟不大的话,整体延迟可轻松达到毫秒级,满足大多数场景的技术诉求。