unity 接收拼接数据进行纹理替换且保存相机纹理到rtsp server(一)

1 rtsp 协议后编码解码

rtsp协议的问题就是，拼接完成后，还需要编码，而unity里面再需要解码，需要的过程多了一步编码再解码，大大加重了

2 rtsp 协议后轻量编码

rtsp协议使用mjpeg进行图片传输。why？这样做的好处是解码端进行像素处理以后不用再进行h264和h265编码，而unity端也不用再解码一次，这样增加了程序运行效率

3 server

1 rtsp server
2 websocket server

c# c++ 都可以做websocket server，
使用c++ 做一个server
server 既能接收推流，又能接收流

4 unity client

unity 上建立一个两个plane，调整位置

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.IO;
//using NetM
public class test : MonoBehaviour
{
    // Start is called before the first frame update
    private string msg = null;
    string[] file1Lines;
    void Start()
    {
        string path = Application.dataPath + "/rtsp.txt";
        
        path = path.Replace("/", "\\");
        //File静态类方式
        if (File.Exists(path))
        {
            Debug.Log("FileExists");
            file1Lines = File.ReadAllLines(path);
            foreach (string line in file1Lines)
            {
                Debug.Log(line);
            }
        }
        else
        {
            Debug.Log("FileNotExists");
            File.CreateText(path);
        }
   
        

        //Texture2D m_resultTure = new Texture2D((int)widthSize, (int)heighSize, TextureFormat.RGB24, false);
        //Texture2D m_resultTure = GetComponent(texture1);
        //texture rigidbody2D = GetComponent(texture1);
        //rawImage = GetComponent();
        //rawImage.texture = texture;
        //NetManager.M_Instance.Connect("ws://127.0.0.1:8888");   //本机地址
    }
    private void OnGUI()
    {
        //绘制输入框，以及获取输入框中的内容
        //PS：第二参数必须是msg，否则在我们输入后，虽然msg可以获得到输入内容，但马上就被第二参数在下一帧重新覆盖。
        msg = GUI.TextField(new Rect(10, 10, 100, 20), msg);

        //绘制按钮，以及按下发送信息按钮，发送信息
        if (GUI.Button(new Rect(120, 10, 80, 20), "发送信息") && msg != null)
        {
            NetManager.M_Instance.Send(msg);
        }

        //绘制按钮，以及按下断开连接按钮，发送断开连接请求
        if (GUI.Button(new Rect(210, 10, 80, 20), "断开连接"))
        {
            Debug.Log("向服务器请求断开连接......");
            NetManager.M_Instance.CloseClientWebSocket();
        }

    }

    void Update()
    {
        
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

4.1 websocket client

using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.Net.WebSockets;
using System.Threading;
using System.Text;

public class NetManager
{
    #region 实现单例的代码
    //变量
    private volatile static NetManager m_instance;          
    private static object m_locker = new object();          

    //属性
    public static NetManager M_Instance
    {
        get
        {
            //线程锁。防止同时判断为null时同时创建对象
            lock (m_locker)
            {
                //如果不存在对象则创建对象
                if (m_instance == null)
                {
                    m_instance = new NetManager();
                }
            }
            return m_instance;
        }
    }
    #endregion

    //私有化构造
    private NetManager() { }

    //客户端webSocket
    private ClientWebSocket m_clientWebSocket;
    //处理接收数据的线程
    private Thread m_dataReceiveThread;
    //线程持续执行的标识符
    private bool m_isDoThread;
    private bool m_isConnected = false;

    /// 

    /// ClientWebSocket，与服务器建立连接。
    /// 
    /// 
    public void Connect(string uriStr)
    {
        try
        {
            //创建ClientWebSocket
            m_clientWebSocket = new ClientWebSocket();

            //初始化标识符
            m_isDoThread = true;

            //创建线程
            m_dataReceiveThread = new Thread(ReceiveData);  //创建数据接收线程  
            m_dataReceiveThread.IsBackground = true;        //设置为后台可以运行，主线程关闭时，此线程也会关闭（实际在Unity中并没什么用，还是要手动关闭）

            //设置请求头部
            //m_clientWebSocket.Options.SetRequestHeader("headerName", "hearValue");

            //开始连接
            var task = m_clientWebSocket.ConnectAsync(new Uri(uriStr), CancellationToken.None);
            task.Wait();    //等待

            //启动数据接收线程
            m_dataReceiveThread.Start(m_clientWebSocket);

            //输出提示
            if (m_clientWebSocket.State == WebSocketState.Open)
            {
                Debug.Log("连接服务器完毕。");
                m_isConnected = true;
            }
        }
        catch (WebSocketException ex)
        {
            Debug.LogError("连接出错：" + ex.Message);
            Debug.LogError("WebSokcet状态：" + m_clientWebSocket.State);
            CloseClientWebSocket();
        }

    }


 
    /// 

    /// 持续接收服务器的信息。
    /// 
    /// 
    private void ReceiveData(object socket)
    {
        //类型转换
        ClientWebSocket socketClient = (ClientWebSocket)socket;
        //持续接收信息
        while (m_isDoThread)
        {
            //接收数据
            string data = Receive(socketClient);
            //数据处理（可以和服务器一样使用事件（委托）来处理）
            if (data != null)
            {
                Debug.Log("接收的服务器消息：" + data);
            }
            //组装纹理
            //Texture2D tex2D = new Texture2D(100, 100);
            //tex2D.LoadImage(bytes);
         

        }
        Debug.Log("接收信息线程结束。");
    }

    /// 

    /// 接收服务器信息。
    /// 
    /// 
    /// 
    private byte[] Receive(ClientWebSocket socket)
    {
        try
        {
            //接收消息时，对WebSocketState是有要求的，所以加上if判断（如果不是这两种状态，会报出异常）
            if (socket != null && (socket.State == WebSocketState.Open || socket.State == WebSocketState.CloseSent))
            {
                byte[] arrry = new byte[1024];  //注意长度，如果服务器发送消息过长，这也需要跟着调整
                ArraySegment<byte> buffer = new ArraySegment<byte>(arrry);  //实例化一个ArraySegment结构体
                //接收数据
                var task = socket.ReceiveAsync(buffer, CancellationToken.None);
                task.Wait();//等待

                //仅作状态展示。在客户端发送关闭消息后，服务器会回复确认信息，在收到确认信息后状态便是CloseReceived，这里打印输出。
                Debug.Log("socekt当前状态：" + socket.State);

                //如果是结束消息确认，则返回null，不再解析信息
                if (socket.State == WebSocketState.CloseReceived || task.Result.MessageType == WebSocketMessageType.Close)
                {
                    return null;
                }
                //如果是字符串
                //return Encoding.UTF8.GetString(buffer.Array, 0, task.Result.Count);
                return buffer.Array;
            }
            else
            {
                return null;
            }
        }
        catch (WebSocketException ex)
        {
            Debug.LogError("接收服务器信息错误：" + ex.Message);
            CloseClientWebSocket();
            return null;
        }
    }

    /// 

    /// 发送消息
    /// 
    /// 
    public void Send(string content)
    {
        try
        {
            //发送消息时，对WebSocketState是有要求的，加上if判断（如果不是这两种状态，会报出异常）
            if (m_clientWebSocket != null && (m_clientWebSocket.State == WebSocketState.Open || m_clientWebSocket.State == WebSocketState.CloseReceived))
            {
                ArraySegment<byte> array = new ArraySegment<byte>(Encoding.UTF8.GetBytes(content)); //创建内容的字节编码数组并实例化一个ArraySegment结构体
                var task = m_clientWebSocket.SendAsync(array, WebSocketMessageType.Binary, true, CancellationToken.None);  //发送
                task.Wait();  //等待

                Debug.Log("发送了一个消息到服务器。");
            }
        }
        catch (WebSocketException ex)
        {
            Debug.LogError("向服务器发送信息错误：" + ex.Message);
            CloseClientWebSocket();
        }
    }

    /// 

    /// 关闭ClientWebSocket。
    /// 
    public void CloseClientWebSocket()
    {
        //关闭Socket
        if (m_clientWebSocket != null && m_clientWebSocket.State == WebSocketState.Open)
        {
            var task = m_clientWebSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, string.Empty, CancellationToken.None);
            Debug.Log("socekt状态：" + m_clientWebSocket.State);
            Debug.Log("close");
        }
        //关闭线程
        if (m_dataReceiveThread != null && m_dataReceiveThread.IsAlive)
        {
            m_isDoThread = false;   
            m_dataReceiveThread = null;
        }
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206

以上为websocket client， 接收到数据以后就可以替换纹理，接下去我们把里面的纹理图像保存出来，这样可以使用多个不同的camera 来保存不同角度和三维场景中不同的图像流。然后使用websocket 传输出去，形成一个rtsp server。

4.2 脚本start时候读取本地纹理

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.IO;
using System;
public class canvasrtsp : MonoBehaviour
{
    // Start is called before the first frame update
    //public RawImage rawImage;//相机渲染的UI
    //public GameObject plane1;//相机渲染的GameObject

    //StartCoroutine(LoadTextureFromInternet("http://avatar.csdnimg.cn/1/E/6/2_u013012420.jpg"));

    //IEnumerator LoadTextureFromInternet(string path)
    //{
    //    UnityWebRequest request = new UnityWebRequest(path);
    //    //给request的downloadhandle赋值，new出来的UnityWebRequest不附加downloadhandle
    //    //数据，不赋值的话访问不到下载出来的数据
    //    DownloadHandlerTexture texture = new DownloadHandlerTexture(true);
    //    request.downloadHandler = texture;
    //    yield return request.Send();
    //    if (string.IsNullOrEmpty(request.error))
    //    {
    //        pic = texture.texture;
    //    }

    //    Image tempImage = GameObject.Find("Image").GetComponent();
    //    Sprite sp = Sprite.Create((Texture2D)pic, new Rect(0, 0, pic.width, pic.height), Vector2.zero);
    //    tempImage.sprite = sp;

    //    GameObject go = GameObject.Find("plane1");
    //    go.GetComponent().material.mainTexture = pic;
    //}

    Texture2D Base64ToRGBA32(string imageData, int offset = 0)
    {
        Texture2D tex2D = new Texture2D(2, 2, TextureFormat.RGB24, false);
        imageData = imageData.Substring(offset);
        byte[] data = Convert.FromBase64String(imageData);
        tex2D.LoadImage(data);
        return tex2D;
    }
    void Start()
    {
        string path1 = @"F:\pic\b.png";
        FileStream fileStream = new FileStream(path1, FileMode.Open, FileAccess.Read);
        fileStream.Seek(0, SeekOrigin.Begin);
        byte[] bye = new byte[fileStream.Length];
        fileStream.Read(bye, 0, bye.Length);
        fileStream.Close();
        //创建texture
        Texture2D texture2D = new Texture2D(240, 144);
        texture2D.LoadImage(bye);
        //plane1.gameObject.SetActive(true);
        GameObject go = GameObject.Find("Plane");
        //GameObject go1 = GameObject.FindWithTag("ppp");
        GameObject go1 = GameObject.Find("Cube");
        //Debug.Log(texture2D.width, texture2D.height);
        go.GetComponent<MeshRenderer>().material.mainTexture = texture2D;
        go1.GetComponent<MeshRenderer>().material.mainTexture = texture2D;
        //T image = GetComponent("image1");
    }

    // Update is called once per frame
    void Update()
    {
        
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

5 保存相机纹理

增加一个camera capture函数，这个函数将当前相机所见保存成图像

 Texture2D CaptureCamera(Camera camera, Rect rect)
    {
        RenderTexture rt = new RenderTexture((int)rect.width, (int)rect.height, 0);//创建一个RenderTexture对象
        camera.targetTexture = rt;//临时设置相关相机的targetTexture为rt, 并手动渲染相关相机
        camera.Render();
        //ps: --- 如果这样加上第二个相机，可以实现只截图某几个指定的相机一起看到的图像。
        //ps: camera2.targetTexture = rt;
        //ps: camera2.Render();
        //ps: -------------------------------------------------------------------

        RenderTexture.active = rt;//激活这个rt, 并从中中读取像素。
        Texture2D screenShot = new Texture2D((int)rect.width, (int)rect.height, TextureFormat.RGB24, false);
        screenShot.ReadPixels(rect, 0, 0);//注：这个时候，它是从RenderTexture.active中读取像素
        screenShot.Apply();

        //重置相关参数，以使用camera继续在屏幕上显示
        camera.targetTexture = null;
        //ps: camera2.targetTexture = null;
        RenderTexture.active = null; //JC: added to avoid errors
        GameObject.Destroy(rt);

        byte[] bytes = screenShot.EncodeToPNG();//最后将这些纹理数据，成一个png图片文件
        //string filename = Application.dataPath + "/Screenshot.png";
        string filename = "e:/qianbo2.png";
        System.IO.File.WriteAllBytes(filename, bytes);
        Debug.Log(string.Format("截屏了一张照片: {0}", filename));

        return screenShot;
    }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

readpixel 方式

readpixel方式一定要注意等到 frameEnd 结束才能读取

 WaitForEndOfFrame frameEnd = new WaitForEndOfFrame();

    IEnumerator CaptureScreenshot2(Rect rect)
    {
        yield return frameEnd;
        Texture2D screenShot = new Texture2D((int)rect.width, (int)rect.height, TextureFormat.RGB24, false);//先创建一个的空纹理，大小可根据实现需要来设置
#pragma warning disable UNT0017 // SetPixels invocation is slow
        screenShot.ReadPixels(rect, 0, 0);//读取屏幕像素信息并存储为纹理数据，
#pragma warning restore UNT0017 // SetPixels invocation is slow
        screenShot.Apply();
        byte[] bytes = screenShot.EncodeToJPG();//然后将这些纹理数据，成一个png图片文件

        string filename = "e:/qianbo1.JPG";
        System.IO.File.WriteAllBytes(filename, bytes);
        Debug.Log(string.Format("截屏了一张图片: {0}", filename));
        //最后，我返回这个Texture2d对象，这样我们直接，所这个截图图示在游戏中，当然这个根据自己的需求的。
        //yield return screenShot;
    }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

update

我们使用update 来测试我们的函数，按下空格，按下S，按下F，分别调用函数

 void Update()
    {
        if (Input.GetKeyDown(KeyCode.Space))
        {
            int screenWidth = Screen.width;
            int screenHeight = Screen.height;
            RenderTexture rt = new RenderTexture(screenWidth, screenHeight, 24);
            Camera.main.targetTexture = rt;
            Camera.main.Render();
            RenderTexture.active = rt;//激活这个rt, 并从中中读取像素。
            Texture2D screenShot = new Texture2D(screenWidth, screenHeight, TextureFormat.RGB24, false);
            screenShot.ReadPixels(new Rect(0, 0, screenWidth, screenHeight), 0, 0);
            screenShot.Apply();
            byte[] bytes = screenShot.EncodeToJPG();
            //string path = Application.dataPath + "/../" + fileName + ".jpg";
            //string path = Application.dataPath + "/../" + fileName + ".jpg";
            string path = "e:/qianbo3.jpg" ;
            File.WriteAllBytes(path, bytes);
            Debug.Log(string.Format("space: {0}", path));
        }
        else if(Input.GetKeyDown(KeyCode.F))
        {
            
            Rect rect = new Rect(10, 10, 300, 200);
            CaptureScreenshot2(rect);
            //Debug.Log(string.Format("F: {0}", path));
        }
        else if (Input.GetKeyDown(KeyCode.S))
        {
            mainCamera = GameObject.Find("Main Camera");

            Rect rect = new Rect(0, 0, 300, 200);
            //获取主摄像机的Camera组件
            //mainCamera= GameObject.Find("MainCamera").GetComponent();
            //Camera ca = Camera.main;
            CaptureCamera(Camera.main, rect);
        }
    }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

以下为存储到磁盘的图像

rtsp 方式传输

我们将使用rtsp方式传送抓取到的纹理图像，使用mjpeg over rtsp， 将jpeg 传送到server，由server提供rtsp 服务，或者http服务，使用c++来写这个server，这个由下一次再讲了。