MNN 实现NV12转BGR格式

MNN 图像处理

MNN中提供了CV模块，可以帮助用户简化图像的处理，还可以免于引入opencv、libyuv等图片处理库。

1. 支持目标Tensor为float或 uint8_t 的数据格式
2. 支持目标Tensor为NC4HW4、NHWC、NCHW的维度格式
3. CV模块支持直接输入Device Tensor，也即由Session中获取的Tensor。

MNN图像处理配置

struct Config
{
    Filter filterType = NEAREST;
    ImageFormat sourceFormat = RGBA;
    ImageFormat destFormat = RGBA;

    //Only valid if the dest type is float
    float mean[4] = {0.0f,0.0f,0.0f, 0.0f};
    float normal[4] = {1.0f, 1.0f, 1.0f, 1.0f};
};
1
2
3
4
5
6
7
8
9
10

CV::ImageProcess::Config中

• 通过sourceFormat和destFormat指定输入和输出的格式，当前支持RGBA、RGB、BGR、GRAY、BGRA、YUV_NV21、YUV_NV12
• 通过filterType指定插值的类型，当前支持NEAREST、BILINEAR和BICUBIC三种插值方式
• 通过mean和normal指定均值归一化，但数据类型不是浮点类型时，设置会被忽略

测试用例 NV12转BGR

#include 
#include 
#include 
#include "MNNTestSuite.h"
#define MNN_OPEN_TIME_TRACE
#include 

using namespace MNN;
using namespace MNN::CV;

static void BGR2NV12(const cv::Mat &bgr_image,
                                unsigned char *buffer) {
  int bgr_width = bgr_image.cols;
  int bgr_height = bgr_image.rows;
  cv::Mat yuv_image = cv::Mat(bgr_height, bgr_width, CV_8UC2);
  cvtColor(bgr_image, yuv_image, CV_BGRA2YUV_I420);

  int len_y = bgr_height * bgr_width;
  int len_u = len_y >> 2;
  unsigned char *pt_yuv_y = yuv_image.data;
  unsigned char *pt_yuv_u = pt_yuv_y + len_y;
  unsigned char *pt_yuv_v = pt_yuv_u + len_u;
  unsigned char *pt_dst_uv = buffer + len_y;
  int i, j;
  // copy y;
  memcpy(buffer, pt_yuv_y, len_y);
  // copy uv;
  for (i = 0, j = 0; i < len_u; i++) {
    pt_dst_uv[j++] = pt_yuv_u[i];
    pt_dst_uv[j++] = pt_yuv_v[i];
  }
}
int main(int argc, char *argv[])
{
    ImageProcess::Config config;
    config.sourceFormat = YUV_NV12;
    config.destFormat   = BGR;
    config.filterType   = NEAREST;
    config.wrap         = CLAMP_TO_EDGE;
    std::shared_ptr<ImageProcess> process(ImageProcess::create(config));
    int sw = 1280;
    int sh = 960;
    Matrix tr;
    process->setMatrix(tr);
    std::shared_ptr<Tensor> tensor(Tensor::create<uint8_t>(std::vector<int>{1, sh, sw, 3}, nullptr, Tensor::TENSORFLOW));
    char src_video[256];
    int n = sprintf(src_video, "test.avi");
    src_video[n] = '\0';

    cv::VideoCapture videoReader;
    videoReader.open(src_video);
    if (!videoReader.isOpened()) {
        return 0;
    }
    cv::Mat yuv_nv12 = cv::Mat::zeros(1440, 1280, CV_8UC1);
    int cnt=0;
    while (true)
	{
        printf("=== %d\n", cnt);
        cv::Mat frame;
        if (!videoReader.read(frame))
		{
			break;
		}

        BGR2NV12(frame, yuv_nv12.data);
        process->convert(yuv_nv12.data, sw, sh, 0, tensor.get());
        cv::Mat dest_image = cv::Mat::zeros(sh, sw, CV_8UC3);
        for (int y = 0; y < sh; ++y) {
            auto dstY    = tensor->host<uint8_t>() + 3 * y * sw;
            auto srcY_Y  = yuv_nv12.data + y * sw;
            auto srcY_UV = yuv_nv12.data + (y / 2) * (sw / 2) * 2 + sw * sh;
            for (int x = 0; x < sw; ++x) {
                auto dstX    = dstY + 3 * x;
                auto srcX_Y  = srcY_Y + x;
                auto srcX_UV = srcY_UV + (x / 2) * 2;
                int Y = srcX_Y[0];
                int U = (int)srcX_UV[0] - 128;
                int V = (int)srcX_UV[1] - 128;
                Y     = Y << 6;
                int r = (Y + 73 * V) >> 6;
                int g = (Y - 25 * U - 37 * V) >> 6;
                int b = (Y + 130 * U) >> 6;
                r         = r < 0 ? 0 : r;
                r         = r > 255 ? 255 : r;
                g         = g < 0 ? 0 : g;
                g         = g > 255 ? 255 : g;
                b         = b < 0 ? 0 : b;
                b         = b > 255 ? 255 : b;
                auto diff = [](int a, int b) { return abs(a - b) > 5; };
                if (diff(dstX[0], b) || diff(dstX[1], g) || diff(dstX[2], r)) {
                    MNN_ERROR("%d, Error for NV12 to RGB: %d:  %d, %d, %d -> %d, %d, %d, wrong: %d, %d, %d\n", y, x,
                                (int)srcX_Y[0], U, V, r, g, b, dstX[0], dstX[1], dstX[2]);
                    return false;
                }
                // printf("%d, %d, %d\n", dstX[0], dstX[1], dstX[2]);
                dest_image.at<cv::Vec3b>(y, x).val[0] = dstX[0];
                dest_image.at<cv::Vec3b>(y, x).val[1] = dstX[1];
                dest_image.at<cv::Vec3b>(y, x).val[2] = dstX[2];
            }
        }
        cv::imshow("result", dest_image);
        cv::waitKey(0);
    }
    videoReader.release();
    return 0;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107