Mediapipe 实现3D人脸检测

下载源码

git clone https://github.com/google/mediapipe.git
1

安装Bazelisk

./bazel-5.2.0-installer-linux-x86_64.sh --user
1

设置环境变量

export PATH="$PATH:$HOME/bin"
1

编译并运行face_mesh样例CPU

bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/face_mesh:face_mesh_cpu

export GLOG_logtostderr=1

bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_cpu --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt

1
2
3
4
5
6

编译并运行face_mesh样例GPU

bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 mediapipe/examples/desktop/face_mesh:face_mesh_gpu

export GLOG_logtostderr=1

bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_gpu --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt --input_video_path=testdir/test2.mp4 --output_video_path=testdir/test2out.mp4
1
2
3
4
5

编译成3D人脸特征点的动态库

配置face_detection

/home/liangbaikai/mediapipe/mediapipe/modules/face_detection/face_detection_short_range.pbtxt  
1

下修改

model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
1

修改landmark 模型路径

/home/liangbaikai/mediapipe/mediapipe/modules/face_landmark/face_landmarks_model_loader.pbtxt
1

node {
  calculator: "SwitchContainer"
  input_side_packet: "ENABLE:with_attention"
  output_side_packet: "PACKET:model_path"
  options: {
    [mediapipe.SwitchContainerOptions.ext] {
      contained_node: {
        calculator: "ConstantSidePacketCalculator"
        options: {
          [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
            packet {
              string_value: "mediapipe/modules/face_landmark/face_landmark.tflite"
            }
          }
        }
      }
      contained_node: {
        calculator: "ConstantSidePacketCalculator"
        options: {
          [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
            packet {
              string_value: "/home/liangbaikai/mediapipe/modules/face_landmark/face_landmark_with_attention.tflite"
            }
          }
        }
      }
    }
  }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

1.在mediapipe/example/desktop/下新建文件夹myface_mesh_so

2.在myface_mesh_so文件夹新建 BUILD文件和face_landmark_gpu.cpp文件

BUILD文件：

licenses(["notice"])
package(default_visibility = ["//mediapipe/examples:__subpackages__"])

# Linux only
cc_binary(
    name = "face_landmark_gpu",
    srcs = ["face_landmark_gpu.cpp"],
    linkshared = True,
    deps = [
        "//mediapipe/graphs/face_mesh:desktop_live_gpu_calculators",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame",
        "//mediapipe/framework/formats:image_frame_opencv",
        "//mediapipe/framework/port:file_helpers",
        "//mediapipe/framework/port:opencv_highgui",
        "//mediapipe/framework/port:opencv_imgproc",
        "//mediapipe/framework/port:opencv_video",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/port:status",
        "//mediapipe/gpu:gl_calculator_helper",
        "//mediapipe/gpu:gpu_buffer",
        "//mediapipe/gpu:gpu_shared_data_internal",
        "@com_google_absl//absl/flags:flag",
        "@com_google_absl//absl/flags:parse",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/formats:rect_cc_proto",
    ],
)

cc_binary(
    name = "face_landmark_cpu",
    srcs = ["face_landmark_cpu.cpp"],
    # linkshared = True,
    deps = [
        "//mediapipe/graphs/face_mesh:desktop_live_calculators",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame",
        "//mediapipe/framework/formats:image_frame_opencv",
        "//mediapipe/framework/port:file_helpers",
        "//mediapipe/framework/port:opencv_highgui",
        "//mediapipe/framework/port:opencv_imgproc",
        "//mediapipe/framework/port:opencv_video",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/port:status",
        "@com_google_absl//absl/flags:flag",
        "@com_google_absl//absl/flags:parse",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/formats:rect_cc_proto",
    ],
)


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

face_landmark_gpu.cpp

#include 

#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_video_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"

#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"


namespace LIANGBAIKAI_3DFACE_LANDMARKS{


absl::Status Face3D_Landmarks_init_(std::string pbfilepath,
                        mediapipe::CalculatorGraph &graph,
                        mediapipe::GlCalculatorHelper &gpu_helper,
                        std::unique_ptr<mediapipe::OutputStreamPoller> &poller,
                        std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks)
{
    absl::Status run_status;
    std::cout << "GetContents ..."<< std::endl;
    std::string calculator_graph_config_contents;
    MP_RETURN_IF_ERROR(mediapipe::file::GetContents(pbfilepath,&calculator_graph_config_contents));

    // std::cout << "Get calculator graph config contents: " << calculator_graph_config_contents << std::endl;
    mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(calculator_graph_config_contents);

    std::cout << "Initialize the calculator graph."<< std::endl;
    MP_RETURN_IF_ERROR(graph.Initialize(config));
    
    std::cout << "Initialize the GPU."<< std::endl;
    ASSIGN_OR_RETURN(auto gpu_resources, mediapipe::GpuResources::Create());
    MP_RETURN_IF_ERROR(graph.SetGpuResources(std::move(gpu_resources)));
    gpu_helper.InitializeForTest(graph.GetGpuResources().get());

    // 添加视频输出流
    std::cout << "Start running the calculator graph." << std::endl;
    mediapipe::StatusOrPoller sop = graph.AddOutputStreamPoller("output_video");
    assert(sop.ok());
    poller = std::make_unique<mediapipe::OutputStreamPoller>(std::move(sop.value()));
    // 添加landmarks输出流
    mediapipe::StatusOrPoller sop_landmark = graph.AddOutputStreamPoller("multi_face_landmarks");
    assert(sop_landmark.ok());
    poller_landmarks = std::make_unique<mediapipe::OutputStreamPoller>(std::move(sop_landmark.value()));

    std::cout << "StartRun"<< std::endl;
    MP_RETURN_IF_ERROR(graph.StartRun({}));
    return absl::OkStatus(); 
}


absl::Status Face3D_Landmarks_run_(cv::Mat &rgba_mat,std::vector<cv::Point3f> &point_v,
                    mediapipe::CalculatorGraph &graph,
                    mediapipe::GlCalculatorHelper &gpu_helper,
                    std::unique_ptr<mediapipe::OutputStreamPoller> &poller,
                    std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks)
{    
    // Wrap Mat into an ImageFrame.
    auto input_frame = absl::make_unique<mediapipe::ImageFrame>(mediapipe::ImageFormat::SRGBA, rgba_mat.cols, rgba_mat.rows,mediapipe::ImageFrame::kGlDefaultAlignmentBoundary);

    cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get());
    rgba_mat.copyTo(input_frame_mat);

    // Prepare and add graph input packet.
    size_t frame_timestamp_us =(double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;

    MP_RETURN_IF_ERROR(gpu_helper.RunInGlContext(
        [&input_frame, &frame_timestamp_us, &graph,&gpu_helper]() -> absl::Status
        {
            // Convert ImageFrame to GpuBuffer.
            auto texture = gpu_helper.CreateSourceTexture(*input_frame.get());
            auto gpu_frame = texture.GetFrame<mediapipe::GpuBuffer>();
            glFlush();
            texture.Release();
            // Send GPU image packet into the graph.
            MP_RETURN_IF_ERROR(graph.AddPacketToInputStream("input_video", mediapipe::Adopt(gpu_frame.release()).At(mediapipe::Timestamp(frame_timestamp_us))));
            return absl::OkStatus(); 
        }
    ));

    mediapipe::Packet packet;
    if (!poller->Next(&packet)){
        std::cout << "no output video" << std::endl;
    }

    mediapipe::Packet packet_landmarks;
    if(poller_landmarks->QueueSize() > 0){
        if (poller_landmarks->Next(&packet_landmarks))
        {
            std::vector<mediapipe::NormalizedLandmarkList> output_landmarks = packet_landmarks.Get<std::vector<mediapipe::NormalizedLandmarkList>>();
            if(1 == output_landmarks.size()){
                mediapipe::NormalizedLandmarkList single_face_LandmarkList = output_landmarks[0];
                point_v.clear();
                for (int i = 0; i < single_face_LandmarkList.landmark_size(); ++i){
                    const mediapipe::NormalizedLandmark landmark = single_face_LandmarkList.landmark(i);
                    // std::cout << "num : " <<  i  << std::endl;
                    // std::cout << "x : " <<  landmark.x() * rgba_mat.cols  << std::endl;
                    // std::cout << "y : " <<  landmark.y() * rgba_mat.cols  << std::endl;
                    // std::cout << "z : " <<  landmark.z() * rgba_mat.cols  << std::endl;
                    point_v.push_back(cv::Point3f(landmark.x() * rgba_mat.cols,landmark.y() * rgba_mat.cols,landmark.z() * rgba_mat.cols));
                }
            }
        }
    }

    return absl::OkStatus();
}


absl::Status Face3D_Landmarks_stop_(mediapipe::CalculatorGraph &graph){
    MP_RETURN_IF_ERROR(graph.CloseInputStream("input_video"));
    return graph.WaitUntilDone();
}


int Face3D_Landmarks_init(std::string pbfilepath,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks){
    absl::Status run_status = Face3D_Landmarks_init_(pbfilepath,graph,gpu_helper,poller,poller_landmarks);
    if (!run_status.ok()){
        LOG(ERROR) << "Failed to init " << run_status.message();
        return 1;
    }
    return 0;
}

int Face3D_Landmarks_run(cv::Mat &rgba_mat,std::vector<cv::Point3f> &point_v,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks){
    absl::Status run_status = Face3D_Landmarks_run_(rgba_mat,point_v,graph,gpu_helper,poller,poller_landmarks);
    if (!run_status.ok()){
        LOG(ERROR) << "Failed to run " << run_status.message();
        return 1;
    }
    return 0;
}

int Face3D_Landmarks_stop(mediapipe::CalculatorGraph &graph){
    absl::Status run_status = Face3D_Landmarks_stop_(graph);
    if (!run_status.ok()){
        LOG(ERROR) << "Failed to stop " << run_status.message();
        return 1;
    }
    return 0;
}

}



// int main(int argc, char **argv)
// {
//     mediapipe::CalculatorGraph graph;
//     mediapipe::GlCalculatorHelper gpu_helper;
//     std::unique_ptr poller;
//     std::unique_ptr poller_landmarks;

//     std::string pbfile = "/home/caijun/mediapipe/mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt"; //GPU
//     LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_init(pbfile,graph,gpu_helper ,poller,poller_landmarks);

//     cv::Mat src = cv::imread("/home/caijun/mediapipe/mediapipe/examples/desktop/myface_mesh_so/test/huge.jpg");
//     cv::Mat rgba_mat;
//     cv::cvtColor(src, rgba_mat, cv::COLOR_BGR2RGBA);
//     std::vector point_v;
//     LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_run(rgba_mat,point_v,graph,gpu_helper,poller,poller_landmarks);

//     // for(auto &i : point_v){
//     //     std::cout << i << std::endl;
//     // }

//     for(unsigned i = 0; i < point_v.size();i ++){
//         std::cout << point_v[i].x << std::endl;

//     }

//     LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_stop(graph);
//     return 0;
// }



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188

3.编译

bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 mediapipe/examples/desktop/myface_mesh_so:face_landmark_gpu
1

成功后会在mediapipe/bazel-bin/mediapipe/examples/desktop/myface_mesh_so/路径下生成动态库文件

4.动态库使用样例

#include
#include
#include 
#include 
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/gpu/gl_calculator_helper.h"

#include 
double const lbk_get_time_ms(){
    double t;
    struct timeval tv;
    gettimeofday(&tv, NULL);
    t =  tv.tv_sec*1000. + tv.tv_usec/1000.;
return t;
}

namespace LIANGBAIKAI_3DFACE_LANDMARKS{
    int Face3D_Landmarks_init(std::string pbfilepath,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks);
    int Face3D_Landmarks_run(cv::Mat &rgba_mat,std::vector<cv::Point3f> &point_v,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks);
    int Face3D_Landmarks_stop(mediapipe::CalculatorGraph &graph);

}

int main(int argc, char **argv)
{
    mediapipe::CalculatorGraph graph;
    mediapipe::GlCalculatorHelper gpu_helper;
    std::unique_ptr<mediapipe::OutputStreamPoller> poller;
    std::unique_ptr<mediapipe::OutputStreamPoller> poller_landmarks;

    std::string pbfile = "/home/liangbaikai/mediapipe/mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt"; //GPU
    LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_init(pbfile,graph,gpu_helper ,poller,poller_landmarks);

    cv::Mat src = cv::imread("/home/liangbaikai/mediapipe/mediapipe/examples/desktop/myface_mesh_so/test/huge.jpg");
    cv::Mat rgba_mat;
    cv::cvtColor(src, rgba_mat, cv::COLOR_BGR2RGBA);
    std::vector<cv::Point3f> point_v;
    LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_run(rgba_mat,point_v,graph,gpu_helper,poller,poller_landmarks);

    for(unsigned i = 0; i < point_v.size();i ++){
        std::cout << point_v[i].x << std::endl;
    }

    LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_stop(graph);
    return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
message("-- project name : test")
project(test)

set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -Wall -fPIC")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3 -Wall -fPIC -g")

set(CMAKE_BUILD_TYPE Release)
set(TARGET test)


set(OpenCV_DIR "/usr/local/share/OpenCV")
find_package(OpenCV REQUIRED)

include_directories(
    ${PROJECT_SOURCE_DIR}/include
    /home/caijun/mediapipe/ 
    /root/anaconda3/pkgs/abseil-cpp-20211102.0-hd4dd3e8_0/include/ 
    /root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/host/bin/ 
    /root/anaconda3/include 
    /root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/k8-opt/bin/external/com_github_glog_glog/src/ 
    /root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/k8-opt/bin/external/com_github_gflags_gflags/_virtual_includes/gflags/ 
    /root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/k8-opt/bin/external/com_github_glog_glog/_virtual_includes/default_glog_headers/
)

link_directories(
    /home/liangbaikai/mediapipe/bazel-bin/mediapipe/examples/desktop/myface_mesh_so
    /root/anaconda3/pkgs/abseil-cpp-20211102.0-hd4dd3e8_0/lib
)


set(SRC_LIST 
    ${PROJECT_SOURCE_DIR}/src/main.cpp
)

# add_library(${TARGET} SHARED ${SRC_LIST})
add_executable(${TARGET}  ${SRC_LIST})              #test demo


target_link_libraries(${TARGET} 
    ${OpenCV_LIBS} 
    face_landmark_gpu
    # face_landmark_cpu
    absl_raw_hash_set
)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46