一个用mediapipe计算脸部朝向的简单方法

import cv2
import copy
import numpy as np
import open3d as o3d
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_holistic = mp.solutions.holistic
from scipy.spatial.transform import Rotation as Rtool



def process_landmark(face_landmarks, width, height):
    landmark1 = face_landmarks.landmark ## 478 is known number
    np_array = np.zeros((len(landmark1), 3), np.float64)
    for i in range(len(landmark1)):
        np_array[i, 0] = landmark1[i].x
        np_array[i, 1] = landmark1[i].y
        np_array[i, 2] = landmark1[i].z
    np_array[:, 0] = np_array[:, 0] * width
    np_array[:, 1] = - np_array[:, 1] * height
    np_array[:, 2] = - np_array[:, 2] * width
    return np_array



def process_face_landmark(face_landmarks, width, height):
    landmark_3d = process_landmark(face_landmarks, width, height)
    x_axis = landmark_3d[280] - landmark_3d[50]
    x_axis += landmark_3d[352] - landmark_3d[123]
    x_axis += landmark_3d[280] - landmark_3d[50]
    x_axis += landmark_3d[376] - landmark_3d[147]
    x_axis += landmark_3d[416] - landmark_3d[192]
    x_axis += landmark_3d[298] - landmark_3d[68]
    x_axis += landmark_3d[301] - landmark_3d[71]

    y_axis = landmark_3d[10] - landmark_3d[152]
    y_axis += landmark_3d[151] - landmark_3d[152]
    y_axis += landmark_3d[8] - landmark_3d[17]
    y_axis += landmark_3d[5] - landmark_3d[200]
    y_axis += landmark_3d[6] - landmark_3d[199]
    y_axis += landmark_3d[8] - landmark_3d[18]
    y_axis += landmark_3d[9] - landmark_3d[175]
    x_axis /= np.linalg.norm(x_axis)
    y_axis /= np.linalg.norm(y_axis)
    z_axis = np.cross(x_axis, y_axis)
    z_axis /= np.linalg.norm(z_axis)
    y_axis = np.cross(z_axis, x_axis)
    matrix = Rtool.from_matrix(np.transpose(np.array([x_axis, y_axis, z_axis]))) * Rtool.from_rotvec([-0.25, 0, 0])
    rotvec = matrix.as_rotvec()
    return rotvec


def process_head_roation(mpeg4file):
  cap = cv2.VideoCapture(mpeg4file)
  rots_head = []
  with mp_holistic.Holistic(min_detection_confidence=0.2, min_tracking_confidence=0.2) as holistic:
    while cap.isOpened():
      success, image = cap.read()
      if not success:
        break
      height, width = image.shape[0], image.shape[1]
      image.flags.writeable = False
      image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
      results = holistic.process(image)
      # Draw landmark annotation on the image.
      # mp_drawing.draw_landmarks(image, results.face_landmarks, mp_holistic.FACEMESH_CONTOURS, landmark_drawing_spec=None,
      #                           connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_contours_style())
      # mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS,
      #                           landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
      # mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS,
      #                           mp_drawing_styles.get_default_hand_landmarks_style(), mp_drawing_styles.get_default_hand_connections_style())
      # mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS,
      #                           mp_drawing_styles.get_default_hand_landmarks_style(), mp_drawing_styles.get_default_hand_connections_style())
      pose_landmarks = process_landmark(results.pose_landmarks)
      left_hand_landmarks = process_landmark(results.left_hand_landmarks)
      right_hand_landmarks = process_landmark(results.right_hand_landmarks)

      rotvec = process_face_landmark(results.face_landmarks, width, height)
      rots_head.append(rotvec.copy())
  return rots_head


if __name__=="__main__":
    name_ = "0.mp4"
    process_head_roation(name_)