• Unity & FACEGOOD Audio2Face 通过音频驱动面部BlendShape

    目录

    Audio2Face简介

    在Unity中应用

    Audio2Face简介

    在元宇宙的热潮下，为了让AI数字人渗透到更多的领域中，FACEGOOD已经将语音驱动口型的算法技术开源，开源地址：

    https://github.com/FACEGOOD/FACEGOOD-Audio2Face

    该技术可以实时将音频数据转换为驱动数字人面部BlendShape的权重数据，不同于ARKit中的52个BlendShape，它的数量多达116个，我们可以通过对应关系得到相应的数值，对应关系如下：

    ARKitVoice2Face
    eyeBlinkLefteye_blink2_l
    eyeLookDownLefteye_lookDown2_l
    eyeLookInLefteye_lookRight_l
    eyeLookOutLefteye_lookLeft_l
    eyeLookUpLefteye_lookUp_l
    eyeSquintLefteye_shutTight_l
    eyeWideLeftmax(eye_downLidRaise_l,eye_upLidRaise_l)
    eyeBlinkRighteye_blink2_r
    eyeLookDownRighteye_lookDown2_r
    eyeLookInRighteye_lookRight_r
    eyeLookOutRighteye_lookLeft_r
    eyeLookUpRighteye_lookUp_r
    eyeSquintRighteye_shutTight_r
    eyeWideRightmax(eye_downLidRaise_r,eye_upLidRaise_r)
    jawForwardjaw_thrust_c
    jawLeftjaw_sideways_l
    jawRightjaw_sideways_r
    jawOpenmouth_stretch_c
    mouthClosemouth_chew_c
    mouthFunnelmax(mouth_funnel_dl,mouth_funnel_dr,mouth_funnel_ul,mouth_funnel_ur)
    mouthPuckermax(mouth_pucker_l,mouth_pucker_r)
    mouthLeftmouth_sideways_l
    mouthRightmouth_sideways_r
    mouthSmileLeftmouth_lipCornerPull_l
    mouthSmileRightmouth_lipCornerPull_r
    mouthFrownLeftmax(mouth_lipCornerDepress_l,mouth_lipCornerDepressFix_l)
    mouthFrownRightmax(mouth_lipCornerDepress_r,mouth_lipCornerDepressFix_r)
    mouthDimpleLeftmouth_dimple_l
    mouthDimpleRightmouth_dimple_r
    mouthStretchLeftmouth_lipStretch_l
    mouthStretchRightmouth_lipStretch_r
    mouthRollLowermax(mouth_suck_dl,mouth_suck_dr)
    mouthRollUppermax(mouth_suck_ul,mouth_suck_ur)
    mouthShrugLowermouth_chinRaise_d
    mouthShrugUppermouth_chinRaise_u
    mouthPressLeftmouth_press_l
    mouthPressRightmouth_press_r
    mouthLowerDownLeftmouth_lowerLipDepress_l
    mouthLowerDownRightmouth_lowerLipDepress_r
    mouthUpperUpLeftmouth_upperLipRaise_l
    mouthUpperUpRightmouth_upperLipRaise_r
    browDownleftbrow_lower_l
    browDownRightbrow_lower_r
    browInnerUpbrow_raise_c
    browOuterUpLeftbrow_raise_l
    browOuterUpRightbrow_raise_r
    cheekPuffmax(cheek_puff_l,cheek_puff_r)
    cheekSquintLeftcheek_up
    cheekSquintRightcheek_up
    noseSneerLeftnose_out_l
    noseSneerRightnose_out_r
    tongueOut

    生产的数据结果如下图所示，可见是116个取值范围为-1~1的小数：

    这116个数值依次对应下面116个BlendShape名称：

    1. brow_lower_l
    2. tongue_Scale__X
    3. tongue_Scale_Y
    4. tongue_Scale__Y
    5. tongue_Scale_Z
    6. tongue_Scale__Z
    7. nose_out_l
    8. nose_out_r
    9. tongue_u
    10. tongue_u_u
    11. brow_raise_d
    12. cheek_suck_r
    13. mouth_stretch_u
    14. tongue_u_d
    15. tooth_d_d
    16. tongue_d
    17. tooth_r
    18. tooth_d_u
    19. cheek_UP
    20. eye_blink1_l
    21. eye_blink1_r
    22. eye_blink2_l
    23. eye_blink2_r
    24. eye_lidTight_l
    25. eye_lidTight_r
    26. eye_shutTight_l
    27. eye_shutTight_r
    28. brow_lower_r
    29. eye_upperLidRaise_l
    30. eye_upperLidRaise_r
    31. eye_downLidRaise_l
    32. eye_downLidRaise_r
    33. jaw_sideways_l
    34. jaw_sideways_r
    35. jaw_thrust_c
    36. mouth_chew_c
    37. mouth_chinRaise_d
    38. mouth_chinRaise_u
    39. brow_raise_c
    40. mouth_dimple_l
    41. mouth_dimple_r
    42. mouth_funnel_dl
    43. mouth_funnel_dr
    44. mouth_funnel_ul
    45. mouth_funnel_ur
    46. mouth_lipCornerDepressFix_l
    47. mouth_lipCornerDepressFix_r
    48. mouth_lipCornerDepress_l
    49. mouth_lipCornerDepress_r
    50. brow_raise_l
    51. mouth_lipCornerPullOpen_l
    52. mouth_lipCornerPullOpen_r
    53. mouth_lipCornerPull_l
    54. mouth_lipCornerPull_r
    55. mouth_lipStretchOpen_l
    56. mouth_lipStretchOpen_r
    57. mouth_lipStretch_l
    58. mouth_lipStretch_r
    59. mouth_lowerLipDepress_l
    60. mouth_lowerLipDepress_r
    61. brow_raise_r
    62. mouth_lowerLipProtrude_c
    63. mouth_oh_c
    64. mouth_oo_c
    65. mouth_pressFix_c
    66. mouth_press_l
    67. mouth_press_r
    68. mouth_pucker_l
    69. mouth_pucker_r
    70. mouth_screamFix_c
    71. mouth_sideways_l
    72. cheek_puff_l
    73. mouth_sideways_r
    74. mouth_stretch_c
    75. mouth_suck_dl
    76. mouth_suck_dr
    77. mouth_suck_ul
    78. mouth_suck_ur
    79. mouth_upperLipRaise_l
    80. mouth_upperLipRaise_r
    81. nose_wrinkle_l
    82. nose_wrinkle_r
    83. cheek_puff_r
    84. tooth_l
    85. eye_lookDown1_l
    86. eye_lookDown2_l
    87. eye_lookLeft_l
    88. eye_lookRight_l
    89. eye_lookUp_l
    90. eye_lookDown1_r
    91. eye_lookDown2_r
    92. eye_lookLeft_r
    93. eye_lookRight_r
    94. cheek_raise_l
    95. eye_lookUp_r
    96. tongue_Rot_1X
    97. tongue_Rot__1X
    98. tongue_Rot_2X
    99. tongue_Rot__2X
    100. tongue_Rot_3X
    101. tongue_Rot__3X
    102. tongue_Rot_1Y
    103. tongue_Rot__1Y
    104. tongue_Rot_2Y
    105. cheek_raise_r
    106. tongue_Rot__2Y
    107. tongue_Rot_3Y
    108. tongue_Rot__3Y
    109. tongue_Rot_1Z
    110. tongue_Rot__1Z
    111. tongue_Rot_2Z
    112. tongue_Rot__2Z
    113. tongue_Rot_3Z
    114. tongue_Rot__3Z
    115. tongue_Scale_X
    116. cheek_suck_l

    在Unity中应用

    可以用过构建python服务，Unity客户端开启麦克风录制音频，将音频数据发送给python服务端，服务端转换为驱动BlendShape的权重数据后，返回给Unity客户端进行驱动。需要注意的是Unity中BlendShape的权重范围并不是[-1,1]，因此需要进行映射。

     例如：

    1. //将[-1,1]映射到[-100,100]
    2. private float Remap(float v)
    3. {
    4.     return v * 100f;
    5. }

    下面是一段测试音频产生的bs权重数据文件，每一行包含116个权重数值，我们拿来进行测试，将其放到StreamingAssets文件夹下。

    测试模型：

    测试代码：

    1. using System.IO;
    2. using System.Collections;
    3. using System.Collections.Generic;
    4.  
    5. using UnityEngine;
    6.  
    7. public class TEST : MonoBehaviour
    8. {
    9.     private Coroutine coroutine;
    10.     private SkinnedMeshRenderer smr;
    11.     private readonly Listfloat>> valueList = new Listfloat>>();
    12.  
    13.     private IEnumerator Start()
    14.     {
    15.         smr = GetComponent();
    16.         string path = Path.Combine(Application.streamingAssetsPath, "weight.txt");
    17.         using (StreamReader streamReader = new StreamReader(path))
    18.         {
    19.             string content;
    20.             while ((content = streamReader.ReadLine()) != null)
    21.             {
    22.                 List<float> list = new List<float>();
    23.                 content = content.Trim();
    24.                 string[] splitArray = content.Split(' ');
    25.                 for (int i = 0; i < splitArray.Length; i++)
    26.                 {
    27.                     float.TryParse(splitArray[i], out float result);
    28.                     list.Add(result);
    29.                 }
    30.                 valueList.Add(list);
    31.                 yield return null;
    32.             }
    33.         }
    34.     }
    35.  
    36.     private IEnumerator ExecuteCoroutine()
    37.     {
    38.         for (int i = 0; i < valueList.Count; i++)
    39.         {
    40.             List<float> list = valueList[i];
    41.  
    42.             smr.SetBlendShapeWeight(0, Remap(list[49]));    //brow_raise_l
    43.             smr.SetBlendShapeWeight(1, Remap(list[60]));    //brow_raise_r
    44.             smr.SetBlendShapeWeight(2, Remap(list[25]));    //eye_shutTight_l
    45.             smr.SetBlendShapeWeight(3, Remap(list[26]));    //eye_shutTight_r
    46.             smr.SetBlendShapeWeight(4, Remap(list[87]));    //eye_lookRight_l
    47.             smr.SetBlendShapeWeight(5, Remap(list[86]));    //eye_lookLeft_l    
    48.             smr.SetBlendShapeWeight(6, Remap(list[92]));    //eye_lookRight_r
    49.             smr.SetBlendShapeWeight(7, Remap(list[91]));    //eye_lookLeft_r
    50.             smr.SetBlendShapeWeight(8, Remap(list[88]));    //eye_lookUp_l
    51.             smr.SetBlendShapeWeight(9, Remap(list[94]));    //eye_lookUp_r
    52.             smr.SetBlendShapeWeight(10, Remap(list[85]));   //eye_lookDown2_l
    53.             smr.SetBlendShapeWeight(11, Remap(list[90]));   //eye_lookDown2_r
    54.             smr.SetBlendShapeWeight(12, Mathf.Max(Remap(list[71]), Remap(list[82])));   //cheek_pull_l cheek_pull_r
    55.             smr.SetBlendShapeWeight(13, Remap(list[18]));   //cheek_UP
    56.             smr.SetBlendShapeWeight(14, Remap(list[18]));   //cheek_UP
    57.             smr.SetBlendShapeWeight(15, Remap(list[6]));    //nose_out_l   
    58.             smr.SetBlendShapeWeight(16, Remap(list[7]));    //nose_out_r
    59.             smr.SetBlendShapeWeight(17, Remap(list[70]));   //mouth_sideways_l
    60.             smr.SetBlendShapeWeight(18, Remap(list[72]));   //mouth_sideways_r
    61.             smr.SetBlendShapeWeight(19, Mathf.Max(Remap(list[67]), Remap(list[68])));   //mouth_pucker_l mouth_pucker_2
    62.             smr.SetBlendShapeWeight(20, Mathf.Max(Remap(list[41]), Remap(list[42]), Remap(list[43]), Remap(list[44])));   //mouth_funnel_dl dr ul ur
    63.             smr.SetBlendShapeWeight(21, Remap(list[52]));   //mouth_lipCornerPull_l
    64.             smr.SetBlendShapeWeight(22, Remap(list[53]));   //mouth_lipCornerPull_r
    65.             smr.SetBlendShapeWeight(23, Mathf.Max(Remap(list[47]), Remap(list[45])));   //mouth_lipCornerDepress_l mouth_lipCornerDepressFix_l
    66.             smr.SetBlendShapeWeight(24, Mathf.Max(Remap(list[48]), Remap(list[46])));   //mouth_lipCornerDepress_r mouth_lipCornerDepressFix_r
    67.             smr.SetBlendShapeWeight(25, Remap(list[39]));   //mouth_dimple_l
    68.             smr.SetBlendShapeWeight(26, Remap(list[40]));   //mouth_dimple_r
    69.             smr.SetBlendShapeWeight(27, Remap(list[65]));   //mouth_press_l
    70.             smr.SetBlendShapeWeight(28, Remap(list[66]));   //mouth_press_r
    71.             smr.SetBlendShapeWeight(29, Remap(list[36]));   //mouth_chinRaise_d
    72.             smr.SetBlendShapeWeight(30, Remap(list[37]));   //mouth_chinRaise_u
    73.             smr.SetBlendShapeWeight(31, Remap(list[56]));   //mouth_lipStretch_l
    74.             smr.SetBlendShapeWeight(32, Remap(list[57]));   //mouth_lipStretch_r
    75.             smr.SetBlendShapeWeight(33, Remap(list[78]));   //mouth_upperLipRaise_l
    76.             smr.SetBlendShapeWeight(34, Remap(list[79]));   //mouth_upperLipRaise_r
    77.             smr.SetBlendShapeWeight(35, Remap(list[58]));   //mouth_lowerLipDepress_l
    78.             smr.SetBlendShapeWeight(36, Remap(list[59]));   //mouth_lowerLipDepress_r
    79.             smr.SetBlendShapeWeight(37, Mathf.Max(Remap(list[76]), Remap(list[77])));    //mouth_suck_ul mouth_suck_ur
    80.             smr.SetBlendShapeWeight(38, Mathf.Max(Remap(list[74]), Remap(list[75])));    //mouth_suck_dl mouth_suck_dr
    81.             smr.SetBlendShapeWeight(39, Remap(list[35]));   //mouth_chew_c
    82.             smr.SetBlendShapeWeight(40, Remap(list[34]));   //jaw_thrust_c
    83.             smr.SetBlendShapeWeight(41, Remap(list[73]));   //mouth_stretch_c
    84.             smr.SetBlendShapeWeight(42, Remap(list[32]));   //jaw_sideways_l
    85.             smr.SetBlendShapeWeight(43, Remap(list[33]));   //jaw_sideways_r
    86.             smr.SetBlendShapeWeight(44, Remap(list[38]));   //brow_raise_c
    87.             smr.SetBlendShapeWeight(45, Remap(list[22]));   //eye_blink2_r
    88.             smr.SetBlendShapeWeight(46, Remap(list[21]));   //eye_blink2_l
    89.             smr.SetBlendShapeWeight(47, Remap(list[0]));   //brow_lower_l
    90.             smr.SetBlendShapeWeight(48, Remap(list[27]));   //brow_lower_r
    91.             smr.SetBlendShapeWeight(49, Mathf.Max(Remap(list[31]), Remap(list[29])));   //eye_downLidRaise_r eye_upLidRaise_r
    92.             smr.SetBlendShapeWeight(50, Mathf.Max(Remap(list[30]), Remap(list[28])));   //eye_downLidRaise_l eye_upLidRaise_l
    93.             yield return new WaitForSeconds(.07f);
    94.         }
    95.         coroutine = null;
    96.     }
    97.     private float Remap(float v)
    98.     {
    99.         return v * 100f;
    100.     }
    101.  
    102.     private void OnGUI()
    103.     {
    104.         GUI.enabled = coroutine == null;
    105.         if (GUILayout.Button("Begin", GUILayout.Width(200f), GUILayout.Height(50f)))
    106.         {
    107.             coroutine = StartCoroutine(ExecuteCoroutine());
    108.         }
    109.         GUI.enabled = coroutine != null;
    110.         if (GUILayout.Button("Stop", GUILayout.Width(200f), GUILayout.Height(50f)))
    111.         {
    112.             StopCoroutine(coroutine);
    113.             coroutine = null;
    114.         }
    115.     }
    116. }

  • 相关阅读:
    搜狗微信APP逆向（二）so层
    链路层3：VLAN的配置与分析
    JavaScript基础05——字面量、变量介绍及变量基本使用
    2024初三集训模拟测试1
    多层感知机（PyTorch）
    数据湖：OPPO数据湖统一存储技术实践
    【Python基础知识】（16）Range的经典用例
    XV6 Network解析-1
    ⑲霍兰德ES*如何选专业？高考志愿填报选专业
    ROS2专题【01】：win10上安装ROS2
  • 原文地址：https://blog.csdn.net/qq_42139931/article/details/125891735