基于yolov5的手掌特效app（安卓）

1.前言：

今天给大家分享一个好玩的项目app，该app和上一个博客的效果有点类似，不同的是在该博客中加入了目标检测部分。首先来看一下效果预览：

2.简介：

介绍一下主要流程，首先使用yolov5训练一个识别手掌的模型，然后准备好需要播放的特效视频。获取摄像头画面进行目标检测，检测到手掌正面以后，在手掌的矩形框内播放特效视频。

3.训练模型

使用yolov5训练模型网上教程有很多，这里不再详细叙述。使用的数据集是本人使用手机拍摄的，一共400张左右，一共分为3个类别，分别是拳头；正面；反面。数据集已经标注完成。如有需要可前往下载
数据集下载地址：
链接：https://pan.baidu.com/s/1BgJsbswReeKHTWrpGDMviQ
提取码：mk9o

4.特效视频

使用的特效视频是直接在快手和抖音上面下载的。部分特效视频如下所示：
螺旋丸特效视频：

5.代码

Mainactivity.java代码：

package com.myapp.handfire2;

import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import androidx.camera.core.Camera;
import androidx.camera.core.CameraControl;
import androidx.camera.core.CameraSelector;
import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageCapture;
import androidx.camera.core.ImageProxy;
import androidx.camera.core.Preview;
import androidx.camera.lifecycle.ProcessCameraProvider;
import androidx.camera.view.PreviewView;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;

import android.annotation.SuppressLint;
import android.content.pm.PackageManager;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.ImageFormat;
import android.graphics.PixelFormat;
import android.graphics.Rect;
import android.graphics.YuvImage;
import android.media.MediaPlayer;
import android.opengl.GLSurfaceView;
import android.os.Bundle;
import android.os.Message;
import android.os.SystemClock;
import android.util.Log;
import android.view.View;
import android.widget.AdapterView;
import android.widget.Button;
import android.widget.Spinner;
import android.widget.TextView;

import com.google.common.util.concurrent.ListenableFuture;
import com.myapp.handfire2.databinding.ActivityMainBinding;

import org.jetbrains.annotations.NotNull;

import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;


public class MainActivity extends AppCompatActivity {

    // Used to load the 'native-lib' library on application startup.
    static {
        System.loadLibrary("native-lib");
    }

    private ActivityMainBinding binding;

    public static MyRenderer myRenderer;
    private GLSurfaceView glSurfaceView;
    private Spinner spinner;

    private Myview myview;

    private final String[] REQUIRED_PERMISSIONS = new String[]{"android.permission.CAMERA", "android.permission.WRITE_EXTERNAL_STORAGE"};
    private int REQUEST_CODE_PERMISSIONS = 1001;

    PreviewView pvCameraPreview;
    private Yolov5n Yolov5Net=new Yolov5n();
    private int[] texiao={R.raw.chuizi,R.raw.fire,R.raw.hunhuan,R.raw.qianniao,R.raw.wanzi,R.raw.wanzi2};

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);

        boolean init=Yolov5Net.Init(getAssets());


        binding = ActivityMainBinding.inflate(getLayoutInflater());
        setContentView(binding.getRoot());
        myview=binding.view;
        spinner=binding.spin;

        spinner.setOnItemSelectedListener(new AdapterView.OnItemSelectedListener() {
            @Override
            public void onItemSelected(AdapterView<?> parent, View view, int position, long id) {
                myRenderer.start(texiao[position]);

            }

            @Override
            public void onNothingSelected(AdapterView<?> parent) {


            }
        });

        glSurfaceView=binding.glview;
        glSurfaceView.setEGLContextClientVersion(2);
        //背景透明
        glSurfaceView.setEGLConfigChooser(8,8,8,8,16,0);
        glSurfaceView.getHolder().setFormat(PixelFormat.TRANSLUCENT);
        glSurfaceView.setZOrderOnTop(true);
        MyRenderer.myMediaPlayer = new MediaPlayer();



        myRenderer=new MyRenderer(this);
        glSurfaceView.setRenderer(myRenderer);
        //myRenderer.start(texiao[0]);
//
//        button=binding.btn;
//        button.setOnClickListener(new View.OnClickListener() {
//            @Override
//            public void onClick(View v) {
//                //myRenderer.start("");
//                Log.i("aa","click");
//
//            }
//        });

        pvCameraPreview=binding.preview;

        //获取权限
        if(allPermissionsGranted()){
            startCamera(); //start camera if permission has been granted by user
        } else{
            ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS);
        }


    }
    //获取权限函数
    private boolean allPermissionsGranted() {
        for (String permission : REQUIRED_PERMISSIONS) {
            if (ContextCompat.checkSelfPermission(this, permission) != PackageManager.PERMISSION_GRANTED) {
                return false;
            }
        }
        return true;
    }

    /**
     * 开始预览
     */
    private CameraControl cameraControl;
    private Executor executor = Executors.newSingleThreadExecutor();
    private void startCamera() {
        ListenableFuture<ProcessCameraProvider> cameraProviderFuture =
                ProcessCameraProvider.getInstance(this);
        ((ListenableFuture<?>) cameraProviderFuture).addListener(new Runnable() {
            @SuppressLint("RestrictedApi")
            @Override
            public void run() {
                try {

                    ImageAnalysis imageAnalysis = new ImageAnalysis.Builder()
                            //.setBackpressureStrategy(ImageAnalysis.STRATEGY_BLOCK_PRODUCER)//阻塞模式
                            //.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888)
                            //.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
                            .build();
                    imageAnalysis.setAnalyzer(executor, new ImageAnalysis.Analyzer() {
                        @Override
                        public void analyze(@NonNull @NotNull ImageProxy image) {

                            //yuv图像数据转bitmap
                            ImageProxy.PlaneProxy[] planes = image.getPlanes();
                            //cameraX 获取yuv
                            ByteBuffer yBuffer = planes[0].getBuffer();
                            ByteBuffer uBuffer = planes[1].getBuffer();
                            ByteBuffer vBuffer = planes[2].getBuffer();

                            int ySize = yBuffer.remaining();
                            int uSize = uBuffer.remaining();
                            int vSize = vBuffer.remaining();

                            byte[] nv21 = new byte[ySize + uSize + vSize];

                            yBuffer.get(nv21, 0, ySize);
                            vBuffer.get(nv21, ySize, vSize);
                            uBuffer.get(nv21, ySize + vSize, uSize);
                            //获取yuvImage
                            YuvImage yuvImage = new YuvImage(nv21, ImageFormat.NV21, image.getWidth(), image.getHeight(), null);
                            //输出流
                            ByteArrayOutputStream out = new ByteArrayOutputStream();
                            //压缩写入out
                            yuvImage.compressToJpeg(new Rect(0, 0, yuvImage.getWidth(), yuvImage.getHeight()), 50, out);
                            //转数组
                            byte[] imageBytes = out.toByteArray();
                            //生成bitmap
                            Bitmap bmp = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.length);

//                            Bitmap rotateBitmap=null;

                            Dector(bmp);

                            image.close();
                        }
                    });
                    //将相机的生命周期和activity的生命周期绑定，camerax 会自己释放
                    ProcessCameraProvider cameraProvider = cameraProviderFuture.get();
                    Preview preview = new Preview.Builder().build();
                    //创建图片的 capture
                    ImageCapture mImageCapture = new ImageCapture.Builder()
                            .setFlashMode(ImageCapture.FLASH_MODE_OFF)
                            .build();
                    //选择前置摄像头
                    CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build();
                    // Unbind use cases before rebinding
                    cameraProvider.unbindAll();

                    // Bind use cases to camera
                    //参数中如果有mImageCapture才能拍照，否则会报下错
                    //Not bound to a valid Camera [ImageCapture:androidx.camera.core.ImageCapture-bce6e930-b637-40ee-b9b9-
                    Camera camera = cameraProvider.bindToLifecycle(MainActivity.this, cameraSelector, preview, imageAnalysis,mImageCapture);

                    cameraControl = camera.getCameraControl();
                    cameraControl.setLinearZoom(0.1f);
                   preview.setSurfaceProvider(pvCameraPreview.getSurfaceProvider());
                } catch (ExecutionException e) {
                    e.printStackTrace();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }, ContextCompat.getMainExecutor(this));
    }
    private void setFrame() {

    }

    /**
     * 检测函数
     */
    private void  Dector(Bitmap bitmap) {
        // draw objects on bitmap
        Bitmap rgba = bitmap.copy(Bitmap.Config.ARGB_8888, true);
        Yolov5n.Obj[] objects = Yolov5Net.Detect(rgba, false, 0.25f);
        float width = rgba.getWidth();
        float height = rgba.getHeight();
        myview.draws(objects,width,height,true,false);
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242

布局代码：

<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    tools:context=".MainActivity">


    <androidx.camera.view.PreviewView
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:id="@+id/preview"
        />


    <android.opengl.GLSurfaceView
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:id="@+id/glview"
        />

    <com.myapp.handfire2.Myview
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:id="@+id/view"
        />
    <Spinner
        android:layout_width="100dp"
        android:layout_height="wrap_content"
        android:layout_gravity="right"
        android:id="@+id/spin"
        android:entries="@array/texiao"
        android:background="@mipmap/xia"
        android:alpha="0.7"
        
        />

FrameLayout>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

6.app下载地址及项目下载地址

如果手机端打不开，可以使用电脑试一下。
项目地址：https://gitee.com/mqwdasddqw/HandFire2

app下载地址：https://www.pgyer.com/bjSZ