• android可见即可说实现方案

    • 依赖于科大讯飞的asr识别能力，使用Android无障碍服务获取页面文本作为热词，注册到讯飞api，注册过后语音识别到热词的asr返回,利用WindowManager和无障碍的点击实现可见即可说功能
    ##  无障碍服务获取需要注册的热词
```
package com..model;

import android.accessibilityservice.AccessibilityService;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.content.ServiceConnection;
import android.graphics.Rect;
import android.os.IBinder;
import android.util.Log;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityNodeInfo;
import androidx.lifecycle.Observer;
import .HotWordsBean;
import .GsonUtil;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

public class MyAccessibilityService extends AccessibilityService implements IAccessibilityHotWord {

    private String TAG = MyAccessibilityService.class.getSimpleName();
    private StringBuilder hotWords = new StringBuilder();
    private VrSpService vrSpeechService;
    private HotWordsBean hotWordsBean = new HotWordsBean();
    private AccessibilityNodeInfo rootInActiveWindow;
    private Set accessibilityNodeInfoSet = new HashSet<>();
    HotWordsBean.UserDataBean userDataBean = new HotWordsBean.UserDataBean();
    HotWordsBean.UserDataBean.CMD cmd = new HotWordsBean.UserDataBean.CMD();
    private String hotWordJsonString = "";
    private Context context;

    @Override
    public void onCreate() {
        super.onCreate();
        Log.i(TAG, "------------ super.onCreate --------------------: ");
        bindVrSpeechService();
        context = this;

        HotWordReceiver.hotWordLiveData.observeForever( new Observer() {
            @Override
            public void onChanged(String hotWord) {
                Log.d(TAG, "onChanged: ytf  ------------ hotWord change :" + hotWord);
                if (accessibilityNodeInfoSet.size() > 0) {
                    for (AccessibilityNodeInfo nodeInfo : accessibilityNodeInfoSet) {
                        if (nodeInfo.getText() != null && nodeInfo.getText().toString().equalsIgnoreCase(hotWord)) {
                            Log.d(TAG, "ytf, hotWord Shot:" + hotWord);
                            handlePerformAction(hotWord);
                        }
                    }
                    // 通过无障碍服务设置seekbar 进度值
                      if ("android.widget.SeekBar".equals(nodeInfo.getClassName())) {
                        // AccessibilityAction: ACTION_SET_PROGRESS - null
                        // mStateDescription 76%
                        Bundle arguments = new Bundle();
                        arguments.putFloat(AccessibilityNodeInfo.ACTION_ARGUMENT_PROGRESS_VALUE, 50.0f);
                        nodeInfo.performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_PROGRESS.getId(), arguments);
                        Log.d(TAG, "onChanged: ytf,seekBar " + nodeInfo);
                        // 反射获取seekbar当前进度progress
                        // readAttributeValue(nodeInfo);
                         try {
                            @SuppressLint("BlockedPrivateApi") Field field = nodeInfo.getClass().getDeclaredField("mStateDescription");
                            //设置对象的访问权限，保证对private的属性的访问
                            field.setAccessible(true);
                            Log.d(TAG, "onChanged: ytf,stateDescription =  " + field.get(nodeInfo));
                        } catch (Exception e) {
                            Log.e(TAG, "onChanged: ytf,========== " + e.toString());
                        }
                        
                    }
                } else {
                    Log.d(TAG, "ytf hotWord Shot: accessibilityNodeInfoSet size = " + accessibilityNodeInfoSet.size());
                }
            }
        });
    }




    @Override
    public void onAccessibilityEvent(AccessibilityEvent accessibilityEvent) {
        String packageName = accessibilityEvent.getPackageName() == null ? "" : accessibilityEvent.getPackageName().toString();
        if (!"com.saicmotor.settings".equals(packageName)) {
            return;
        }
        int eventType = accessibilityEvent.getEventType();
//        Log.d(TAG, "ytf,onAccessibilityEvent [eventType: " + eventType + "], [ packageName: " + packageName + "]");

        switch (eventType) {
//            case AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED:
            case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED:
            case AccessibilityEvent.TYPE_VIEW_CLICKED:
                accessibilityNodeInfoSet.clear();
                hotWords.setLength(0);
                rootInActiveWindow = getRootInActiveWindow();
                if (null == rootInActiveWindow) {
                    Log.d(TAG, "ytf, onAccessibilityEvent: rootInActiveWindow == null");
                    return;
                } else {
                    recycle(rootInActiveWindow);
                }
                String[] splitHotWords = hotWords.toString().split(",");

                //vrSpeechService.notifyHotWordLoad(GsonUtil.HOT_WORD_TEST_1);

                hotWordsBean.setHotWords(splitHotWords);
                cmd.setActiveStatus("");
                userDataBean.setCmd(cmd);
                hotWordsBean.setUserData(userDataBean);
                try {
                    hotWordJsonString = GsonUtil.getInstance().getGson().toJson(hotWordsBean);
                    if (hotWordJsonString != null) {
                        vrSpeechService.notifyHotWordLoad(hotWordJsonString);
                        Log.d(TAG, "onAccessibilityEvent: ytf, hotWordJsonString = " + hotWordJsonString);
                    }
                } catch (Exception e) {
                    Log.e(TAG, "onAccessibilityEvent: ytf, e: " + e.toString());
                }

                break;
            default:
                break;
        }

    }

    private void recycle(AccessibilityNodeInfo info) {
        if (info.getChildCount() == 0) {
            if ("android.widget.SeekBar".equals(info.getClassName())) {
//                Class aClass = info.getClass();
//                Log.d(TAG, "recycle: ytf aClass = " + aClass.getSimpleName());
//                SeekBar seekBar = (SeekBar) info.getClassName();
//                Log.d(TAG, "recycle: ytf, SeekBar 调节前：" + seekBar.getProgress() + ", getViewIdResourceName:" + info.getViewIdResourceName());
//                Bundle bundle = new Bundle();
//                bundle.putFloat(AccessibilityNodeInfo.ACTION_ARGUMENT_PROGRESS_VALUE, 50.0f);
//                seekBar.performAccessibilityAction(R.id.accessibilityActionSetProgress, bundle);
//                Log.d(TAG, "recycle: ytf, SeekBar 调节后：" + seekBar.getProgress() + ", getViewIdResourceName:" + info.getViewIdResourceName());
            } else if ("android.widget.ScrollView".contentEquals(info.getClassName())) {
//                Log.d(TAG, "recycle: ytf, android.widget.ScrollView ACTION_SCROLL_FORWARD");
//                info.performAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD);
            }
//            Log.i(TAG, "recycle ytf, [ClassName: " + info.getClassName() + "], [Text: " + info.getText() + "], [resId: " + info.getViewIdResourceName() + "]");

            if (null != info.getText()) {
                String text = info.getText().toString();
                hotWords.append(text + ",");
                accessibilityNodeInfoSet.add(info);
            }


        } else {
            for (int i = 0; i < info.getChildCount(); i++) {
                if (info.getChild(i) != null) {
//                    Log.d(TAG, "ytf 容器: [" + info.getClassName() + "], [resId:" + info.getViewIdResourceName() + "]");
                    recycle(info.getChild(i));
                }
            }
        }
    }

    private void handlePerformAction(String targetHotWord) {
        Log.d(TAG, "handlePerformAction: ytf, accessibilityNodeInfoSet.size = " + accessibilityNodeInfoSet.size());
        for (AccessibilityNodeInfo nodeInfo : accessibilityNodeInfoSet) {
//            Log.d(TAG, "ytf handlePerformAction:  nodeInfo.getText().toString() = " + nodeInfo.getText().toString() + ", targetHotWord = " + targetHotWord);
            if (nodeInfo.getText().toString().equalsIgnoreCase(targetHotWord)) {
                Log.d(TAG, "ytf, 命中可见即可说 handlePerformAction: " + nodeInfo.getText().toString());
                forceClick(nodeInfo);
//              nodeInfo.performAction(AccessibilityNodeInfo.ACTION_CLICK);

            }
        }
    }

    private void forceClick(AccessibilityNodeInfo nodeInfo) {
        Log.d(TAG, "forceClick: ytf,------------");
        try {
            Rect rect = new Rect();
            nodeInfo.getBoundsInScreen(rect);
            Log.d(TAG, "ytf, forceClick: " + rect.left + " " + rect.top + " " + rect.right + " " + rect.bottom);
            int x = (rect.left + rect.right) / 2;
            int y = (rect.top + rect.bottom) / 2;
            String cmd = "input tap " + String.valueOf(x) + " " + String.valueOf(y);
            ProcessBuilder builder = new ProcessBuilder();
            String[] order = {
                    "input",
                    "tap",
                    String.valueOf(x),
                    String.valueOf(y)
            };

            try {
                builder.command(order).start();
                Log.d(TAG, "ytf, forceClick: [ " + x + ", " + y + "]");
            } catch (IOException e) {
                Log.d(TAG, "ytf, forceClick: error: " + e.toString());
            }

        } catch (Exception e) {
            Log.e(TAG, "ytf,error forceClick: " + e.toString());
        }
    }

    @Override
    public void onInterrupt() {
        Log.i(TAG, "ytf, onInterrupt");
    }

    private void bindVrSpeechService() {
        Intent intent = new Intent(getApplicationContext(), VrSpeechService.class);
        boolean result = bindService(intent, serviceConnection, Context.BIND_AUTO_CREATE);
        Log.d(TAG, "vrSpeechService: ytf bind result:" + result);
    }

    private ServiceConnection serviceConnection = new ServiceConnection() {

        @Override
        public void onServiceConnected(ComponentName componentName, IBinder iBinder) {
            try {
                vrSpeechService = ((VrSpeechService.LocalBinder) iBinder).getService();
                Log.d(TAG, "onServiceConnected: ytf, vrSpeechService bind ");
                vrSpeechService.setAccessibilityHotWord((IAccessibilityHotWord) context);
            } catch (Exception e) {
                e.printStackTrace();
                Log.e(TAG, "ytf, onServiceConnected: " + e.toString() );
            }
        }

        @Override
        public void onServiceDisconnected(ComponentName componentName) {

        }
    };

    @Override
    public void hotWordShot(String hotWord) {
//        Log.d(TAG, "ytf, hotWordShot: " + hotWord);
        if (accessibilityNodeInfoSet.size() > 0) {
            for (AccessibilityNodeInfo nodeInfo : accessibilityNodeInfoSet) {
                if (nodeInfo.getText().toString().equalsIgnoreCase(hotWord)) {
                    Log.d(TAG, "ytf, hotWordShot:" + hotWord);
                    handlePerformAction(hotWord);
                }
            }
        } else {
            Log.d(TAG, "ytf hotWordShot: accessibilityNodeInfoSet size = " + accessibilityNodeInfoSet.size());
        }
    }
}

```

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    • 39
    • 40
    • 41
    • 42
    • 43
    • 44
    • 45
    • 46
    • 47
    • 48
    • 49
    • 50
    • 51
    • 52
    • 53
    • 54
    • 55
    • 56
    • 57
    • 58
    • 59
    • 60
    • 61
    • 62
    • 63
    • 64
    • 65
    • 66
    • 67
    • 68
    • 69
    • 70
    • 71
    • 72
    • 73
    • 74
    • 75
    • 76
    • 77
    • 78
    • 79
    • 80
    • 81
    • 82
    • 83
    • 84
    • 85
    • 86
    • 87
    • 88
    • 89
    • 90
    • 91
    • 92
    • 93
    • 94
    • 95
    • 96
    • 97
    • 98
    • 99
    • 100
    • 101
    • 102
    • 103
    • 104
    • 105
    • 106
    • 107
    • 108
    • 109
    • 110
    • 111
    • 112
    • 113
    • 114
    • 115
    • 116
    • 117
    • 118
    • 119
    • 120
    • 121
    • 122
    • 123
    • 124
    • 125
    • 126
    • 127
    • 128
    • 129
    • 130
    • 131
    • 132
    • 133
    • 134
    • 135
    • 136
    • 137
    • 138
    • 139
    • 140
    • 141
    • 142
    • 143
    • 144
    • 145
    • 146
    • 147
    • 148
    • 149
    • 150
    • 151
    • 152
    • 153
    • 154
    • 155
    • 156
    • 157
    • 158
    • 159
    • 160
    • 161
    • 162
    • 163
    • 164
    • 165
    • 166
    • 167
    • 168
    • 169
    • 170
    • 171
    • 172
    • 173
    • 174
    • 175
    • 176
    • 177
    • 178
    • 179
    • 180
    • 181
    • 182
    • 183
    • 184
    • 185
    • 186
    • 187
    • 188
    • 189
    • 190
    • 191
    • 192
    • 193
    • 194
    • 195
    • 196
    • 197
    • 198
    • 199
    • 200
    • 201
    • 202
    • 203
    • 204
    • 205
    • 206
    • 207
    • 208
    • 209
    • 210
    • 211
    • 212
    • 213
    • 214
    • 215
    • 216
    • 217
    • 218
    • 219
    • 220
    • 221
    • 222
    • 223
    • 224
    • 225
    • 226
    • 227
    • 228
    • 229
    • 230
    • 231
    • 232
    • 233
    • 234
    • 235
    • 236
    • 237
    • 238
    • 239
    • 240
    • 241
    • 242
    • 243
    • 244
    • 245
    • 246
    • 247
    • 248
    • 249
    • 250
    • 251
    • 252
    • 253
    • 254
    • 255

    • 清单文件：



                 

       


           
               
           
       

      • 1
      • 2
      • 3
      • 4
      • 5
      • 6
      • 7
      • 8
      • 9
      • 10
      • 11
      • 12
      • 13

    • @xml/accessibility

    

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7

    • 模拟asr热词命中
      /**
      * @Author yangtianfu
      * @Date 2023/9/15 13:05
      * @Describe 监听热词回传执行可见可说
      * adb shell am broadcast -a com.saicmotor.voiceservice.hotword -n com.saicmotor.voiceservice/.model.HotWordReceiver --es hotWord “sound”
      */
      public class HotWordReceiver extends BroadcastReceiver {
      private static final String TAG = “HotWordReceiver”;
      private final String ACTION_HOT_WORD_RECEIVER = “com.saicmotor.voiceservice.hotword”;
      // private IAccessibilityHotWord iAccessibilityHotWord;
      public static MutableLiveData hotWordLiveData = new MutableLiveData<>();

         @Override
   public void onReceive(Context context, Intent intent) {
       String action = intent.getAction();
       Log.d(TAG, "ytf, onReceive: intent,action = " + action);
       if (ACTION_HOT_WORD_RECEIVER.equals(action)) {
           String hotWord = intent.getStringExtra("hotWord");
           Log.d(TAG, "ytf, onReceive: com.saicmotor.voiceservice.hotword :" + hotWord);
           hotWordLiveData.postValue(hotWord);
       }

   }

      • 1
      • 2
      • 3
      • 4
      • 5
      • 6
      • 7
      • 8
      • 9
      • 10
      • 11

      }

    ## 科大讯飞注册热词
    热词格式：
        public static final String HOT_WORD_TEST_1 = "{\n" +
                "    \"HotWords\":[\"High\"],\n" +
                "    \"UserData\":{\n" +
                "        \"cmd\":{\n" +
                "            \"activeStatus\":\"bg\",\n" +
                "            \"data\":{\n" +
                "\n" +
                "            },\n" +
                "            \"sceneStatus\":\"default\"\n" +
                "        }\n" +
                "    }\n" +
                "}";
    int result = libisssr.uploadData(hotWords, 2);
##  VuiService.java语音Vui服务（带有语音形象的app）

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    透明activity无法跨应用实现点击穿透效果，会导致可见即可说点击无效果，需要在service中使用WindowManager的addview方法，把语音的app作为view添加到WindowManager中,这样就可以实现语音app全透明状态下识别到asr之后可以利用Android无障碍服务去点击指定位置或者指定控件。

public class VoiceVuiService extends Service {
 private WindowManager.LayoutParams mParams;
    private WindowManager mWindowManager;
    private ImageView voiceImage;
    private TextView textView;

 @Override
    public void onCreate() {
        mParams = new WindowManager.LayoutParams();
        //设置type.系统提示型窗口，一般都在应用程序窗口之上.
        mParams.type = WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY;
//        mParams.type = WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY;

        //设置效果为背景透明.
//        mParams.format = PixelFormat.RGBA_8888;
        mParams.format = PixelFormat.TRANSLUCENT;
        //设置flags.不可聚焦及不可使用按钮对悬浮窗进行操控.
//        mParams.flags = WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE | WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE |
//                WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL | WindowManager.LayoutParams.FIRST_SYSTEM_WINDOW;
        mParams.flags = WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE | WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE;
        mParams.alpha = 0.8f;
        mWindowManager = (WindowManager) getSystemService(Context.WINDOW_SERVICE);
        DisplayMetrics dm = new DisplayMetrics();
        mWindowManager.getDefaultDisplay().getMetrics(dm);
        mParams.width = 136;
        mParams.height = 136;
//        mParams.x = 100;
//        mParams.y = 100;
        voiceImage = new ImageView(this);
        voiceImage.setBackground(getResources().getDrawable(R.mipmap.assistant100025));

        textView = new TextView(this);
        textView.setTextSize(36);
        textView.setTextColor(Color.RED);
        textView.setText("语音形象");


//        vrView = new HSPortraitVrViewForA11V(getApplicationContext());
//        vrView.startFlipping();


        Log.d(TAG, "initView: ytf, dm.widthPixels = " + dm.widthPixels + ",mParams.height = " + mParams.height);
        mParams.gravity = Gravity.TOP;
        LogUtils.i(TAG, "onCreate.....");
        super.onCreate();


-----------------------


  @Override
    public int onStartCommand(Intent intent, int flags, int startId) {
        LogUtils.i(TAG, "onStartCommand,LteService onStartCommand");
        // Android 8以上特殊处理
        setNotificationChannel();
//        mWindowManager.addView(voiceImage, mParams);
//        mWindowManager.addView(textView, mParams);

//        if(CommonUtils.isUseHalfServiceMode()){
//            if(offlineAgentService == null){
//                bindHalfEngineService();
//            }
//        }
        return START_STICKY;
    }
-------------------------------
    @Override
    public void onDestroy() {

//        if (voiceImage.getParent() != null) {
//            mWindowManager.removeView(voiceImage);
//        }
//        if (textView.getParent() != null) {
//            mWindowManager.removeView(textView);
//        }
        super.onDestroy();
  }

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    • 39
    • 40
    • 41
    • 42
    • 43
    • 44
    • 45
    • 46
    • 47
    • 48
    • 49
    • 50
    • 51
    • 52
    • 53
    • 54
    • 55
    • 56
    • 57
    • 58
    • 59
    • 60
    • 61
    • 62
    • 63
    • 64
    • 65
    • 66
    • 67
    • 68
    • 69
    • 70
    • 71
    • 72
    • 73
    • 74
    • 75
    • 76
    • 77
    • 78
    • 79
    private static void readAttributeValue(Object obj) {
        String nameVlues = "";
        //得到class
        Class cls = obj.getClass();
        //得到所有属性
        Field[] fields = cls.getDeclaredFields();
        for (int i = 0; i < fields.length; i++) {//遍历
            try {
                //得到属性
                Field field = fields[i];
                //打开私有访问
                field.setAccessible(true);
                //获取属性
                String name = field.getName();
                //获取属性值
                Object value = field.get(obj);
                //一个个赋值
                nameVlues += field.getName() + ":" + value + ",";
            } catch (IllegalAccessException e) {
                e.printStackTrace();
            }
        }
        //获取最后一个逗号的位置
        int lastIndex = nameVlues.lastIndexOf(",");

        //不要最后一个逗号","
        String result = nameVlues.substring(0, lastIndex);
        System.out.println("ytf, 反射获取：" + result);
    }

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
  • 相关阅读:
    高等数学（第七版）同济大学 习题3-2 个人解答
    Leetcode Top100(23)环形链表
    nginx出现504 Gateway Time-out错误的原因分析及解决
    react hook:useMemo
    设置Redis最大占用内存
    C/C++程序的内存开辟
    【c++】虚函数和虚函数表（多态）
    APT预警攻击平台截获Nday
    ip组成及解决
    【Java小知识点】类加载器的区别
  • 原文地址：https://blog.csdn.net/ytfunnysite/article/details/132989563