Python实现APP UI自动化以及OpenCV图像识别元素

 OpenCV图像识别元素代码

# -*- encoding=utf-8 -*-

__author__ = 'Jeff.xie'

import cv2
import sys

def _tran_canny(image):
    """消除噪声"""
    image = cv2.GaussianBlur(image, (3, 3), 0)
    return cv2.Canny(image, 50, 150)

def get_center_location(img_slider_path,image_background_path,x_percent,y_percent):

    # java传递过来的参数都是str类型，所以需要强转成int类型
    xper = int(x_percent)
    yper = int(y_percent)

    # # 参数0是灰度模式
    image = cv2.imread(img_slider_path, 0)
    template = cv2.imread(image_background_path, 0)

    # 寻找最佳匹配
    res = cv2.matchTemplate(_tran_canny(image), _tran_canny(template), cv2.TM_CCOEFF_NORMED)
    # 最小值，最大值，并得到最小值, 最大值的索引
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)

    #获得背景图像高和宽
    src_img = cv2.imread(image_background_path,cv2.IMREAD_GRAYSCALE)
    h,w = src_img.shape

    #获得需要寻找图像高和宽
    des_img = cv2.imread(img_slider_path,cv2.IMREAD_GRAYSCALE)
    des_img_h,des_img_w = des_img.shape

    trows,tcols = image.shape[:2]  #获得图片的宽度,两种方式都可以
    top_left = max_loc[0]  # 横坐标
    # 展示圈出来的区域
    x, y = max_loc
    # max_loc获取x,y位置坐标，
    xLocation = x + int(des_img_w*xper/100)
    yLocation = y + int(des_img_h*yper/100)
    print("xLocation: "+str(xLocation))
    print("yLocation: "+str(yLocation))
    return xLocation,yLocation

def get_location_percent(img_slider_path,image_background_path,x_percent,y_percent):

    # java传递过来的参数都是str类型，所以需要强转成int类型
    xper = int(x_percent)
    yper = int(y_percent)
    # # 参数0是灰度模式
    image = cv2.imread(img_slider_path, 0)
    template = cv2.imread(image_background_path, 0)

    # 寻找最佳匹配
    res = cv2.matchTemplate(_tran_canny(image), _tran_canny(template), cv2.TM_CCOEFF_NORMED)
    # 最小值，最大值，并得到最小值, 最大值的索引
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)

    #获得背景图像高和宽
    src_img = cv2.imread(image_background_path,cv2.IMREAD_GRAYSCALE)
    h,w = src_img.shape
    print(h)
    print(w)

    #获得需要寻找图像高和宽
    des_img = cv2.imread(img_slider_path,cv2.IMREAD_GRAYSCALE)
    des_img_h,des_img_w = des_img.shape

    trows,tcols = image.shape[:2]  #获得图片的宽度,两种方式都可以
    top_left = max_loc[0]  # 横坐标
    # 展示圈出来的区域
    x, y = max_loc
    # max_loc获取x,y位置坐标，
    xLocation = x + int(des_img_w*xper/100)
    yLocation = y + int(des_img_h*yper/100)

    x_loc_percent= int(xLocation*100/w)
    y_loc_percent= int(yLocation*100/h)
    print("x_loc_percent: "+str(x_loc_percent))
    print("y_loc_percent: "+str(y_loc_percent))
    return x_loc_percent,y_loc_percent

OpenCV图像识别元素代码第二种方法

# -*- encoding=utf-8 -*-
__author__ = 'Jeff.xie'

import cv2
import sys

def get_location_percent(img_slider_path, image_background_path, x_percent, y_percent):
    xper = int(x_percent)
    yper = int(y_percent)
    img1=cv2.imread(image_background_path)  #大图
    img2=cv2.imread(img_slider_path)
    #使用SIFT算法获取图像特征的关键点和描述符
    sift=cv2.xfeatures2d.SIFT_create()
    kp1,des1=sift.detectAndCompute(img1,None)
    kp2,des2=sift.detectAndCompute(img2,None)

    #定义FLANN匹配器
    indexParams=dict(algorithm=0,trees=10)
    searchParams=dict(checks=50)
    flann=cv2.FlannBasedMatcher(indexParams,searchParams)
    #使用KNN算法实现图像匹配，并对匹配结果排序
    matches=flann.knnMatch(des1,des2,k=2)
    matches=sorted(matches,key=lambda x:x[0].distance)

    #去除错误匹配，0.5是系数，系数大小不同，匹配的结果页不同
    goodMatches=[]
    for m,n in matches:
        if m.distance<0.5*n.distance:
            goodMatches.append(m)
    #获取某个点的坐标位置
    #index是获取匹配结果的中位数
    index=int(len(goodMatches)/2)
    #queryIdx是目标图像的描述符索引
    x,y=kp1[goodMatches[0].queryIdx].pt  #获取小图片中心点
    #将坐标位置勾画在2.png图片上，并显示
    print(x)
    print(y)
    src_img = cv2.imread(image_background_path,cv2.IMREAD_GRAYSCALE)
    height,width = src_img.shape
    # print("width:",width)
    # print("height:",height)
    des_img = cv2.imread(img_slider_path,cv2.IMREAD_GRAYSCALE)
    des_height,des_width = des_img.shape
    # print("des_width:",des_width)
    # print("des_height:",des_height)

    x1 = int(x-des_width/2)
    y1 = int(y-des_height/2)
    x2 = int(x+des_width/2)
    y2 = int(y+des_height/2)
    click_x = x1+des_width*(xper/100)
    click_y = y1+des_width*(yper/100)
    # print("click_x",click_x)
    # print("click_y",click_y)
    x_loc_percent = int(click_x/width*100)
    y_loc_percent = int(click_y/height*100)
    print("x_loc_percent: "+str(x_loc_percent))
    print("y_loc_percent: "+str(y_loc_percent))
    return x_loc_percent,y_loc_percent

Python实现APP UI自动化Demo

__author__ = 'Jeff.xie'
# coding:utf-8
from time import sleep
from appium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from myopencv.get_location_by_opencv2 import get_location_percent


def click_by_location(driver,xper,yper,width,height):
    x = int(xper*width/100)
    y = int(yper*height/100)
    print("x: ",x)
    print("y: ",y)
    driver.tap([(x,y)],1000)

def click_picture(driver,picture):

    current_page=r"D:/current_page.png"
    driver.get_screenshot_as_file(current_page)
    xper,yper= get_location_percent(picture,current_page,50,50)
    print(xper)
    print(yper)
    sleep(1)
    width = driver.get_window_size()['width']
    height = driver.get_window_size()['height']
    click_by_location(driver,xper,yper,width,height)

#获得机器屏幕大小x,y
def getSize(driver):
    x = driver.get_window_size()['width']
    y = driver.get_window_size()['height']
    return (x, y)

#屏幕向上滑动
def swipeUp(driver,duration):  # duration 滑动时间（默认5毫秒）；
    l = getSize(driver)
    x1 = int(l[0] * 0.5)    #x坐标
    y1 = int(l[1] * 0.75)   #起始y坐标
    y2 = int(l[1] * 0.25)   #终点y坐标
    driver.swipe(x1, y1, x1, y2,duration)

#需要打开Appium桌面端
desired_caps = {}
desired_caps['platformName'] = 'Android'
desired_caps['platformVersion'] = '10'
desired_caps['deviceName'] = 'emulator-5554'
desired_caps['appPackage'] = 'com.android.settings'
desired_caps['appActivity'] = 'com.android.settings.Settings'
desired_caps["resetKeyboard"] = "True"
desired_caps["noReset"] = "True"
driver = webdriver.Remote("http://127.0.0.1:4723/wd/hub",desired_caps)
WebDriverWait(driver,60)
sleep(2)
sleep(10)

driver.update_settings({"getMatchedImageResult": True})
click_picture(driver,"WIFI1.png")
sleep(3)
sleep(5)

if __name__ == '__main__':

    pass