ios免越狱自动化测试脚本开发(10)-屏幕截图与OCR文字识别

OCR测试

在IDEA中运行如下代码：

    logd("开始执行脚本...")

    if (appLaunch("com.ss.iphone.ugc.Aweme")) {
        sleep(5000);
        logd('已启动应用')
    } else {
        logw('应用启动失败')
    }

    //ocr初始化，参数要根据自己的电脑系统和环境进行修改
    let ocrLite = {
        "type": "ocrLite",
        "baseDir": "/Users/feiyun/Downloads/IOS/v2.1.0/center/ios-center-2.1.0/OcrLiteNcnn",
        "cpuType": "Darwin-Lib-CPU"
    }

    let inited = ocr.initOcr(ocrLite)
    logd("初始化结果 -> " + inited);
    if (!inited) {
        loge("error : " + ocr.getErrorMsg());
        return;
    }

    var cap = image.captureFullScreen();   //屏幕截图
    if (cap) {
        var data = file.getSandBoxDir();   //获取当前设备沙盒的文件夹路径
        let path = data + "/Temp/Image/"   //临时图片存储路径
        if (!file.exists(path)) {
            file.mkdirs(path);   //创建临时文件目录
        }

        if (image.saveTo(cap, path + "screen.png")) {
            image.recycle(cap);  //图片回收

            //读取一个bitmap
            let bitmap = image.readBitmap(path + "screen.png");
            if (!bitmap) {
                loge("读取图片失败");
            }

            console.time("1")
            logd("start---ocr");
            // 对图片进行识别
            let result = ocr.ocrBitmap(bitmap, 20 * 1000, {});
            //图片要回收
            image.recycle(bitmap)
            //释放所有资源
            ocr.releaseAll();

            logd(result)
            if (result) {
                logd("ocr结果-》 " + JSON.stringify(result));
                for (var i = 0; i < result.length; i++) {
                    var value = result[i];
                    logd("文字 : " + value.label + " x: " + value.x + " y: " + value.y + " width: " + value.width + " height: " + value.height);
                }
            } else {
                logw("未识别到结果");
            }

            logd("耗时: " + console.timeEnd(1) + " ms")
        } else {
            loge("文件保存失败");
        }
    } else {
        loge("截图失败");
    }

运行日志如下：

[2022-03-26 17:51:54.835]开始编译 second
[2022-03-26 17:51:54.835]没有设置密码，无需加密
[2022-03-26 17:51:54.838]源码编译选项: dex
[2022-03-26 17:51:54.838]DEX模式展示行号: 是
[2022-03-26 17:51:54.839]jsConvert2Jar 开始编译: /Volumes/D/IDEAProjects/second/build/src/js/public.js
[2022-03-26 17:51:54.842]jsConvert2Jar 开始编译: /Volumes/D/IDEAProjects/second/build/src/js/main.js
[2022-03-26 17:51:54.854]uiJSConvert2Dex 开始编译: /Volumes/D/IDEAProjects/second/build/src/layout/ui.js
[2022-03-26 17:51:54.878]【远程设备】脚本开始执行
[2022-03-26 17:51:54.878]运行成功
[2022-03-26 17:51:54.882]【远程设备】开始执行,请稍等
[2022-03-26 17:51:54.888]【远程设备】执行clz:com.js.public
[2022-03-26 17:51:54.890]【远程设备】执行clz:com.js.main
[2022-03-26 17:51:54.890]【远程设备】(js/main.js#13 )检查自动化环境...
[2022-03-26 17:51:54.902]【远程设备】(js/main.js#20 )开始执行脚本...
[2022-03-26 17:52:00.04]【远程设备】(js/main.js#26 )已启动应用
[2022-03-26 17:52:00.39]【远程设备】(js/main.js#38 )初始化结果 -> true
[2022-03-26 17:52:01.178]【远程设备】(js/main.js#61 )start---ocr
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#69 )[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object],[object Object]
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#71 )ocr结果-》 [{"confidence":99.96422,"x":11,"width":138,"y":5,"label":"无SIM卡 令","height":27},{"confidence":99.99506,"x":320,"width":109,"y":4,"label":"下午5:52","height":28},{"confidence":75.275475,"x":591,"width":143,"y":6,"label":"@ 100%国","height":25},{"confidence":99.98155,"x":650,"width":67,"y":64,"label":"搜索","height":36},{"confidence":99.998375,"x":165,"width":341,"y":65,"label":"女子做手术后嘴无法张开","height":33},{"confidence":78.55733,"x":31,"width":277,"y":152,"label":"仙本那马布岛风景","height":34},{"confidence":99.95308,"x":394,"width":205,"y":152,"label":"海景如何拍照","height":31},{"confidence":99.995,"x":31,"width":277,"y":228,"label":"郑州唯爱视觉摄影","height":34},{"confidence":39.28642,"x":314,"width":356,"y":228,"label":"丽剪映天空调色教程","height":34},{"confidence":99.95535,"x":27,"width":711,"y":349,"label":"抖音热榜抗疫榜直播榜音乐榜品牌榜","height":39},{"confidence":85.84744,"x":55,"width":454,"y":458,"label":"不 31省新增本土1280+4320","height":39},{"confidence":99.99988,"x":108,"width":433,"y":546,"label":"机上120人身份已确认 1126w","height":39},{"confidence":96.6226,"x":111,"width":503,"y":628,"label":"今天上海人都在晒这样东西 1102w","height":40},{"confidence":99.99104,"x":111,"width":471,"y":716,"label":"国防部回应美希望中国汲取...","height":36},{"confidence":99.99436,"x":596,"width":84,"y":719,"label":"1058w","height":23},{"confidence":99.97786,"x":56,"width":620,"y":797,"label":"4重庆报告1例本土无症状感染者1039w","height":34},{"confidence":99.455,"x":57,"width":592,"y":881,"label":"5坠机撞击点附近已无生命迹象 1015w","height":33},{"confidence":99.93808,"x":108,"width":518,"y":969,"label":"暂停发售4月8日及以后车票1013w","height":33},{"confidence":99.86972,"x":59,"width":603,"y":1051,"label":"7救援人员用手一点点挖出遗物995.>","height":37},{"confidence":99.97694,"x":111,"width":554,"y":1137,"label":"见义勇为军人张昆获记一等功990.8w","height":33},{"confidence":99.986115,"x":109,"width":554,"y":1218,"label":"已搜寻到一部紧急定位发射仪92.0w","height":39},{"confidence":80.72285,"x":50,"width":472,"y":1299,"label":"10情但民家的直实写昭9557w","height":27}]
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#74 )文字 : 无SIM卡 令 x: 11 y: 5 width: 138 height: 27
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#74 )文字 : 下午5:52 x: 320 y: 4 width: 109 height: 28
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#74 )文字 : @ 100%国 x: 591 y: 6 width: 143 height: 25
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#74 )文字 : 搜索 x: 650 y: 64 width: 67 height: 36
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#74 )文字 : 女子做手术后嘴无法张开 x: 165 y: 65 width: 341 height: 33
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#74 )文字 : 仙本那马布岛风景 x: 31 y: 152 width: 277 height: 34
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#74 )文字 : 海景如何拍照 x: 394 y: 152 width: 205 height: 31
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#74 )文字 : 郑州唯爱视觉摄影 x: 31 y: 228 width: 277 height: 34
[2022-03-26 17:52:02.229]【远程设备】(js/main.js#74 )文字 : 丽剪映天空调色教程 x: 314 y: 228 width: 356 height: 34
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 抖音热榜抗疫榜直播榜音乐榜品牌榜 x: 27 y: 349 width: 711 height: 39
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 不 31省新增本土1280+4320 x: 55 y: 458 width: 454 height: 39
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 机上120人身份已确认 1126w x: 108 y: 546 width: 433 height: 39
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 今天上海人都在晒这样东西 1102w x: 111 y: 628 width: 503 height: 40
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 国防部回应美希望中国汲取... x: 111 y: 716 width: 471 height: 36
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 1058w x: 596 y: 719 width: 84 height: 23
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 4重庆报告1例本土无症状感染者1039w x: 56 y: 797 width: 620 height: 34
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 5坠机撞击点附近已无生命迹象 1015w x: 57 y: 881 width: 592 height: 33
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 暂停发售4月8日及以后车票1013w x: 108 y: 969 width: 518 height: 33
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 7救援人员用手一点点挖出遗物995.> x: 59 y: 1051 width: 603 height: 37
[2022-03-26 17:52:02.230]【远程设备】(js/main.js#74 )文字 : 见义勇为军人张昆获记一等功990.8w x: 111 y: 1137 width: 554 height: 33
[2022-03-26 17:52:02.231]【远程设备】(js/main.js#74 )文字 : 已搜寻到一部紧急定位发射仪92.0w x: 109 y: 1218 width: 554 height: 39
[2022-03-26 17:52:02.231]【远程设备】(js/main.js#74 )文字 : 10情但民家的直实写昭9557w x: 50 y: 1299 width: 472 height: 27
[2022-03-26 17:52:02.231]【远程设备】(js/main.js#80 )耗时: 1053 ms
[2022-03-26 17:52:03.235]【远程设备】(js/main.js#91 )脚本执行时长：8.33 秒
[2022-03-26 17:52:03.236]【远程设备】脚本结束执行

屏幕截图

代码优化

在前文中，我们详细讲解了ocr初始化参数的获取方法。
其中，参数baseDir是ocr库的目录路径。这里我们可以自动获取。
这样，在不同的系统环境中，就不需要判断和配置了。

来运行一段测试demo：

let baseDir = file.getSandBoxDir();   //获取当前设备沙盒的文件夹路径
    var arrurl = baseDir.split('/');
    if(arrurl[arrurl.length-2] == 'scriptbin') {
        baseDir = baseDir.replace(arrurl[arrurl.length-2] + "/" + arrurl[arrurl.length-1],"") + "OcrLiteNcnn"
        logd(baseDir);
    }

运行后，即可得到OcrLiteNcnn库文件的目录：
/Users/feiyun/Downloads/IOS/v2.1.0/center/ios-center-2.1.0/OcrLiteNcnn

温馨提示：此处内容需要登录后才能查看！

参数cpuType，可以根据baseDir的值，进行粗略判断。
具体的判断方法，大家可以自行思考并测试。由于我有点懒，
这里就不提供具体的代码了。

开发环境

电脑设备：MacBook Pro
系统版本：macOS Monterey 12.2.1
IDEA版本：IntelliJ IDEA Ultimate 2021.3
EC插件版本：2.1.0

交流群

IOS免越狱自动化测试交流群：691997586