前端智能化：实现自己的控件识别模型

$ npm install -g @pipcook/pipcook-cli

$ mkdir pipcook-example && cd pipcook-example
$ pipcook init
$ pipcook board

> @pipcook/pipcook-board-server@1.0.0 dev /Users/zhenyankun.zyk/work/node/pipcook/example/.server
> egg-bin dev
[egg-ts-helper] create typings/app/controller/index.d.ts (2ms)
[egg-ts-helper] create typings/config/index.d.ts (9ms)
[egg-ts-helper] create typings/config/plugin.d.ts (2ms)
[egg-ts-helper] create typings/app/service/index.d.ts (1ms)
[egg-ts-helper] create typings/app/index.d.ts (1ms)
2020-04-16 11:52:22,053 INFO 26016 [master] node version v12.16.2
2020-04-16 11:52:22,054 INFO 26016 [master] egg version 2.26.0
2020-04-16 11:52:22,839 INFO 26016 [master] agent_worker#1:26018 started (782ms)
2020-04-16 11:52:24,262 INFO 26016 [master] egg started on http://127.0.0.1:7001 (2208ms)

{
  "plugins": {
    "dataCollect": {
      "package": "@pipcook/plugins-mnist-data-collect",
      "params": {
        "trainCount": 8000,
        "testCount": 2000
      }
    },

const mnist = require('mnist');

"dependencies": {
    "@pipcook/pipcook-core": "^0.5.9",
    "@tensorflow/tfjs-node-gpu": "1.7.0",
    "@types/cli-progress": "^3.4.2",
    "cli-progress": "^3.6.0",
    "jimp": "^0.10.0",
    "mnist": "^1.1.0"
  },

train-images-idx3-ubyte.gz:  training set images (9912422 bytes) 
train-labels-idx1-ubyte.gz:  training set labels (28881 bytes)

git clone https://github.com/ApelSYN/mnist_dl.git
正克隆到 'mnist_dl'...
remote: Enumerating objects: 36, done.
remote: Total 36 (delta 0), reused 0 (delta 0), pack-reused 36
展开对象中: 100% (36/36), 完成.

# 数据源目录，用来下载 LeCun 大神的数据
$ mkdir data
# 数据集目录，用来存放 mnist_dl 处理后的 Json 数据
$ mkdir digits

node mnist_dl.js --count 10000
DB digits Version: 2051
Total digits: 60000
x x y: 28 x 28
60000
47040000
Pass 0 items...
Pass 1000 items...
Pass 2000 items...
Pass 3000 items...
Pass 4000 items...
Pass 5000 items...
Pass 6000 items...
Pass 7000 items...
Pass 8000 items...
Pass 9000 items...
Finish processing 10000 items...
Start make "0.json with 1001 images"
Start make "1.json with 1127 images"
Start make "2.json with 991 images"
Start make "3.json with 1032 images"
Start make "4.json with 980 images"
Start make "5.json with 863 images"
Start make "6.json with 1014 images"
Start make "7.json with 1070 images"
Start make "8.json with 944 images"
Start make "9.json with 978 images"

$ git clone https://github.com/cazala/mnist.git
正克隆到 'mnist'...
remote: Enumerating objects: 143, done.
remote: Total 143 (delta 0), reused 0 (delta 0), pack-reused 143
接收对象中: 100% (143/143), 18.71 MiB | 902.00 KiB/s, 完成.
处理 delta 中: 100% (73/73), 完成.
$ npm install
$ cd src
$ cd digits
$ ls
0.json 1.json 2.json 3.json 4.json 5.json 6.json 7.json 8.json 9.json

# 进入工作目录
$ cd src
# 先备份一下
$ mv digits digits-bk
# 再拷贝之前处理的json数据
$ cp -R ../mnist_dl/digits ./
$ ls
digits    digits-bk mnist.js

stream.on('readable', function () {
        let buf = stream.read();
        if (buf) {
            if (ver != 2051) {
                ver = buf.readInt32BE(0);
                console.log(`DB digits Version: ${ver}`);
                digitCount = buf.readInt32BE(4);
                console.log(`Total digits: ${digitCount}`);
                x = buf.readInt32BE(8);
                y = buf.readInt32BE(12);
                console.log(`x x y: ${x} x ${y}`);
                start = 16;
            }
            for (let i = start; i< buf.length; i++) {
                digits.push(buf.readUInt8(i));
            }
            start = 0;
        }
    });

<link rel="stylesheet" href="t1.min.css">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
<div align="middle">
<p>
    <button class="btn btn-primary">Primary</button>
</p>
<p>
    <button class="btn btn-info">Info</button>
</p>
<p>
    <button class="btn btn-success">Success</button>
</p>
<p>
    <button class="btn btn-warning">Warning</button>
</p>
<p>
    <button class="btn btn-danger">Danger</button>
</p>
<p>
    <button class="btn btn-lg btn-primary" type="button">Large button</button>
</p>
<p>
    <button class="btn btn-primary" type="button">Default button</button>
</p>
<p>
    <button class="btn btn-sm btn-primary" type="button">Mini button</button>
</p>
<p>
    <a href="#" class="btn btn-xs btn-primary disabled">Primary link disabled state</a>
</p>
<p>
    <button class="btn btn-lg btn-block btn-primary" type="button">Block level button</button>
</p>
<p>
    <button type="button" class="btn btn-primary">Primary</button>
</p>
<p>
    <button type="button" class="btn btn-secondary">Secondary</button>
</p>
<p>
    <button type="button" class="btn btn-success">Success</button>
</p>
<p>
    <button type="button" class="btn btn-danger">Danger</button>
</p>
<p>
    <button type="button" class="btn btn-warning">Warning</button>
</p>
<p>
    <button type="button" class="btn btn-info">Info</button>
</p>
<p>
    <button type="button" class="btn btn-light">Light</button>
</p>
<p>
    <button type="button" class="btn btn-dark">Dark</button>
</p>
</div>

$ mkdir pupp && cd pupp
$ npm init --yes
$ npm i puppeteer --save
# or "yarn add puppeteer"

$ brew install graphicsmagick
$ npm i gm --save

const puppeteer = require("puppeteer");
const fs = require("fs");
const Q = require("Q");
function delay(ms) {
  var deferred = Q.defer();
  setTimeout(deferred.resolve, ms);
  return deferred.promise;
}
const urls = [
  "file:///Users/zhenyankun.zyk/work/node/pipcook/pupp/htmlData/page1.html",
  "file:///Users/zhenyankun.zyk/work/node/pipcook/pupp/htmlData/page2.html",
  "file:///Users/zhenyankun.zyk/work/node/pipcook/pupp/htmlData/page3.html",
  "file:///Users/zhenyankun.zyk/work/node/pipcook/pupp/htmlData/page4.html",
  "file:///Users/zhenyankun.zyk/work/node/pipcook/pupp/htmlData/page5.html",
];
(async () => {
  // Launch a headful browser so that we can see the page navigating.
  const browser = await puppeteer.launch({
    headless: true,
    args: ["--no-sandbox", "--disable-gpu"],
  });
  const page = await browser.newPage();
  await page.setViewport({
    width: 375,
    height: 812,
    isMobile: true,
  }); //Custom Width
  //start shortcut every page
  let counter = 0;
  for (url of urls) {
    await page.goto(url, {
      timeout: 0,
      waitUntil: "networkidle0",
    });
    await delay(100);
    let btnElements = await page.$$("button");
    for (btn of btnElements) {
      const btnData = await btn.screenshot({
        encoding: "binary",
        type: "jpeg",
        quality: 90,
      });
      let fn = "data/btn" + counter + ".jpg";
      Q.nfcall(fs.writeFileSync, fn, btnData);
      counter++;
    }
  }
  await page.close();
  await browser.close();
})();

const gm = require("gm");
const fs = require("fs");
const path = require("path");
const basePath = "./data/";
const chars = [
  "0",
  "1",
  "2",
  "3",
  "4",
  "5",
  "6",
  "7",
  "8",
  "9",
  "A",
  "B",
  "C",
  "D",
  "E",
  "F",
  "G",
  "H",
  "I",
  "J",
  "K",
  "L",
  "M",
  "N",
  "O",
  "P",
  "Q",
  "R",
  "S",
  "T",
  "U",
  "V",
  "W",
  "X",
  "Y",
  "Z",
];
let randomRange = (min, max) => {
  return Math.random() * (max - min) + min;
};
let randomChars = (rangeNum) => {
  let tmpChars = "";
  for (let i = 0; i < rangeNum; i++) {
    tmpChars += chars[Math.ceil(Math.random() * 35)];
  }
  return tmpChars;
};
//获取此文件夹下所有的文件(数组)
const files = fs.readdirSync(basePath);
for (let file of files) {
  let filePath = path.join(basePath, file);
  gm(filePath)
    .quality(100)
    .gravity("Center")
    .drawText(randomRange(-5, 5), 0, randomChars(5))
    .channel("Gray")
    // .monochrome()
    .resize(28)
    .extent(28, 28)
    .write(filePath, function (err) {
      if (!err) console.log("At " + filePath + " done! ");
      else console.log(err);
    });
}

for (let file of files) {
  for (let i = 0; i < 3; i++) {
    let rawfilePath = path.join(basePath, file);
    let newfilePath = path.join(basePath, i + file);
    gm(rawfilePath)
      .quality(100)
      .gravity("Center")
      .drawText(randomRange(-5, 5), 0, randomChars(5))
      .channel("Gray")
      // .monochrome()
      .resize(28)
      .extent(28, 28)
      .write(newfilePath, function (err) {
        if (!err) console.log("At " + newfilePath + " done! ");
        else console.log(err);
      });
  }
}

$ npm install @pipcook/boa --save

 $ ./node_modules/@pipcook/boa/.miniconda/bin/pip install opencv-python pillow

const boa = require("@pipcook/boa");
// 引入一些 python 语言内置的数据结构
const { int, tuple, list } = boa.builtins();
// 引入 OpenCV
const cv2 = boa.import("cv2");
const np = boa.import("numpy");
const Image = boa.import("PIL.Image");
const ImageFont = boa.import("PIL.ImageFont");
const ImageDraw = boa.import("PIL.ImageDraw");
let img = np.zeros(tuple([28, 28, 3]), np.uint8);
img = Image.fromarray(img);
let draw = ImageDraw.Draw(img);
draw.text(list([0, 0]), "Shadow");
img.save("./test.tiff");

import numpy as np
import cv2
from PIL import ImageFont, ImageDraw, Image
img = np.zeros((150,150,3),np.uint8)
img = Image.fromarray(img)
draw = ImageDraw.Draw(img)
draw.text((0,0),"Shadow")
img.save()

const Image = boa.import("PIL.Image");
const ImageFont = boa.import("PIL.ImageFont");
const ImageDraw = boa.import("PIL.ImageDraw");

#/usr/bin/env python2.7
#coding:utf-8
import os
import cv2
import numpy
import sys
import struct
DEFAULT_WIDTH = 28
DEFAULT_HEIGHT = 28
DEFAULT_IMAGE_MAGIC = 2051
DEFAULT_LBAEL_MAGIC = 2049
IMAGE_BASE_OFFSET = 16
LABEL_BASE_OFFSET = 8
def usage_generate():
    print "python mnist_helper generate path_to_image_dir"
    print "\t path_to_image_dir/subdir, subdir is the label"
    print ""
    pass
def create_image_file(image_file):
    fd = open(image_file, 'w+b')
    buf = struct.pack(">IIII", DEFAULT_IMAGE_MAGIC, 0, DEFAULT_WIDTH, DEFAULT_HEIGHT)
    fd.write(buf)
    fd.close()
    pass
def create_label_file(label_file):
    fd = open(label_file, 'w+b')
    buf = struct.pack(">II", DEFAULT_LBAEL_MAGIC, 0)
    fd.write(buf)
    fd.close()
    pass
def update_file(image_file, label_file, image_list, label_list):
    ifd = open(image_file, 'r+')
    ifd.seek(0)
    image_magic, image_count, rows, cols = struct.unpack(">IIII", ifd.read(IMAGE_BASE_OFFSET))
    image_len = rows * cols
    image_offset = image_count * rows * cols + IMAGE_BASE_OFFSET
    ifd.seek(image_offset)
    for image in image_list:
        ifd.write(image.astype('uint8').reshape(image_len).tostring())
    image_count += len(image_list)
    ifd.seek(0, 0)
    buf = struct.pack(">II", image_magic, image_count)
    ifd.write(buf)
    ifd.close()
    lfd = open(label_file, 'r+')
    lfd.seek(0)
    label_magic, label_count = struct.unpack(">II", lfd.read(LABEL_BASE_OFFSET))
    buf = ''.join(label_list)
    label_offset = label_count + LABEL_BASE_OFFSET
    lfd.seek(label_offset)
    lfd.write(buf)
    lfd.seek(0)
    label_count += len(label_list)
    buf = struct.pack(">II", label_magic, label_count)
    lfd.write(buf)
    lfd.close()
def mnist_generate(image_dir):
    if not os.path.isdir(image_dir):
        raise Exception("{0} is not exists!".format(image_dir))
    image_file = os.path.join(image_dir, "user-images-ubyte")
    label_file = os.path.join(image_dir, "user-labels-ubyte")
    create_image_file(image_file)
    create_label_file(label_file)
    for i in range(10):
        path = os.path.join(image_dir, "{0}".format(i))
        if not os.path.isdir(path):
            continue
        image_list = []
        label_list = []
        for f in os.listdir(path):
            fn = os.path.join(path, f)
            image = cv2.imread(fn, 0)
            w, h = image.shape
            if w and h and (w <> 28) or (h <> 28):
                simg = cv2.resize(image, (28, 28))
                image_list.append(simg)
                label_list.append(chr(i))
        update_file(image_file, label_file, image_list, label_list)
    print "user data generate successfully"
    print "output files: \n\t {0}\n\t {1}".format(image_file, label_file)
    pass

{
  "plugins": {
    "dataCollect": {
      "package": "@pipcook/plugins-mnist-data-collect",
      "params": {
        "trainCount": 8000,
        "testCount": 2000
      }
    },
    "dataAccess": {
      "package": "@pipcook/plugins-pascalvoc-data-access"
    },
    "dataProcess": {
      "package": "@pipcook/plugins-image-data-process",
      "params": {
        "resize": [28,28]
      }
    },
    "modelDefine": {
      "package": "@pipcook/plugins-tfjs-simplecnn-model-define"
    },
    "modelTrain": {
      "package": "@pipcook/plugins-image-classification-tfjs-model-train",
      "params": {
        "epochs": 15
      }
    },
    "modelEvaluate": {
      "package": "@pipcook/plugins-image-classification-tfjs-model-evaluate"
    }
  }
}

$ pipcook run examples/pipelines/mnist-image-classification.json

$ pipcook board