离线推理精度问题分析

import numpy as np
import torch
from torchvision.io import read_image
from torchvision.models import resnet50, ResNet50_Weights

import onnxruntime as ort

from ais_bench.infer.interface import InferSession

# pytorch在线推理
class PytorchInferencer:
    def __init__(self):
        weights = ResNet50_Weights.DEFAULT
        self.model = resnet50(weights=weights)
        self.model.eval()
        self.transforms = weights.transforms()
        self.categories = weights.meta["categories"]

    def preprocess(self, image_path):
        """预处理"""
        # print(self.transforms)
        img = read_image(image_path)
        model_input = self.transforms(img).unsqueeze(0)
        return model_input

    def model_inference(self, model_input):
        """执行推理"""
        with torch.no_grad():
            model_output = self.model(model_input)
        return model_output

    def postprocess(self, model_output):
        """后处理"""
        model_output = model_output.squeeze(0).softmax(0)
        class_id = model_output.argmax().item()
        score = model_output[class_id].item()
        category_name = self.categories[class_id]
        return dict(category=category_name, class_id=class_id, score=score)

    def e2e_inference(self, image_path):
        """端到端推理"""
        model_input = self.preprocess(image_path)
        model_output = self.model_inference(model_input)
        prediction = self.postprocess(model_output)
        return prediction
    
    def export_to_onnx(self, image_path):
        torch.onnx.export(
            self.model,                # pytorch网络模型
            self.preprocess(image_path),          # 随机的模拟输入
            "resnet.onnx",        # 导出的onnx文件位置
            export_params=True,   # 导出训练好的模型参数
            verbose=True,         # verbose=True，支持打印onnx节点和对应的PyTorch代码行
            training=torch.onnx.TrainingMode.EVAL,  # 导出模型调整到推理状态，将dropout，BatchNorm等涉及的超参数固定
            input_names=["input_data"],    # 为静态网络图中的输入节点设置别名，在进行onnx推理时，将input_names字段与输入数据绑定
            output_names=["output_data"],  # 为输出节点设置别名
            # 如果不设置dynamic_axes，那么对于输入形状为[1, 3, 224, 224]，在以后使用onnx进行推理时也必须输入[1, 3, 224, 224]
            # 下面设置了输入的第0维是动态的，以后推理时batch_size的大小可以是其他动态值
            #dynamic_axes={
            #    "input_data": {0: "-1"},
            #    "output_data": {0: "-1"}
            #},
            keep_initializers_as_inputs=None,  #是否将模型参数作为输入数据的一部分进行导出
            opset_version=17                  # ONNX 运算符的版本号
        )
        print("export onnx model successfully!!")


# onnx在线推理
class OnnxInferencer(PytorchInferencer):
    def __init__(self, onnx_path):
        super(OnnxInferencer, self).__init__()
        # 可以构造预处理不一样，如修改归一化均值
        # self.transforms.__dict__["mean"] = [0.1, 0.1, 0.1]
        self.session = ort.InferenceSession(onnx_path)

    def model_inference(self, model_input):
        input_data = {"input_data": model_input.numpy()}
        model_output = self.session.run([], input_data)
        print(f"onnx output type: {type(model_output)}, onnx output[0] type: {type(model_output[0])}")
        model_output = torch.from_numpy(model_output[0])
        return model_output


# om离线推理
class OmInferencer(PytorchInferencer):
    def __init__(self, om_path, device_id=0):
        super(OmInferencer, self).__init__()
        self.session = InferSession(device_id=device_id, model_path=om_path)

    def model_inference(self, model_input):
        # 可以取值'static'(静态模型)、'dymbatch'(动态batch模型)、'dymhw'(动态分辨率模型)、'dymdims'(动态dims模型)、'dymshape'(动态shape模型)
        mode = "static" 
        # om模型推理输入输出都是numpy.array格式，而在线推理的模型输入输出是torch.Tensor，注意转换。
        model_output = torch.from_numpy(self.session.infer(feeds=[model_input.numpy()], mode=mode)[0])
        return model_output


inferencer = PytorchInferencer()
print(inferencer.e2e_inference("ILSVRC2012_val_00006083.jpeg"))
# {'category': 'Yorkshire terrier', 'score': 0.2925560474395752}

# export onnx model
inferencer.export_to_onnx("ILSVRC2012_val_00006083.jpeg")


# 在shell执行atc模型转换
"""
atc --framework=5 --model=resnet.onnx --output=resnet50_bs1 --input_format=NCHW --input_shape=input_data:1,3,224,224 --log=error --soc_version=Ascend910B4
"""

inferencer = OmInferencer('./resnet50_bs1.om')
print(inferencer.e2e_inference("./ILSVRC2012_val_00006083.jpeg"))
# {'category': 'Yorkshire terrier', 'score': 0.2925560474395752}


# 在线推理和离线推理结果对比
def precision_compare(pth_output, om_output):
    pth_output1 = pth_output.flatten().astype(np.float64)
    om_output1 = om_output.flatten().astype(np.float64)
    cosine_similarity = np.dot(pth_output1, om_output1) \
                        / (np.linalg.norm(om_output1) * np.linalg.norm(om_output1))
    absolute_errors = np.abs(om_output1 - pth_output1)
    relative_errors = absolute_errors / pth_output1 * 100
    print('余弦相似度：', cosine_similarity)
    print('最大绝对误差：', absolute_errors.max())
    print('最大相对误差：', relative_errors.max() )

def run_precision():
    pth_inferencer = PytorchInferencer()
    om_inferencer = OmInferencer('./resnet50_bs1.om')
    model_input = pth_inferencer.preprocess('./ILSVRC2012_val_00006083.jpeg')
    pth_output = pth_inferencer.model_inference(model_input).numpy()
    om_output = om_inferencer.model_inference(model_input).numpy()
    precision_compare(pth_output, om_output)

# 输入输出统一转成numpy.array格式，保存为.npy文件

# 1.pytorch在线推理，模型输入输出格式为torch.Tensor
import numpy as np
np.save("input_pth.npy", pth_input.numpy())
np.save("output_pth.npy", pth_output.numpy())

# 2.onnx在线推理，模型输入格式为np.array，输出为list,取索引0，格式为np.array
np.save("input_onnx.npy", onnx_input)
np.save("output_onnx.npy", onnx_output[0])

# 3. om离线推理，输入输出格式为np.array
np.save("input_om.npy", om_input)
np.save("output_om.npy", om_output)

# 也可以通过ais_bench工具dump 输出
'''
python -m ais_bench --model ./resnet50_bs1.om --input ./input_pth.npy --output ./ais_bench_out --outfmt NPY
'''

import numpy as np
from scipy import spatial


def compare(path1, path2):
    # 1.直接比较是否完全一致
    print("是否完全一致：", np.array_equal(path1, path2))

    # 2.浮点数的容差比较
    print("是否近似相同：", np.allclose(path1, path2))

    # 3.余弦相似度（适合向量/特征）,常用的判断标准为余弦相似度大于0.99
    cos_sim = 1 - spatial.distance.cosine(path1.reshape(-1), path2.reshape(-1))
    print("余弦相似度：", cos_sim)

def input_output_compare(ref1, ref2):
    print(f"{ref1} vs {ref2} input compare: ")
    input1 = np.load(f"input_{ref1}.npy")
    input2 = np.load(f"input_{ref2}.npy")
    compare(input1, input2)
    
    print("---" * 30)

    print(f"{ref1} vs {ref2} output compare: ")
    input1 = np.load(f"output_{ref1}.npy")
    input2 = np.load(f"output_{ref2}.npy")
    compare(input1, input2)

    print("---" * 30)

for values in [("pth", "onnx"), ("pth", "om"), ("onnx", "om")]:
    input_output_compare(*values)

msit debug compare -gm ./resnet.onnx -om ./resnet50_bs1.om -i ./input_onnx.npy -o ./msit_compare

msit debug compare -gm ./resnet.onnx -om ./resnet50_bs1.om -i ./input_onnx.bin -o ./msit_compare/advisor --advisor

msit debug surgeon extract --input resnet.onnx --output-file sub_ops.onnx --start-node-names "/layer2/layer2.3/conv1/Conv" --end-node-names "/layer2/layer2.3/conv1/Conv"

from auto_optimizer import OnnxGraph

# 创建单算子ONNX
op_model = OnnxGraph('conv.onnx')
# 读取原ONNX模型
model = OnnxGraph.parse('resnet50.onnx')
# 找到问题算子
node = model['Conv_1']

# 复制算子至单算子模型
op_model.add_node(node.name, node.op_type, inputs=node.inputs, outputs=node.outputs, attrs=node.attrs)
init = [node.name for node in model.initializers]
for inp in node.inputs:
    if inp in init:
        op_model.add_initializer(inp, model[inp].value)
    else:
        op_model.add_input(inp, dtype='float32', shape=[]) # shape若为空，则转OM时需指定input_shape
for out in node.outputs:
    op_model.add_output(out, dtype='float32', shape=[])

# 保存单算子模型
op_model.save('conv.onnx')