tensor - 张量1.板端C代码里评估内存//示例代码 rknn_context ctx 0; // Load RKNN Model int ret rknn_init(ctx, model_path, 0, NULL, NULL); if (ret 0) { printf(rknn_init fail! ret%d\n, ret); return -1; } // Get weight and internal mem size rknn_mem_size mem_size; ret rknn_query(ctx, RKNN_QUERY_MEM_SIZE, mem_size,sizeof(mem_size)); if (ret ! RKNN_SUCC) { printf(rknn_query fail! ret%d\n, ret); return -1; } printf(total weight size: %d, total internal size: %d\n,mem_size.total_weight_size, mem_size.total_internal_size);2.ubuntu 里python 连板推理方式内存评估使用方法# 在init_runtime时把eval_mem 设置为True rknn.init_runtime(targetargs.target, eval_memTrue) # 然后调用 rknn.eval_memory()完整例程rknn_model_zoo-2.0.0/examples/mobilenet/python/mobilenet.pyimport argparse import os import sys import urllib import urllib.request import time import traceback import numpy as np import cv2 from rknn.api import RKNN from scipy.special import softmax DATASET_PATH ../../../datasets/imagenet/ILSVRC2012_img_val_samples/dataset_20.txt MODEL_DIR ../model/ MODEL_PATH MODEL_DIR mobilenetv2-12.onnx OUT_RKNN_PATH MODEL_DIR mobilenet_v2.rknn CLASS_LABEL_PATH MODEL_DIR synset.txt RKNPU1_TARGET [rk1808, rv1109, rv1126] def readable_speed(speed): speed_bytes float(speed) speed_kbytes speed_bytes / 1024 if speed_kbytes 1024: speed_mbytes speed_kbytes / 1024 if speed_mbytes 1024: speed_gbytes speed_mbytes / 1024 return {:.2f} GB/s.format(speed_gbytes) else: return {:.2f} MB/s.format(speed_mbytes) else: return {:.2f} KB/s.format(speed_kbytes) def show_progress(blocknum, blocksize, totalsize): speed (blocknum * blocksize) / (time.time() - start_time) speed_str Speed: {}.format(readable_speed(speed)) recv_size blocknum * blocksize f sys.stdout progress (recv_size / totalsize) progress_str {:.2f}%.format(progress * 100) n round(progress * 50) s (# * n).ljust(50, -) f.write(progress_str.ljust(8, ) [ s ] speed_str) f.flush() f.write(\r\n) def check_and_download_origin_model(): global start_time if not os.path.exists(MODEL_PATH): print(-- Download {}.format(MODEL_PATH)) url https://ftrg.zbox.filez.com/v2/delivery/data/95f00b0fc900458ba134f8b180b3f7a1/examples/MobileNet/mobilenetv2-12.onnx download_file MODEL_PATH try: start_time time.time() urllib.request.urlretrieve(url, download_file, show_progress) except: print(Download {} failed..format(download_file)) print(traceback.format_exc()) exit(-1) print(done) if __name__ __main__: parser argparse.ArgumentParser( descriptionMobileNet Python Demo, add_helpTrue) parser.add_argument(--target, typestr, defaultrk3566, helpRKNPU target platform) parser.add_argument(--npu_device_test, actionstore_true, defaultFalse, helpConnected npu device run) parser.add_argument(--accuracy_analysis, actionstore_true, defaultFalse, helpAccuracy analysis) parser.add_argument(--eval_perf, actionstore_true, defaultFalse, helpTime consuming evaluation) parser.add_argument(--eval_memory, actionstore_true, defaultFalse, helpMemory evaluation) parser.add_argument(--model, typestr, defaultMODEL_PATH, helponnx model path) parser.add_argument(--output_path, typestr, defaultOUT_RKNN_PATH, helpoutput rknn model path) parser.add_argument(--dtype, typestr, defaulti8, helpdtype of model, i8/fp32 for RKNPU2, u8/fp32 for RKNPU1) args parser.parse_args() # Download model if not exist (from https://ftrg.zbox.filez.com/v2/delivery/data/95f00b0fc900458ba134f8b180b3f7a1/examples/MobileNet/mobilenetv2-12.onnx) check_and_download_origin_model() # Create RKNN object rknn RKNN(verboseFalse) # Pre-process config print(-- Config model) rknn.config(mean_values[[255*0.485, 255*0.456, 255*0.406]], std_values[[ 255*0.229, 255*0.224, 255*0.225]], target_platformargs.target) print(done) # Load model print(-- Loading model) if args.target in RKNPU1_TARGET: ret rknn.load_onnx(modelargs.model, inputs[input], input_size_list[[3,224,224]]) else: ret rknn.load_onnx(modelargs.model, inputs[input], input_size_list[[1,3,224,224]]) if ret ! 0: print(Load model failed!) exit(ret) print(done) # Build model print(-- Building model) do_quant True if (args.dtype i8 or args.dtype u8) else False ret rknn.build(do_quantizationdo_quant, datasetDATASET_PATH) if ret ! 0: print(Build model failed!) exit(ret) print(done) # Export rknn model print(-- Export rknn model) ret rknn.export_rknn(args.output_path) if ret ! 0: print(Export rknn model failed!) exit(ret) print(done) # Set inputs img cv2.imread(../model/bell.jpg) img cv2.resize(img, (224, 224)) img np.expand_dims(img, 0) # Init runtime environment print(-- Init runtime environment) if args.npu_device_test or args.target in RKNPU1_TARGET: # For RKNPU1, the simulator has beed disabled since version 1.7.5 print(-------------------- init_runtime here ---------------------) ret rknn.init_runtime(targetargs.target, perf_debugTrue, eval_memTrue) elif args.eval_perf or args.eval_memory: ret rknn.init_runtime( targetargs.target, perf_debugTrue, eval_memTrue) else: if args.target in RKNPU1_TARGET: print(The target {} does not support simulator..format(args.target)) print(Please set --npu_device_test to init runtime with real target.) exit(-1) ret rknn.init_runtime() if ret ! 0: print(Init runtime environment failed!) exit(ret) print(done) # Eval Perf if args.eval_perf: print(-- Eval Perf) rknn.eval_perf() print(done) # Eval Memory if args.eval_memory: print(-- Eval Memory) rknn.eval_memory() print(done) # Inference print(-- Running model) outputs rknn.inference(inputs[img]) # Post Process print(-- PostProcess) with open(CLASS_LABEL_PATH, r) as f: labels [l.rstrip() for l in f] scores softmax(outputs[0]) # print the top-5 inferences class scores np.squeeze(scores) a np.argsort(scores)[::-1] print(-----TOP 5-----) for i in a[0:5]: print([%d] score%.2f class%s % (i, scores[i], labels[i])) print(done) # Accuracy analysis if args.accuracy_analysis: print(-- Accuracy analysis) if args.npu_device_test: ret rknn.accuracy_analysis( inputs[../model/bell.jpg], targetargs.target) else: ret rknn.accuracy_analysis(inputs[../model/bell.jpg]) if ret ! 0: print(Accuracy analysis failed!) exit(ret) print(done) # Release rknn.release()测试python mobilenet.py --target rk3588 --eval_memory --npu_device_test输出3.内存优化方法3.1 零拷贝在推理 RKNN 模型时原始数据要经过输入处理、NPU推理、输出处理三大流程。根据不同模型输入格式和量化方式接口内部会存在通用 API 和零拷贝 API 两种处理流程两组 API 的主要区别在于通用接口每次更新帧数据需要将外部模块分配的数据拷贝到 NPU 的输入内存而零拷贝流程的接口会直接使用预先分配的内存包括 NPU 运行时创建的或外部其他框架创建的比如 DRM 框架减少了内存拷贝的花销性能更优带宽更少。RK3588 是UMA 统一内存架构CPU/NPU 共用一片 LPDDR 物理内存零拷贝Zero-Copy依靠 Linux dma-buf 共享缓冲区框架硬件之间只传递内存 fd文件描述符不搬运图像 / 张量原始数据彻底消除 CPU 内存拷贝降低带宽占用、减少延迟、节省 DDR 内存。3.2 内存复用Internal 内存复用RKNN API 提 供 了 外 部 管 理 NPU 内 存 的 机 制 通过RKNN_FLAG_MEM_ALLOC_OUTSIDE 参数用户可以指定模型中间的 feature 内存由外部分配。该功能的典型应用场景如下部署时所有 NPU 内存均是用户自行分配便于对整个系统内存进行统筹安排。用于多个模型串行运行场景中间 feature 内存在不同上下文复用特别是针对RV1103/RV1106 这种内存极为紧张的情况。例如下图中有两个模型模型 1 的 Internal Tensor 占用大于模型 2如果模型 1 和模型 2 顺序地运行可以只开辟 0x00000000~0x000c4000 地址的一块内存给模型 1 和 2 共用模型 1 推理结束后这块内存可以被模型 2 用来读写 Internal Tensor 数据从而节省内存。图 9-1 两个模型 Internal Tensor 共享同一块内存地址空间的示例图内存复用 同一块物理内存缓冲区在不同阶段重复利用用完立刻释放 / 覆盖不重复新开内存