《k230-AI
from libs.PipeLine import PipeLine, ScopedTiming
from libs.AIBase import AIBase
from libs.AI2D import Ai2d
import os
import ujson
from media.media import *
from time import *
import nncase_runtime as nn
import ulab.numpy as np
import time
import utime
import image
import random
import gc
import sys
import aidemo
from machine import PWM, FPIOA# 配置蜂鸣器IO口功能
beep_io = FPIOA()
beep_io.set_function(43, FPIOA.PWM1)# 初始化蜂鸣器PWM通道
beep_pwm = PWM(1, 4000, 50, enable=False) # 默认频率4kHz,占空比50%# 自定义YOLOv8检测类
class ObjectDetectionApp(AIBase):def __init__(self,kmodel_path,labels,model_input_size,max_boxes_num,confidence_threshold=0.5,nms_threshold=0.2,rgb888p_size=[224,224],display_size=[1920,1080],debug_mode=0):super().__init__(kmodel_path,model_input_size,rgb888p_size,debug_mode)self.kmodel_path=kmodel_pathself.labels=labels# 模型输入分辨率self.model_input_size=model_input_size# 阈值设置self.confidence_threshold=confidence_thresholdself.nms_threshold=nms_thresholdself.max_boxes_num=max_boxes_num# sensor给到AI的图像分辨率self.rgb888p_size=[ALIGN_UP(rgb888p_size[0],16),rgb888p_size[1]]# 显示分辨率self.display_size=[ALIGN_UP(display_size[0],16),display_size[1]]self.debug_mode=debug_mode# 检测框预置颜色值self.color_four=[(255, 220, 20, 60), (255, 119, 11, 32), (255, 0, 0, 142), (255, 0, 0, 230),(255, 106, 0, 228), (255, 0, 60, 100), (255, 0, 80, 100), (255, 0, 0, 70),(255, 0, 0, 192), (255, 250, 170, 30), (255, 100, 170, 30), (255, 220, 220, 0),(255, 175, 116, 175), (255, 250, 0, 30), (255, 165, 42, 42), (255, 255, 77, 255),(255, 0, 226, 252), (255, 182, 182, 255), (255, 0, 82, 0), (255, 120, 166, 157)]# 宽高缩放比例self.x_factor = float(self.rgb888p_size[0])/self.model_input_size[0]self.y_factor = float(self.rgb888p_size[1])/self.model_input_size[1]# Ai2d实例，用于实现模型预处理self.ai2d=Ai2d(debug_mode)# 设置Ai2d的输入输出格式和类型self.ai2d.set_ai2d_dtype(nn.ai2d_format.NCHW_FMT,nn.ai2d_format.NCHW_FMT,np.uint8, np.uint8)# 配置预处理操作，这里使用了resize，Ai2d支持crop/shift/pad/resize/affine，具体代码请打开/sdcard/libs/AI2D.py查看def config_preprocess(self,input_image_size=None):with ScopedTiming("set preprocess config",self.debug_mode > 0):# 初始化ai2d预处理配置，默认为sensor给到AI的尺寸，您可以通过设置input_image_size自行修改输入尺寸ai2d_input_size=input_image_size if input_image_size else self.rgb888p_sizeself.ai2d.resize(nn.interp_method.tf_bilinear, nn.interp_mode.half_pixel)self.ai2d.build([1,3,ai2d_input_size[1],ai2d_input_size[0]],[1,3,self.model_input_size[1],self.model_input_size[0]])# 自定义当前任务的后处理def postprocess(self,results):with ScopedTiming("postprocess",self.debug_mode > 0):result=results[0]result = result.reshape((result.shape[0] * result.shape[1], result.shape[2]))output_data = result.transpose()boxes_ori = output_data[:,0:4]scores_ori = output_data[:,4:]confs_ori = np.max(scores_ori,axis=-1)inds_ori = np.argmax(scores_ori,axis=-1)boxes,scores,inds = [],[],[]for i in range(len(boxes_ori)):if confs_ori[i] > confidence_threshold:scores.append(confs_ori[i])inds.append(inds_ori[i])x = boxes_ori[i,0]y = boxes_ori[i,1]w = boxes_ori[i,2]h = boxes_ori[i,3]left = int((x - 0.5 * w) * self.x_factor)top = int((y - 0.5 * h) * self.y_factor)right = int((x + 0.5 * w) * self.x_factor)bottom = int((y + 0.5 * h) * self.y_factor)boxes.append([left,top,right,bottom])if len(boxes)==0:return []boxes = np.array(boxes)scores = np.array(scores)inds = np.array(inds)# NMS过程keep = self.nms(boxes,scores,nms_threshold)dets = np.concatenate((boxes, scores.reshape((len(boxes),1)), inds.reshape((len(boxes),1))), axis=1)dets_out = []for keep_i in keep:dets_out.append(dets[keep_i])dets_out = np.array(dets_out)dets_out = dets_out[:self.max_boxes_num, :]return dets_out# 绘制结果def draw_result(self,pl,dets):with ScopedTiming("display_draw",self.debug_mode >0):if dets:pl.osd_img.clear()for det in dets:x1, y1, x2, y2 = map(lambda x: int(round(x, 0)), det[:4])x= x1*self.display_size[0] // self.rgb888p_size[0]y= y1*self.display_size[1] // self.rgb888p_size[1]w = (x2 - x1) * self.display_size[0] // self.rgb888p_size[0]h = (y2 - y1) * self.display_size[1] // self.rgb888p_size[1]pl.osd_img.draw_rectangle(x,y, w, h, color=self.get_color(int(det[5])),thickness=4)l_in=int(det[5])l_in=self.labels[l_in]pl.osd_img.draw_string_advanced( x , y-50,32," " + self.labels[int(det[5])] + " " + str(round(det[4],2)) , color=self.get_color(int(det[5])))if l_in == "person":# 使能PWM通道输出beep_pwm.enable(1)# 延时50mstime.sleep_ms(50)# 关闭PWM输出 防止蜂鸣器吵闹beep_pwm.enable(0)else:pl.osd_img.clear()# 多目标检测 非最大值抑制方法实现def nms(self,boxes,scores,thresh):"""Pure Python NMS baseline."""x1,y1,x2,y2 = boxes[:, 0],boxes[:, 1],boxes[:, 2],boxes[:, 3]areas = (x2 - x1 + 1) * (y2 - y1 + 1)order = np.argsort(scores,axis = 0)[::-1]keep = []while order.size > 0:i = order[0]keep.append(i)new_x1,new_y1,new_x2,new_y2,new_areas = [],[],[],[],[]for order_i in order:new_x1.append(x1[order_i])new_x2.append(x2[order_i])new_y1.append(y1[order_i])new_y2.append(y2[order_i])new_areas.append(areas[order_i])new_x1 = np.array(new_x1)new_x2 = np.array(new_x2)new_y1 = np.array(new_y1)new_y2 = np.array(new_y2)xx1 = np.maximum(x1[i], new_x1)yy1 = np.maximum(y1[i], new_y1)xx2 = np.minimum(x2[i], new_x2)yy2 = np.minimum(y2[i], new_y2)w = np.maximum(0.0, xx2 - xx1 + 1)h = np.maximum(0.0, yy2 - yy1 + 1)inter = w * hnew_areas = np.array(new_areas)ovr = inter / (areas[i] + new_areas - inter)new_order = []for ovr_i,ind in enumerate(ovr):if ind < thresh:new_order.append(order[ovr_i])order = np.array(new_order,dtype=np.uint8)return keep# 根据当前类别索引获取框的颜色def get_color(self, x):idx=x%len(self.color_four)return self.color_four[idx]if __name__=="__main__":# 显示模式，默认"hdmi",可以选择"hdmi"和"lcd"display_mode="lcd"if display_mode=="hdmi":display_size=[1920,1080]else:display_size=[800,480]# 模型路径kmodel_path="/sdcard/examples/kmodel/yolov8n_320.kmodel"labels = ["person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"]# 其它参数设置confidence_threshold = 0.2nms_threshold = 0.2max_boxes_num = 50rgb888p_size=[320,320]# 初始化PipeLinepl=PipeLine(rgb888p_size=rgb888p_size,display_size=display_size,display_mode=display_mode)pl.create()# 初始化自定义目标检测实例ob_det=ObjectDetectionApp(kmodel_path,labels=labels,model_input_size=[320,320],max_boxes_num=max_boxes_num,confidence_threshold=confidence_threshold,nms_threshold=nms_threshold,rgb888p_size=rgb888p_size,display_size=display_size,debug_mode=0)ob_det.config_preprocess()clock = time.clock()while True:clock.tick()img=pl.get_frame() # 获取当前帧数据res=ob_det.run(img) # 推理当前帧ob_det.draw_result(pl,res) # 绘制结果到PipeLine的osd图像pl.show_image() # 显示当前的绘制结果gc.collect()print(clock.fps()) #打印帧率
