From 934855af80706d659baff0b47d3de4f8f559e5ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=86=8A=E7=8E=AE?= Date: Fri, 22 Nov 2024 14:59:00 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81image=20framework?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/detect-gui.iml | 2 +- .idea/misc.xml | 2 +- components/image_framework.py | 254 +++++++++++++++++++++++++++++++--- config.ini | 3 + req.txt | 3 +- widget/task_run.py | 58 +++++--- 6 files changed, 283 insertions(+), 39 deletions(-) diff --git a/.idea/detect-gui.iml b/.idea/detect-gui.iml index a1b377a..80f67ae 100644 --- a/.idea/detect-gui.iml +++ b/.idea/detect-gui.iml @@ -4,7 +4,7 @@ - + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index 9f2e3a2..66b3d02 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,5 +3,5 @@ - + \ No newline at end of file diff --git a/components/image_framework.py b/components/image_framework.py index 7f03d79..edeb3bb 100644 --- a/components/image_framework.py +++ b/components/image_framework.py @@ -1,4 +1,5 @@ import os +import threading from ctypes import * import numpy as np from PyQt5.QtCore import QObject, pyqtSignal @@ -6,6 +7,7 @@ from dynaconf.base import Settings from numpy.ctypeslib import as_array import platform import time +import cv2 from core.edge_component import action, EdgeComponent, service from util import get_system_and_library_suffix @@ -49,10 +51,139 @@ CallbackInfo = _CallbackInfo_ # 回调函数类型定义 CallbackType = CFUNCTYPE(None, POINTER(CallbackInfo)) +# emit的结构体 +MSG_ONLY_RECORD = -1 +MSG_LOCATION_RECORD = 0 +MSG_DETECTION_RECORD = 1 + +class Msg: + def __init__(self, msg_type, record, im2D): + self.msg_type = msg_type # 0：刷新广角图片， 1：刷新高清， -1：不刷新图片，纯消息 + self.record = record + self.im2D = im2D + + +TO_CLIENT_NOTIFY_BASE = 1000 +SYS_BASE = 1000 +CAL_2D3D_BASE = 2000 +CAMERA_BASE = 4000 +LIDAR_BASE = 5000 +SYS_OK = 0 +CAL_OK = 0 +LIDAR_OK = 0 +MSG_LIST = [ + { 1 + TO_CLIENT_NOTIFY_BASE : "开始相机拍摄" }, + { 2 + TO_CLIENT_NOTIFY_BASE : "相机拍摄已完成" }, + { 3 + TO_CLIENT_NOTIFY_BASE : "开始雷达采集数据" }, + { 4 + TO_CLIENT_NOTIFY_BASE : "雷达采集数据已完成" }, + { 5 + TO_CLIENT_NOTIFY_BASE : "开始定位检测" }, # 这个消息是相机实时定位，不用打印 + { 6 + TO_CLIENT_NOTIFY_BASE : "定位检测已完成" }, + { 7 + TO_CLIENT_NOTIFY_BASE : "开始角点2D检测" }, + { 8 + TO_CLIENT_NOTIFY_BASE : "角点2D检测已完成" }, + { 9 + TO_CLIENT_NOTIFY_BASE : "开始角点3D检测" }, + { 10 + TO_CLIENT_NOTIFY_BASE : "角点3D检测已完成" }, + { 11 + TO_CLIENT_NOTIFY_BASE : "开始计算预埋件尺寸" }, + { 12 + TO_CLIENT_NOTIFY_BASE : "预埋件尺寸计算已完成" }, + + { 1 - SYS_BASE : "驱动初始化错误" }, + { 2 - SYS_BASE : "NPU计算单元初始化错误" }, + { 3 - SYS_BASE : "系统错误" }, + { 4 - SYS_BASE : "相机初始化错误" }, + { 5 - SYS_BASE : "当前没有可以读取的高清相片"}, + + { 1 - CAL_2D3D_BASE : "读取图片文件错误" }, + { 2 - CAL_2D3D_BASE : "写图片文件错误" }, + { 3 - CAL_2D3D_BASE : "空指针" }, + { 4 - CAL_2D3D_BASE : "空参数" }, + { 5 - CAL_2D3D_BASE : "读取图像错误" }, + { 6 - CAL_2D3D_BASE : "NPU加载数据错误" }, + { 7 - CAL_2D3D_BASE : "NPU模型执行错误" }, + { 8 - CAL_2D3D_BASE : "NPU获取结果错误" }, + { 9 - CAL_2D3D_BASE : "NPU卸载数据错误" }, + { 10 - CAL_2D3D_BASE : "YOLO推理无可信ROI区域" }, + { 11 - CAL_2D3D_BASE : "ROI处理线程错误" }, + { 12 - CAL_2D3D_BASE : "H计算错误" }, + { 13 - CAL_2D3D_BASE : "3D角点检测错误" }, + { 14 - CAL_2D3D_BASE : "2D角点定位错误" }, + { 15 - CAL_2D3D_BASE : "3D投影计算错误" }, + { 16 - CAL_2D3D_BASE : "3D点云为空" }, + { 17 - CAL_2D3D_BASE : "未检测到2D角点" }, + { 18 - CAL_2D3D_BASE : "未检测到预埋件" }, + { 19 - CAL_2D3D_BASE : "相机捕获图像失败" }, + + { 1 - CAMERA_BASE : "相机DLL初始化失败" }, + + { 1 - LIDAR_BASE : "雷达初始化SDK失败" }, + { 2 - LIDAR_BASE : "雷达SO初始化失败" }, + { 3 - LIDAR_BASE : "雷达广播失败" }, + { 5 - LIDAR_BASE : "雷达开始采样失败" }, + { 6 - LIDAR_BASE : "雷达结束采样失败" }, + + { -10001 : "已经在检测模式中"}, + { -10002 : "定位模式不支持执行该操作"} +] + +#################################################################################### +# 全局变量定义 +#################################################################################### + +# 全局 msg 缓存 +g_msg_cache = [] + +# 全局 frame 缓存 +g_frame_cache = [] + +# 运行模式 +RUNNING_MODE_NONE = -1 # 停止状态 +RUNNING_MODE_LOCATION = 0 # 广角相机运行模式 +RUNNING_MODE_DETECTION = 1 # 高清检测运行模式 +g_running_mode = RUNNING_MODE_LOCATION # 默认是广角相机运行模式 + +#################################################################################### +# 内部接口定义 +#################################################################################### +# 获取消息 +def _get_msg_info(code) -> str: + global MSG_LIST + for msg in MSG_LIST: + if code == msg.keys()[0]: + return msg.values + return "未知错误" + +# 加工检测图像 +def _update_location_image(frame): + global needHandle, recordCache, ckinfoCache + def switch(choice): + if ckinfoCache.code == 1006: #TO_CLIENT_NOTIFY_DETECTION_END: 1006 + for i in range(recordCache.roi_size): + subRois = recordCache.subRois[i] + s = "" + for j in range(8): + s += str(subRois.mRoiPoints[j])+ " " + else: + print("default Case") + +# 在回调函数中将 record 数据拷贝出来 +def _copy_record(record): + new_record = CallbackInfo() + new_record.code = record.code + new_record.errorInfo = record.errorInfo + new_record.bGetData = record.bGetData + for i in range(record.roi_size): + subRois = record.subRois[i] + newSubRois = new_record.subRois[i] + for j in range(8): + newSubRois.mRoiPoints[j] = subRois.mRoiPoints[j] + newSubRois.mInnerConners[j] = subRois.mInnerConners[j] + for j in range(12): + newSubRois.mInnerConners3D[j] = subRois.mInnerConners3D[j] + + return new_record + class ImageFrameworkSignals(QObject): # QT 信号 - on_image_result = pyqtSignal(object) + on_image_detect_result = pyqtSignal(Msg) @action("image-framework", auto_start=True) @@ -63,9 +194,16 @@ class ImageFramework(EdgeComponent): super().__init__(context) self.image_framework_sdk = None self.data_callback = None + self.location_thread = None + self.process_thread = None def configure(self, setting: Settings) -> None: self.init_image_framework_sdk() + self.location_thread = threading.Thread(target=self._location_processing_thread) + self.location_thread.daemon = True + self.process_thread = threading.Thread(target=self._main_processing_thread) + self.process_thread.daemon = True + self.logger.info(f"Image framework configure done.") def init_image_framework_sdk(self): @@ -81,23 +219,42 @@ class ImageFramework(EdgeComponent): def init_callback(self): def _callback(data): - contents = data.contents - self.logger.info(f"image framework callback data, bGetData:{contents.bGetData}") - if contents.bGetData: - record = CallbackInfo() - self.image_framework_sdk.LibapGetRecord(byref(record)) - self.logger.info(f"image framework callback data, roi_size:{record.roi_size}") - self.logger.info(f"image framework callback data, code:{record.code}") - self.logger.info(f"image framework callback data, errorInfo:{record.errorInfo}") - for i in range(record.roi_size): - roi_data = record.subRois[i] - self.logger.info(f"roi data, roi_data[{i}].mId:{roi_data.mId}") - self.logger.info(f"roi data, roi_data[{i}].isGood:{roi_data.isGood}") - self.logger.info(f"roi data, roi_data[{i}].mRoiPoints:{list(roi_data.mRoiPoints)}") + global g_msg_cache, g_frame_cache, g_running_mode - self.signals.on_image_result.emit(record.roi_size) + record = CallbackInfo(code=SYS_OK, errorInfo=b"", bGetData=False) + frame = None + ret = SYS_OK + record.code = data.contents.code + record.errorInfo = data.contents.errorInfo + # 当定位模式下，接收到【定位完成】消息 + if g_running_mode == RUNNING_MODE_LOCATION and record.code == (6 + TO_CLIENT_NOTIFY_BASE): + if len(g_frame_cache) == 0: + self.logger.error("当前没有广角数据") + return - self.logger.info("image framework callback done.") + frame = g_frame_cache.pop() # 从图像缓存中去除图像 + g_frame_cache = [] # 清空缓存，避免无限增长 + msg = Msg(MSG_LOCATION_RECORD, _copy_record(record), frame) + # 当检测模式下，接收到【拍照完成】消息 + elif g_running_mode == RUNNING_MODE_DETECTION and record.code == (2 + TO_CLIENT_NOTIFY_BASE): + frame.zeros(9344, 7000, cv2.CV_8UC3) + src_frame_ptr = c_char_p(frame.data.tobytes()) + ret = self.image_framework_sdk.LibapiGetHQImage( + src_frame_ptr, frame.shape[1], frame.shape[0], 3, c_char_p(b"PythonProcessing")) + if ret != SYS_OK: + record = CallbackInfo(code=ret, errorInfo=_get_msg_info(ret), bGetData=False) + msg = Msg(MSG_ONLY_RECORD, record, None) + # emit msg + return + msg = Msg(MSG_DETECTION_RECORD, _copy_record(record), frame) + elif g_running_mode == RUNNING_MODE_LOCATION or g_running_mode == RUNNING_MODE_DETECTION: + msg = Msg(MSG_ONLY_RECORD, _copy_record(record), None) + else: + print("未知回调") + return + + g_msg_cache.append(msg) + # self.logger.info("image framework callback done.") self.data_callback = CallbackType(_callback) @@ -115,11 +272,25 @@ class ImageFramework(EdgeComponent): @service() def stop(self): + if self.location_thread is not None: + self.location_thread.join() + if self.process_thread is not None: + self.process_thread.join() + self.image_framework_sdk.stop_sdk() self.logger.info("Image framework stopped.") + @service() + def start_location(self): + self.location_thread.start() + self.process_thread.start() + @service() def start_detect(self): + global g_running_mode + if g_running_mode != RUNNING_MODE_DETECTION: + g_running_mode = RUNNING_MODE_DETECTION # 切换为检测模式 + ret = self.image_framework_sdk.LibapiStartDetection() if ret != 0: raise RuntimeError("启动检测失败!") @@ -132,6 +303,55 @@ class ImageFramework(EdgeComponent): @service() def stop_detect(self): + global g_running_mode + if g_running_mode == RUNNING_MODE_LOCATION: # 如果在定位任务模式下，该接口不执行 + raise RuntimeError("正在检测，请稍后停止!") + + g_running_mode = RUNNING_MODE_NONE # 切换为无模式 ret = self.image_framework_sdk.LibapStopDetection() if ret != 0: - raise RuntimeError("停止检测失败!") \ No newline at end of file + raise RuntimeError("停止检测失败!") + + # 广角相机运行定位处理线程 + def _location_processing_thread(self): + global g_msg_cache, g_frame_cache, g_running_mode + # 打开广角摄像头 + capture = cv2.VideoCapture(0) + while True: + # 如果当前不是定位模式，不处理任何事务 + if g_running_mode != RUNNING_MODE_LOCATION: + time.sleep(0.5) + continue + # 获取一帧图像 + _, frame = capture.read() + # 如果图像为空，等待一秒，上报消息 + if frame is None: + time.sleep(0.1) + self.logger.error("[ERROR] 广角相机未捕获到图像") + continue + + src_frame_ptr = c_char_p(frame.data.tobytes()) + ret = self.image_framework_sdk.LibapiCameraSendMsgWithImage( + src_frame_ptr, frame.shape[1], frame.shape[0], 3, + c_char_p(b"PythonProcessing"), c_char_p(b"_t_PluginCameraPro")) + if ret != SYS_OK: + self.logger.error(_get_msg_info(ret)) + # emit msg + continue + # 向 g_frame_cache 中加入消息，对图像进行二次加工 + g_frame_cache.append(frame) + + # 高清相机运行检测处理线程 + def _main_processing_thread(self): + global g_msg_cache, g_frame_cache, g_running_mode + + while True: + if len(g_msg_cache) <= 0: + time.sleep(0.01) + continue + + msg = g_msg_cache.pop() + if msg.msg_type == MSG_LOCATION_RECORD: + # cv2.imshow("LOCATION", msg.im2D) + # cv2.waitKey(1) + self.signals.on_image_detect_result.emit(msg) diff --git a/config.ini b/config.ini index 2f0de86..47b265f 100644 --- a/config.ini +++ b/config.ini @@ -14,6 +14,7 @@ k3=-0.15639485 yolo_label=0 yolo_prob=0.25 yolo_modol_path=./best_rgb_ymj_csc_80_2.om +save_image=true # 3D config [3D] @@ -30,6 +31,8 @@ dmaxy=2.5 dminz=0.0 dmaxz=3.8 kk=0.02 +cloud_need_points_size=300000 +save_cload=true # system config [sys] diff --git a/req.txt b/req.txt index 6141723..4746cb4 100644 --- a/req.txt +++ b/req.txt @@ -8,4 +8,5 @@ paho-mqtt==2.1.0 dynaconf==3.2.5 fastapi==0.111.0 uvicorn==0.30.1 -SQLAlchemy==2.0.34 \ No newline at end of file +SQLAlchemy==2.0.34 +opencv-python==4.10.0.84 \ No newline at end of file diff --git a/widget/task_run.py b/widget/task_run.py index d9047ed..df990c1 100644 --- a/widget/task_run.py +++ b/widget/task_run.py @@ -1,8 +1,10 @@ +import cv2 from PyQt5.QtCore import Qt -from PyQt5.QtGui import QPixmap +from PyQt5.QtGui import QPixmap, QImage from PyQt5.QtWidgets import QDialog, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QScrollArea, QPushButton, \ QSpacerItem, QSizePolicy, QLabel +from components.image_framework import MSG_LOCATION_RECORD from core.context import AppContext from widget.embed_item import EmbedItem @@ -23,17 +25,32 @@ class TaskRunDialog(QDialog): self.check_widget = None self.embed_widgets = [] + self.view_label = None + self.picture_label = None + self.start_adjust_button = None self.start_check_button = None self.stop_check_button = None self.stop_task_button = None self.init_ui() + self.init_event() - AppContext.get_edge_context().get_component('image-framework').signals.on_image_result.connect(lambda data: self.image_result(data)) + def init_event(self): + AppContext.get_edge_context().get_component('image-framework').signals.on_image_detect_result.connect( + lambda data: self.image_result(data)) + + def image_result(self, msg): + if msg.msg_type == MSG_LOCATION_RECORD: + rgb_frame = cv2.cvtColor(msg.im2D, cv2.COLOR_BGR2RGB) + + # 转换为 QImage + height, width, channels = rgb_frame.shape + bytes_per_line = channels * width + qt_image = QImage(rgb_frame.data, width, height, bytes_per_line, QImage.Format_RGB888) + + self.view_label.setPixmap(QPixmap.fromImage(qt_image)) - def image_result(self, data): - ccc = data def init_ui(self): # bim @@ -115,31 +132,31 @@ class TaskRunDialog(QDialog): content_widget = QWidget() content_widget.setObjectName("contentWidget") - picture_label = QLabel() - picture_label.setObjectName("pictureWidget") - picture_label.setFixedWidth(450) - picture_label.setFixedHeight(450) - picture_label.setScaledContents(True) + self.picture_label = QLabel() + self.picture_label.setObjectName("pictureWidget") + self.picture_label.setFixedWidth(450) + self.picture_label.setFixedHeight(450) + self.picture_label.setScaledContents(True) picture_pixmap = QPixmap("assets/img2.png") - picture_label.setPixmap(picture_pixmap) + self.picture_label.setPixmap(picture_pixmap) view_widget = QWidget() view_widget.setObjectName("viewWidget") view_widget.setFixedHeight(450) - view_label = QLabel(view_widget) - view_label.setObjectName("viewLabel") - view_label.setFixedHeight(450) - view_label.setScaledContents(True) + self.view_label = QLabel(view_widget) + self.view_label.setObjectName("viewLabel") + self.view_label.setFixedHeight(450) + self.view_label.setScaledContents(True) view_pixmap = QPixmap("assets/img1.jpg") sc = view_pixmap.height() / 450 - view_label.setFixedWidth(view_pixmap.width() / sc) - view_label.setPixmap(view_pixmap) - view_label.setGeometry((view_widget.width() - view_label.width() / 2), 0, view_label.width(), view_label.height()) + self.view_label.setFixedWidth(view_pixmap.width() / sc) + self.view_label.setPixmap(view_pixmap) + self.view_label.setGeometry((view_widget.width() - self.view_label.width() / 2), 0, self.view_label.width(), self.view_label.height()) view_check_widget = QWidget(view_widget) view_check_widget.setObjectName("viewCheckWidget") - view_check_widget.setGeometry(view_label.geometry().x() + 145, 170, 240, 240) + view_check_widget.setGeometry(self.view_label.geometry().x() + 145, 170, 240, 240) # view_widget = QWidget() # view_widget.setObjectName("viewWidget") @@ -202,7 +219,7 @@ class TaskRunDialog(QDialog): content_layout.setSpacing(0) content_layout.addWidget(view_widget) content_layout.addSpacing(20) - content_layout.addWidget(picture_label) + content_layout.addWidget(self.picture_label) content_layout.addSpacing(20) content_layout.addWidget(tool_widget) content_widget.setLayout(content_layout) @@ -256,3 +273,6 @@ class TaskRunDialog(QDialog): self.start_check_button.setEnabled(False) # self.stop_check_button.setEnabled(False) self.checkIndex += 1 + + def showEvent(self, e): + AppContext.get_edge_context().get_component('image-framework').start_location() \ No newline at end of file