1、image framework

.idea/detect-gui.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/venv" />
     </content>
-    <orderEntry type="jdk" jdkName="detect-gui-env" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="detect-gui" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -3,5 +3,5 @@
   <component name="Black">
     <option name="sdkName" value="detect-gui" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="detect-gui-env" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="detect-gui" project-jdk-type="Python SDK" />
 </project>

components/image_framework.py

@@ -1,4 +1,5 @@
 import os
+import threading
 from ctypes import *
 import numpy as np
 from PyQt5.QtCore import QObject, pyqtSignal
@@ -6,6 +7,7 @@ from dynaconf.base import Settings
 from numpy.ctypeslib import as_array
 import platform
 import time
+import cv2
 
 from core.edge_component import action, EdgeComponent, service
 from util import get_system_and_library_suffix
@@ -49,10 +51,139 @@ CallbackInfo = _CallbackInfo_
 # 回调函数类型定义
 CallbackType = CFUNCTYPE(None, POINTER(CallbackInfo))
 
+# emit的结构体
+MSG_ONLY_RECORD         = -1
+MSG_LOCATION_RECORD     = 0
+MSG_DETECTION_RECORD    = 1
+
+class Msg:
+    def __init__(self, msg_type, record, im2D):
+        self.msg_type = msg_type  # 0：刷新广角图片， 1：刷新高清， -1：不刷新图片，纯消息
+        self.record = record
+        self.im2D = im2D
+
+
+TO_CLIENT_NOTIFY_BASE =  1000
+SYS_BASE = 1000
+CAL_2D3D_BASE = 2000
+CAMERA_BASE = 4000
+LIDAR_BASE = 5000
+SYS_OK = 0
+CAL_OK = 0
+LIDAR_OK = 0
+MSG_LIST = [
+    { 1 + TO_CLIENT_NOTIFY_BASE : "开始相机拍摄" },
+    { 2 + TO_CLIENT_NOTIFY_BASE : "相机拍摄已完成" },
+    { 3 + TO_CLIENT_NOTIFY_BASE : "开始雷达采集数据" },
+    { 4 + TO_CLIENT_NOTIFY_BASE : "雷达采集数据已完成" },
+    { 5 + TO_CLIENT_NOTIFY_BASE : "开始定位检测" }, # 这个消息是相机实时定位，不用打印
+    { 6 + TO_CLIENT_NOTIFY_BASE : "定位检测已完成" }, 
+    { 7 + TO_CLIENT_NOTIFY_BASE : "开始角点2D检测" },
+    { 8 + TO_CLIENT_NOTIFY_BASE : "角点2D检测已完成" },
+    { 9 + TO_CLIENT_NOTIFY_BASE : "开始角点3D检测" },
+    { 10 + TO_CLIENT_NOTIFY_BASE : "角点3D检测已完成" },
+    { 11 + TO_CLIENT_NOTIFY_BASE : "开始计算预埋件尺寸" },
+    { 12 + TO_CLIENT_NOTIFY_BASE : "预埋件尺寸计算已完成" },
+
+    { 1 - SYS_BASE : "驱动初始化错误" },
+    { 2 - SYS_BASE : "NPU计算单元初始化错误" },
+    { 3 - SYS_BASE : "系统错误" },
+    { 4 - SYS_BASE : "相机初始化错误" },
+    { 5 - SYS_BASE : "当前没有可以读取的高清相片"},
+
+    { 1 - CAL_2D3D_BASE : "读取图片文件错误" }, 
+    { 2 - CAL_2D3D_BASE : "写图片文件错误" },
+    { 3 - CAL_2D3D_BASE : "空指针" },
+    { 4 - CAL_2D3D_BASE : "空参数" },
+    { 5 - CAL_2D3D_BASE : "读取图像错误" },
+    { 6 - CAL_2D3D_BASE : "NPU加载数据错误" },
+    { 7 - CAL_2D3D_BASE : "NPU模型执行错误" },
+    { 8 - CAL_2D3D_BASE : "NPU获取结果错误" },
+    { 9 - CAL_2D3D_BASE : "NPU卸载数据错误" },
+    { 10 - CAL_2D3D_BASE : "YOLO推理无可信ROI区域" },
+    { 11 - CAL_2D3D_BASE : "ROI处理线程错误" },
+    { 12 - CAL_2D3D_BASE : "H计算错误" },
+    { 13 - CAL_2D3D_BASE : "3D角点检测错误" },
+    { 14 - CAL_2D3D_BASE : "2D角点定位错误" },
+    { 15 - CAL_2D3D_BASE : "3D投影计算错误" },
+    { 16 - CAL_2D3D_BASE : "3D点云为空" },
+    { 17 - CAL_2D3D_BASE : "未检测到2D角点" },
+    { 18 - CAL_2D3D_BASE : "未检测到预埋件" },
+    { 19 - CAL_2D3D_BASE : "相机捕获图像失败" },
+
+    { 1 - CAMERA_BASE : "相机DLL初始化失败" },
+
+    { 1 - LIDAR_BASE : "雷达初始化SDK失败" },
+    { 2 - LIDAR_BASE : "雷达SO初始化失败" },
+    { 3 - LIDAR_BASE : "雷达广播失败" },
+    { 5 - LIDAR_BASE : "雷达开始采样失败" },
+    { 6 - LIDAR_BASE : "雷达结束采样失败" },
+
+    { -10001 : "已经在检测模式中"},
+    { -10002 : "定位模式不支持执行该操作"}
+]
+
+#################################################################################### 
+#                                   全局变量定义
+#################################################################################### 
+
+# 全局 msg 缓存
+g_msg_cache = []
+
+# 全局 frame 缓存
+g_frame_cache = []
+
+# 运行模式
+RUNNING_MODE_NONE = -1 # 停止状态
+RUNNING_MODE_LOCATION = 0 # 广角相机运行模式
+RUNNING_MODE_DETECTION = 1 # 高清检测运行模式
+g_running_mode = RUNNING_MODE_LOCATION # 默认是广角相机运行模式
+
+#################################################################################### 
+#                                   内部接口定义
+#################################################################################### 
+# 获取消息
+def _get_msg_info(code) -> str: 
+    global MSG_LIST
+    for msg in MSG_LIST:
+        if code == msg.keys()[0]:
+            return msg.values
+    return "未知错误"
+
+# 加工检测图像
+def _update_location_image(frame):
+    global needHandle, recordCache, ckinfoCache
+    def switch(choice):
+        if ckinfoCache.code == 1006: #TO_CLIENT_NOTIFY_DETECTION_END: 1006
+            for i in range(recordCache.roi_size):
+                subRois = recordCache.subRois[i]
+                s = ""
+                for j in range(8):
+                    s += str(subRois.mRoiPoints[j])+ " "
+        else:
+            print("default Case")
+
+# 在回调函数中将 record 数据拷贝出来
+def _copy_record(record):
+    new_record = CallbackInfo()
+    new_record.code = record.code
+    new_record.errorInfo = record.errorInfo
+    new_record.bGetData = record.bGetData
+    for i in range(record.roi_size):
+        subRois = record.subRois[i]
+        newSubRois = new_record.subRois[i]
+        for j in range(8):
+            newSubRois.mRoiPoints[j] = subRois.mRoiPoints[j]
+            newSubRois.mInnerConners[j] = subRois.mInnerConners[j]
+        for j in range(12):
+            newSubRois.mInnerConners3D[j] = subRois.mInnerConners3D[j]
+
+    return new_record
+
 
 class ImageFrameworkSignals(QObject):
     # QT 信号
-    on_image_result = pyqtSignal(object)
+    on_image_detect_result = pyqtSignal(Msg)
 
 
 @action("image-framework", auto_start=True)
@@ -63,9 +194,16 @@ class ImageFramework(EdgeComponent):
         super().__init__(context)
         self.image_framework_sdk = None
         self.data_callback = None
+        self.location_thread = None
+        self.process_thread = None
 
     def configure(self, setting: Settings) -> None:
         self.init_image_framework_sdk()
+        self.location_thread = threading.Thread(target=self._location_processing_thread)
+        self.location_thread.daemon = True
+        self.process_thread = threading.Thread(target=self._main_processing_thread)
+        self.process_thread.daemon = True
+
         self.logger.info(f"Image framework configure done.")
 
     def init_image_framework_sdk(self):
@@ -81,23 +219,42 @@ class ImageFramework(EdgeComponent):
 
     def init_callback(self):
         def _callback(data):
-            contents = data.contents
+            global g_msg_cache, g_frame_cache, g_running_mode
-            self.logger.info(f"image framework callback data, bGetData:{contents.bGetData}")
-            if contents.bGetData:
-                record = CallbackInfo()
-                self.image_framework_sdk.LibapGetRecord(byref(record))
-                self.logger.info(f"image framework callback data, roi_size:{record.roi_size}")
-                self.logger.info(f"image framework callback data, code:{record.code}")
-                self.logger.info(f"image framework callback data, errorInfo:{record.errorInfo}")
-                for i in range(record.roi_size):
-                    roi_data = record.subRois[i]
-                    self.logger.info(f"roi data, roi_data[{i}].mId:{roi_data.mId}")
-                    self.logger.info(f"roi data, roi_data[{i}].isGood:{roi_data.isGood}")
-                    self.logger.info(f"roi data, roi_data[{i}].mRoiPoints:{list(roi_data.mRoiPoints)}")
 
-                self.signals.on_image_result.emit(record.roi_size)
+            record = CallbackInfo(code=SYS_OK, errorInfo=b"", bGetData=False)
+            frame = None
+            ret = SYS_OK
+            record.code = data.contents.code
+            record.errorInfo = data.contents.errorInfo
+            # 当定位模式下，接收到【定位完成】消息
+            if g_running_mode == RUNNING_MODE_LOCATION and record.code == (6 + TO_CLIENT_NOTIFY_BASE):
+                if len(g_frame_cache) == 0:
+                    self.logger.error("当前没有广角数据")
+                    return
 
-            self.logger.info("image framework callback done.")
+                frame = g_frame_cache.pop()  # 从图像缓存中去除图像
+                g_frame_cache = []  # 清空缓存，避免无限增长
+                msg = Msg(MSG_LOCATION_RECORD, _copy_record(record), frame)
+            # 当检测模式下，接收到【拍照完成】消息
+            elif g_running_mode == RUNNING_MODE_DETECTION and record.code == (2 + TO_CLIENT_NOTIFY_BASE):
+                frame.zeros(9344, 7000, cv2.CV_8UC3)
+                src_frame_ptr = c_char_p(frame.data.tobytes())
+                ret = self.image_framework_sdk.LibapiGetHQImage(
+                    src_frame_ptr, frame.shape[1], frame.shape[0], 3, c_char_p(b"PythonProcessing"))
+                if ret != SYS_OK:
+                    record = CallbackInfo(code=ret, errorInfo=_get_msg_info(ret), bGetData=False)
+                    msg = Msg(MSG_ONLY_RECORD, record, None)
+                    # emit msg
+                    return
+                msg = Msg(MSG_DETECTION_RECORD, _copy_record(record), frame)
+            elif g_running_mode == RUNNING_MODE_LOCATION or g_running_mode == RUNNING_MODE_DETECTION:
+                msg = Msg(MSG_ONLY_RECORD, _copy_record(record), None)
+            else:
+                print("未知回调")
+                return
+
+            g_msg_cache.append(msg)
+            # self.logger.info("image framework callback done.")
 
         self.data_callback = CallbackType(_callback)
 
@@ -115,11 +272,25 @@ class ImageFramework(EdgeComponent):
 
     @service()
     def stop(self):
+        if self.location_thread is not None:
+            self.location_thread.join()
+        if self.process_thread is not None:
+            self.process_thread.join()
+
         self.image_framework_sdk.stop_sdk()
         self.logger.info("Image framework stopped.")
 
+    @service()
+    def start_location(self):
+        self.location_thread.start()
+        self.process_thread.start()
+
     @service()
     def start_detect(self):
+        global g_running_mode
+        if g_running_mode != RUNNING_MODE_DETECTION:
+            g_running_mode = RUNNING_MODE_DETECTION  # 切换为检测模式
+
         ret = self.image_framework_sdk.LibapiStartDetection()
         if ret != 0:
             raise RuntimeError("启动检测失败!")
@@ -132,6 +303,55 @@ class ImageFramework(EdgeComponent):
 
     @service()
     def stop_detect(self):
+        global g_running_mode
+        if g_running_mode == RUNNING_MODE_LOCATION:  # 如果在定位任务模式下，该接口不执行
+            raise RuntimeError("正在检测，请稍后停止!")
+
+        g_running_mode = RUNNING_MODE_NONE  # 切换为无模式
         ret = self.image_framework_sdk.LibapStopDetection()
         if ret != 0:
             raise RuntimeError("停止检测失败!")
+
+    # 广角相机运行定位处理线程
+    def _location_processing_thread(self):
+        global g_msg_cache, g_frame_cache, g_running_mode
+        # 打开广角摄像头
+        capture = cv2.VideoCapture(0)
+        while True:
+            # 如果当前不是定位模式，不处理任何事务
+            if g_running_mode != RUNNING_MODE_LOCATION:
+                time.sleep(0.5)
+                continue
+            # 获取一帧图像
+            _, frame = capture.read()
+            # 如果图像为空，等待一秒，上报消息
+            if frame is None:
+                time.sleep(0.1)
+                self.logger.error("[ERROR] 广角相机未捕获到图像")
+                continue
+
+            src_frame_ptr = c_char_p(frame.data.tobytes())
+            ret = self.image_framework_sdk.LibapiCameraSendMsgWithImage(
+                src_frame_ptr, frame.shape[1], frame.shape[0], 3,
+                c_char_p(b"PythonProcessing"), c_char_p(b"_t_PluginCameraPro"))
+            if ret != SYS_OK:
+                self.logger.error(_get_msg_info(ret))
+                # emit msg
+                continue
+            # 向 g_frame_cache 中加入消息，对图像进行二次加工
+            g_frame_cache.append(frame)
+
+    # 高清相机运行检测处理线程
+    def _main_processing_thread(self):
+        global g_msg_cache, g_frame_cache, g_running_mode
+
+        while True:
+            if len(g_msg_cache) <= 0:
+                time.sleep(0.01)
+                continue
+
+            msg = g_msg_cache.pop()
+            if msg.msg_type == MSG_LOCATION_RECORD:
+                # cv2.imshow("LOCATION", msg.im2D)
+                # cv2.waitKey(1)
+                self.signals.on_image_detect_result.emit(msg)

config.ini

@@ -14,6 +14,7 @@ k3=-0.15639485
 yolo_label=0
 yolo_prob=0.25
 yolo_modol_path=./best_rgb_ymj_csc_80_2.om
+save_image=true
 
 # 3D config
 [3D]
@@ -30,6 +31,8 @@ dmaxy=2.5
 dminz=0.0
 dmaxz=3.8
 kk=0.02
+cloud_need_points_size=300000
+save_cload=true
 
 # system config
 [sys]

req.txt

@@ -9,3 +9,4 @@ dynaconf==3.2.5
 fastapi==0.111.0
 uvicorn==0.30.1
 SQLAlchemy==2.0.34
+opencv-python==4.10.0.84

widget/task_run.py

@@ -1,8 +1,10 @@
+import cv2
 from PyQt5.QtCore import Qt
-from PyQt5.QtGui import QPixmap
+from PyQt5.QtGui import QPixmap, QImage
 from PyQt5.QtWidgets import QDialog, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QScrollArea, QPushButton, \
     QSpacerItem, QSizePolicy, QLabel
 
+from components.image_framework import MSG_LOCATION_RECORD
 from core.context import AppContext
 from widget.embed_item import EmbedItem
 
@@ -23,17 +25,32 @@ class TaskRunDialog(QDialog):
         self.check_widget = None
         self.embed_widgets = []
 
+        self.view_label = None
+        self.picture_label = None
+
         self.start_adjust_button = None
         self.start_check_button = None
         self.stop_check_button = None
         self.stop_task_button = None
 
         self.init_ui()
+        self.init_event()
 
-        AppContext.get_edge_context().get_component('image-framework').signals.on_image_result.connect(lambda data: self.image_result(data))
+    def init_event(self):
+        AppContext.get_edge_context().get_component('image-framework').signals.on_image_detect_result.connect(
+            lambda data: self.image_result(data))
+
+    def image_result(self, msg):
+        if msg.msg_type == MSG_LOCATION_RECORD:
+            rgb_frame = cv2.cvtColor(msg.im2D, cv2.COLOR_BGR2RGB)
+
+            # 转换为 QImage
+            height, width, channels = rgb_frame.shape
+            bytes_per_line = channels * width
+            qt_image = QImage(rgb_frame.data, width, height, bytes_per_line, QImage.Format_RGB888)
+
+            self.view_label.setPixmap(QPixmap.fromImage(qt_image))
 
-    def image_result(self, data):
-        ccc = data
 
     def init_ui(self):
         # bim
@@ -115,31 +132,31 @@ class TaskRunDialog(QDialog):
         content_widget = QWidget()
         content_widget.setObjectName("contentWidget")
 
-        picture_label = QLabel()
+        self.picture_label = QLabel()
-        picture_label.setObjectName("pictureWidget")
+        self.picture_label.setObjectName("pictureWidget")
-        picture_label.setFixedWidth(450)
+        self.picture_label.setFixedWidth(450)
-        picture_label.setFixedHeight(450)
+        self.picture_label.setFixedHeight(450)
-        picture_label.setScaledContents(True)
+        self.picture_label.setScaledContents(True)
         picture_pixmap = QPixmap("assets/img2.png")
-        picture_label.setPixmap(picture_pixmap)
+        self.picture_label.setPixmap(picture_pixmap)
 
         view_widget = QWidget()
         view_widget.setObjectName("viewWidget")
         view_widget.setFixedHeight(450)
 
-        view_label = QLabel(view_widget)
+        self.view_label = QLabel(view_widget)
-        view_label.setObjectName("viewLabel")
+        self.view_label.setObjectName("viewLabel")
-        view_label.setFixedHeight(450)
+        self.view_label.setFixedHeight(450)
-        view_label.setScaledContents(True)
+        self.view_label.setScaledContents(True)
         view_pixmap = QPixmap("assets/img1.jpg")
         sc = view_pixmap.height() / 450
-        view_label.setFixedWidth(view_pixmap.width() / sc)
+        self.view_label.setFixedWidth(view_pixmap.width() / sc)
-        view_label.setPixmap(view_pixmap)
+        self.view_label.setPixmap(view_pixmap)
-        view_label.setGeometry((view_widget.width() - view_label.width() / 2), 0, view_label.width(), view_label.height())
+        self.view_label.setGeometry((view_widget.width() - self.view_label.width() / 2), 0, self.view_label.width(), self.view_label.height())
 
         view_check_widget = QWidget(view_widget)
         view_check_widget.setObjectName("viewCheckWidget")
-        view_check_widget.setGeometry(view_label.geometry().x() + 145, 170, 240, 240)
+        view_check_widget.setGeometry(self.view_label.geometry().x() + 145, 170, 240, 240)
 
         # view_widget = QWidget()
         # view_widget.setObjectName("viewWidget")
@@ -202,7 +219,7 @@ class TaskRunDialog(QDialog):
         content_layout.setSpacing(0)
         content_layout.addWidget(view_widget)
         content_layout.addSpacing(20)
-        content_layout.addWidget(picture_label)
+        content_layout.addWidget(self.picture_label)
         content_layout.addSpacing(20)
         content_layout.addWidget(tool_widget)
         content_widget.setLayout(content_layout)
@@ -256,3 +273,6 @@ class TaskRunDialog(QDialog):
         self.start_check_button.setEnabled(False)
         # self.stop_check_button.setEnabled(False)
         self.checkIndex += 1
+
+    def showEvent(self, e):
+        AppContext.get_edge_context().get_component('image-framework').start_location()