添加deepcraft-ai示例工程

Author: Rbb666

.github/workflows/action.yml

@@ -65,6 +65,7 @@ jobs:
          - {RTT_BSP: "Edgi_Talk_CherryUSB/Edgi_Talk_Extend_Screen"}
          - {RTT_BSP: "Edgi_Talk_CherryUSB/Edgi_Talk_M55_USB_UVC"}
          - {RTT_BSP: "Edgi_Talk_CherryUSB/Edgi_Talk_USB_Deepcraft"}
+         - {RTT_BSP: "Edgi_Talk_M55_DEEPCRAFT_Deploy_Vision"}
          - {RTT_BSP: "Edgi_Talk_M55_CoreMark"}
          - {RTT_BSP: "Edgi_Talk_M33_CDC_Echo"}
          - {RTT_BSP: "Edgi_Talk_M33_HyperRam"}

libraries/Common/board/ports/usb/SConscript

@@ -1,19 +1,48 @@
 from building import *
 
 cwd = GetCurrentDir()
+project_root = Dir('#').abspath.replace('\\', '/')
+cherryusb_root = project_root + '/libraries/components/CherryUSB-1.6.0'
+build_uvc_from_source = GetDepend(['RT_CHERRYUSB_HOST_BUILD_FROM_SOURCE'])
+
 src = [
     'usbh_uvc_app.c',
     'usbh_uvc_display.c',
     'tjpgd.c',
 ]
 
+cpp_path = [
+    cwd,
+    project_root + '/applications',
+]
+
+libs = []
+libpath = []
+
+if build_uvc_from_source:
+    src += [
+        'usbh_uvc_port.c',
+        'usbh_uvc_stream.c',
+    ]
+    cpp_path += [
+        cherryusb_root + '/common',
+        cherryusb_root + '/core',
+        cherryusb_root + '/class/hub',
+        cherryusb_root + '/class/video',
+        cherryusb_root + '/third_party/cherrymp',
+        cherryusb_root + '/port/dwc2',
+    ]
+else:
+    libpath = [cwd]
+    libs = ['usbh_uvc']
+
 group = DefineGroup(
     'UsbUvc',
     src,
     depend=['RT_CHERRYUSB_HOST_VIDEO'],
-    CPPPATH=[cwd],
-    LIBPATH=[cwd],
-    LIBS=['usbh_uvc_port', 'usbh_uvc_stream'],
+    CPPPATH=cpp_path,
+    LIBPATH=libpath,
+    LIBS=libs,
 )
 
 Return('group')

…mon/board/ports/usb/libusbh_uvc_stream.a …ies/Common/board/ports/usb/libusbh_uvc.alibraries/Common/board/ports/usb/libusbh_uvc_stream.a renamed to libraries/Common/board/ports/usb/libusbh_uvc.a libraries/Common/board/ports/usb/libusbh_uvc_stream.a renamed to libraries/Common/board/ports/usb/libusbh_uvc.a

@@ -7,6 +7,33 @@
 #include <rtdevice.h>
 #include "usbh_uvc_stream.h"
 
+#ifdef BSP_USING_DEEPCRAFT_AI
+#include "uvc_ai_app.h"
+#else
+static void uvc_ai_app_enforce_mode(uint8_t *fmt, uint16_t *width, uint16_t *height)
+{
+    (void)fmt;
+    (void)width;
+    (void)height;
+}
+
+static int uvc_ai_app_start(uint16_t src_width, uint16_t src_height)
+{
+    (void)src_width;
+    (void)src_height;
+    return 0;
+}
+
+static void uvc_ai_app_process_frame(const struct usbh_videoframe *frame)
+{
+    (void)frame;
+}
+
+static void uvc_ai_app_stop(void)
+{
+}
+#endif
+
 #undef  USB_DBG_TAG
 #define USB_DBG_TAG "uvc_app"
 #include "usb_log.h"
@@ -32,10 +59,44 @@ static volatile uint8_t uvc_app_running;
 static volatile uint32_t g_uvc_display_fps;
 static volatile uint32_t g_uvc_display_drop_count;
 
+static void uvc_filter_empty_formats(struct usbh_video *video_class)
+{
+    uint8_t read_idx;
+    uint8_t write_idx;
+
+    if (video_class == RT_NULL) {
+        return;
+    }
+
+    /*
+     * Some cameras expose an extra VS format entry with zero frames.
+     * Keep CherryUSB core unchanged and compact valid entries here,
+     * so usbh_video_open() does not pick an empty format.
+     */
+    write_idx = 0U;
+    for (read_idx = 0U; read_idx < video_class->num_of_formats; read_idx++) {
+        if (video_class->format[read_idx].num_of_frames == 0U) {
+            continue;
+        }
+
+        if (write_idx != read_idx) {
+            video_class->format[write_idx] = video_class->format[read_idx];
+        }
+        write_idx++;
+    }
+
+    if (write_idx != video_class->num_of_formats) {
+        USB_LOG_WRN("UVC filter empty format: %u -> %u\r\n",
+                    video_class->num_of_formats, write_idx);
+        video_class->num_of_formats = write_idx;
+    }
+}
+
 /* ---------- CherryUSB video class callbacks ---------- */
 
 void usbh_video_run(struct usbh_video *video_class)
 {
+    uvc_filter_empty_formats(video_class);
     USB_LOG_INFO("UVC device connected\r\n");
     usbh_video_list_info(video_class);
 }
@@ -52,15 +113,6 @@ void usbh_video_stop(struct usbh_video *video_class)
 extern volatile uint32_t g_uvc_fps;
 extern volatile uint32_t uvc_transfer_count;
 extern volatile uint32_t video_complete_count;
-extern volatile uint32_t uvc_dbg_eof_count;
-extern volatile uint32_t uvc_dbg_fid_toggle;
-extern volatile uint32_t uvc_dbg_empty_pkt;
-extern volatile uint32_t uvc_dbg_data_pkt;
-extern volatile uint32_t uvc_dbg_last_eof_offset;
-extern volatile uint32_t uvc_dbg_frame_done;
-extern volatile uint32_t uvc_dbg_err_bit;
-extern volatile uint32_t uvc_dbg_payload_bytes;
-extern volatile uint32_t uvc_dbg_hdronly_pkt;
 
 static void uvc_display_fps_record(void)
 {
@@ -104,16 +156,10 @@ static void uvc_frame_keep_latest(struct usbh_videoframe **frame)
 static void uvc_fps_thread_entry(void *arg)
 {
     while (uvc_app_running) {
-        USB_LOG_INFO("UVC fps:%d disp:%d drop:%u urb:%d eof:%d fid:%d err:%d empty:%d data:%d hdronly:%d payload:%d done:%d last_off:%u\r\n",
+        USB_LOG_INFO("UVC fps:%d disp:%d drop:%u urb:%d\r\n",
                      (int)g_uvc_fps, (int)g_uvc_display_fps,
                      (unsigned)g_uvc_display_drop_count,
-                     (int)video_complete_count,
-                     (int)uvc_dbg_eof_count, (int)uvc_dbg_fid_toggle,
-                     (int)uvc_dbg_err_bit, (int)uvc_dbg_empty_pkt,
-                     (int)uvc_dbg_data_pkt,
-                     (int)uvc_dbg_hdronly_pkt, (int)uvc_dbg_payload_bytes,
-                     (int)uvc_dbg_frame_done,
-                     (unsigned)uvc_dbg_last_eof_offset);
+                     (int)video_complete_count);
         rt_thread_mdelay(3000);
     }
 }
@@ -132,6 +178,7 @@ static void uvc_frame_thread_entry(void *arg)
 
     /* initialise display output */
     uvc_display_init();
+    (void)uvc_ai_app_start(uvc_cam_w, uvc_cam_h);
 
     while (uvc_app_running) {
         ret = usbh_video_stream_dequeue(&frame, RT_TICK_PER_SECOND);
@@ -149,6 +196,7 @@ static void uvc_frame_thread_entry(void *arg)
         }
 
         usbh_video_stream_get_info(&stream_w, &stream_h, RT_NULL);
+        uvc_ai_app_process_frame(frame);
 
         /* render frame to LCD */
         uvc_display_frame(frame,
@@ -158,14 +206,18 @@ static void uvc_frame_thread_entry(void *arg)
 
         usbh_video_stream_enqueue(frame);
     }
+
+    uvc_ai_app_stop();
 }
 
 /* ---------- msh commands ---------- */
 
 static int cmd_usbh_uvc_start(int argc, char **argv)
 {
     uint8_t fmt = USBH_VIDEO_FORMAT_UNCOMPRESSED;  /* default: YUYV */
-    uint16_t w = 640, h = 480;
+    uint16_t w = 320;
+    uint16_t h = 240;
+    uint32_t frame_bufsize = UVC_FRAME_BUF_SIZE;
     int ret;
 
     if (uvc_app_running) {
@@ -181,14 +233,27 @@ static int cmd_usbh_uvc_start(int argc, char **argv)
         h = atoi(argv[3]);
     }
 
+    uvc_ai_app_enforce_mode(&fmt, &w, &h);
+
     USB_LOG_INFO("UVC start: %ux%u format=%s\r\n", w, h,
                  fmt == USBH_VIDEO_FORMAT_MJPEG ? "mjpeg" : "yuyv");
 
+    if (fmt == USBH_VIDEO_FORMAT_UNCOMPRESSED) {
+        uint32_t yuyv_size = (uint32_t)w * (uint32_t)h * 2U;
+
+        if ((w == 0U) || (h == 0U) || (yuyv_size > UVC_FRAME_BUF_SIZE)) {
+            USB_LOG_ERR("Invalid YUYV frame size: %ux%u (%lu)\r\n",
+                        w, h, (unsigned long)yuyv_size);
+            return -RT_EINVAL;
+        }
+        frame_bufsize = yuyv_size;
+    }
+
     /* initialise frame pool */
     {
         for (uint32_t i = 0; i < UVC_FRAME_BUF_COUNT; i++) {
             frame_pool[i].frame_buf = frame_buffer[i];
-            frame_pool[i].frame_bufsize = UVC_FRAME_BUF_SIZE;
+            frame_pool[i].frame_bufsize = frame_bufsize;
         }
     }
 
@@ -213,7 +278,7 @@ static int cmd_usbh_uvc_start(int argc, char **argv)
     if (t) rt_thread_startup(t);
 
     t = rt_thread_create("uvc_frm", uvc_frame_thread_entry, NULL,
-                         8192, 22, 10);
+                         65536, 22, 10);
     if (t) rt_thread_startup(t);
 
     /* start the video stream (frames are queued inside stream_start) */

libraries/Common/board/ports/usb/libusbh_uvc_port.a

@@ -13,6 +13,7 @@
 #include "cy_graphics.h"
 #include "usbh_uvc_stream.h"
 #include "tjpgd.h"
+#include "usbh_uvc_display_hook.h"
 
 #undef  USB_DBG_TAG
 #define USB_DBG_TAG "uvc_disp"
@@ -28,20 +29,28 @@
 #define UVC_ITCM_SECTION __attribute__((section(".cy_itcm")))
 #define UVC_DTCM_SECTION __attribute__((section(".cy_dtcm")))
 #define UVC_SHARED_SECTION __attribute__((section(".m33_m55_shared_hyperram")))
+#define UVC_RGB565_STAGE_SECTION __attribute__((section(".cy_uvc_rgb565_data")))
 
 #define UVC_MJPEG_WORKBUF_SIZE   4096U
 #define UVC_MJPEG_MAX_FRAME_SIZE (128U * 1024U)
 #define UVC_MJPEG_MAX_WIDTH      640U
 #define UVC_MJPEG_MAX_HEIGHT     480U
 #define UVC_MJPEG_MAX_PIXELS     (UVC_MJPEG_MAX_WIDTH * UVC_MJPEG_MAX_HEIGHT)
+#define UVC_YUYV_STAGE_W         320U
+#define UVC_YUYV_STAGE_H         240U
+#define UVC_YUYV_STAGE_PIXELS    (UVC_YUYV_STAGE_W * UVC_YUYV_STAGE_H)
+#define UVC_YUYV_STAGE_BYTES     (UVC_YUYV_STAGE_PIXELS * sizeof(uint16_t))
 
 static uint16_t   *lcd_fb;          /* pointer into LCD framebuffer */
 static uint32_t    lcd_fb_size;     /* bytes */
 static rt_bool_t   g_uvc_lcd_hw_ready;
 
 static GFXSS_Type *g_uvc_gfxbase = (GFXSS_Type*)GFXSS;
 static cy_stc_gfx_context_t g_uvc_gfx_context;
-CY_SECTION(".cy_gpu_buf") static uint8_t g_uvc_lcd_fb_mem[LCD_BUF_SIZE] __attribute__((aligned(128))) = {0};
+/* Reuse LCD driver's framebuffer instead of allocating an additional 800x512 RGB565 surface. */
+extern uint8_t graphics_buffer[LCD_BUF_SIZE];
+static uvc_display_overlay_cb_t g_uvc_overlay_cb;
+static void *g_uvc_overlay_cb_ctx;
 
 struct uvc_update_rect {
     uint16_t x;
@@ -108,6 +117,8 @@ static UVC_DTCM_SECTION int32_t g_uvc_v_to_g_lut[256];
 static UVC_DTCM_SECTION uint8_t g_uvc_mjpeg_diag_budget = 8U;
 static UVC_SHARED_SECTION uint16_t g_uvc_mjpeg_rgb565[UVC_MJPEG_MAX_PIXELS];
 static UVC_SHARED_SECTION uint8_t g_uvc_mjpeg_frame[UVC_MJPEG_MAX_FRAME_SIZE];
+/* Keep a 320x240 RGB565 draw-stage buffer in shared SoCMEM to reduce YUV conversion load. */
+static UVC_RGB565_STAGE_SECTION uint16_t g_uvc_yuyv_stage_rgb565[UVC_YUYV_STAGE_PIXELS];
 
 #define UVC_MJPEG_STD_DHT_SIZE 420U
 
@@ -1047,8 +1058,8 @@ static UVC_ITCM_SECTION rt_err_t uvc_display_hw_init(void)
     /* LCD driver is responsible for GFXSS/panel initialization. */
     memset(&g_uvc_gfx_context, 0, sizeof(g_uvc_gfx_context));
 
-    lcd_fb = (uint16_t *)g_uvc_lcd_fb_mem;
-    lcd_fb_size = sizeof(g_uvc_lcd_fb_mem);
+    lcd_fb = (uint16_t *)graphics_buffer;
+    lcd_fb_size = LCD_BUF_SIZE;
     memset(lcd_fb, 0, lcd_fb_size);
 
     g_uvc_lcd_hw_ready = RT_TRUE;
@@ -1061,8 +1072,8 @@ static UVC_ITCM_SECTION rt_err_t uvc_display_refresh_fb_info(void)
         return -RT_ERROR;
     }
 
-    lcd_fb = (uint16_t *)g_uvc_lcd_fb_mem;
-    lcd_fb_size = sizeof(g_uvc_lcd_fb_mem);
+    lcd_fb = (uint16_t *)graphics_buffer;
+    lcd_fb_size = LCD_BUF_SIZE;
     return RT_EOK;
 }
 
@@ -1326,6 +1337,41 @@ static UVC_ITCM_SECTION void uvc_display_yuyv(const uint8_t *src, uint32_t src_s
         return;
 
     uvc_display_prepare_yuv_lut();
+    if ((src_w == UVC_YUYV_STAGE_W) && (src_h == UVC_YUYV_STAGE_H)) {
+        uint16_t *dst = g_uvc_yuyv_stage_rgb565;
+
+        if (src_size < (uint32_t)UVC_YUYV_STAGE_W * UVC_YUYV_STAGE_H * 2U) {
+            return;
+        }
+
+        for (uint16_t y = 0; y < UVC_YUYV_STAGE_H; y++) {
+            const uint8_t *srow = src + (uint32_t)y * UVC_YUYV_STAGE_W * 2U;
+            uint16_t *drow = dst + (uint32_t)y * UVC_YUYV_STAGE_W;
+
+            for (uint16_t pair = 0; pair < (UVC_YUYV_STAGE_W / 2U); pair++) {
+                uint8_t y0 = srow[0];
+                uint8_t u = srow[1];
+                uint8_t y1 = srow[2];
+                uint8_t v = srow[3];
+                int32_t u_to_b = g_uvc_u_to_b_lut[u];
+                int32_t u_to_g = g_uvc_u_to_g_lut[u];
+                int32_t v_to_r = g_uvc_v_to_r_lut[v];
+                int32_t v_to_g = g_uvc_v_to_g_lut[v];
+
+                drow[0] = yuv_to_rgb565_fast(y0, u_to_b, u_to_g, v_to_r, v_to_g);
+                drow[1] = yuv_to_rgb565_fast(y1, u_to_b, u_to_g, v_to_r, v_to_g);
+
+                srow += 4;
+                drow += 2;
+            }
+        }
+
+        uvc_display_rgb565(g_uvc_yuyv_stage_rgb565,
+                           UVC_YUYV_STAGE_BYTES,
+                           UVC_YUYV_STAGE_W, UVC_YUYV_STAGE_H);
+        return;
+    }
+
     if (uvc_display_prepare_geometry(src_w, src_h) != RT_EOK) {
         return;
     }
@@ -1388,6 +1434,32 @@ int uvc_display_init(void)
     return 0;
 }
 
+void uvc_display_set_overlay_callback(uvc_display_overlay_cb_t cb, void *user_ctx)
+{
+    g_uvc_overlay_cb = cb;
+    g_uvc_overlay_cb_ctx = user_ctx;
+}
+
+static void uvc_display_run_overlay_callback(void)
+{
+    uvc_display_overlay_info_t info;
+
+    if ((g_uvc_overlay_cb == RT_NULL) || (lcd_fb == RT_NULL)) {
+        return;
+    }
+
+    info.framebuffer = lcd_fb;
+    info.lcd_width = LCD_W;
+    info.lcd_height = LCD_H;
+    info.src_width = g_uvc_display_ctx.src_w;
+    info.src_height = g_uvc_display_ctx.src_h;
+    info.dst_width = g_uvc_display_ctx.dst_w;
+    info.dst_height = g_uvc_display_ctx.dst_h;
+    info.dst_y_offset = g_uvc_display_ctx.y_offset;
+
+    g_uvc_overlay_cb(&info, g_uvc_overlay_cb_ctx);
+}
+
 UVC_ITCM_SECTION void uvc_display_frame(struct usbh_videoframe *frame,
                                         uint16_t src_w, uint16_t src_h)
 {
@@ -1415,6 +1487,7 @@ UVC_ITCM_SECTION void uvc_display_frame(struct usbh_videoframe *frame,
 
     /* YUYV rendering */
     uvc_display_yuyv(frame->frame_buf, frame->frame_size, src_w, src_h);
+    uvc_display_run_overlay_callback();
 
     /* Flush only the active image band to reduce LCD transfer time. */
     (void)uvc_display_flush(&g_uvc_display_ctx.update_rect);