From 31982ba7b9f833e5b28722b99fa919867d97877f Mon Sep 17 00:00:00 2001 From: Rdzleo Date: Fri, 15 May 2026 15:53:21 +0800 Subject: [PATCH] =?UTF-8?q?feat(ui):=20Phase=2010=20-=20=E6=95=B0=E5=AD=97?= =?UTF-8?q?=E4=BA=BA=E6=A8=A1=E5=BC=8F=20LVGL=20=E2=86=92=20esp=5Femote=5F?= =?UTF-8?q?gfx=20=E5=AE=8C=E6=95=B4=E5=88=87=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✅ 验证完成： - 音频卡顿明显改善（用户实测） - 数字人 hiyori 动画正常显示 - nm 验证：固件中 0 个 lv_*/lvgl_* 函数符号 - kapi.bin: 4.7MB → 2.75MB（-42%） 关键改动: - main/dzbj/ai_chat_ui_eaf.c (404 行新增)： 完全替代 LVGL 版 ai_chat_ui.c，提供同名 C API（ai_chat_screen_init / set_status / set_emotion / set_chat_message / resume_animation）。 AiChatDisplay C++ 桥接层无需改动。 内部用 gfx_emote_init + gfx_disp_add + gfx_anim + mmap_assets。 - main/CMakeLists.txt：双轨编译 CONFIG_BAJI_BADGE_MODE=y → ai_chat_ui.c (LVGL) + bg_gif_demo.c CONFIG_BAJI_BADGE_MODE=n → ai_chat_ui_eaf.c (esp_emote_gfx) - main/dzbj/dzbj_init.c：EAF 模式跳过 lvgl_lcd_init() 调用 - main/dzbj/lcd.c/h：暴露 lcd_io_handle 给 EAF 注册 IO 完成回调 踩坑修复（commit message 留档供后续参考）: 1. esp_mmap_assets v2.0.0 在 use_fs=true 模式下 mmap_assets_get_mem() 返回的是文件内偏移量而非 RAM 指针（fseek bug + offset 没加 data_section_start），导致 LoadProhibited panic。 解决：完全绕过 mmap_assets，自己 fopen + 解析 MMAP bin 头 （layout: 头 16B + 每 entry 28B + data 段每文件 2B magic + 数据）。 2. esp_emote_gfx 期望 esp_lcd_touch v2.x 新 API，项目用 v1.1.2 旧 API。 在 managed_components 内 gfx_touch.c 加 shim 桥接（local patch， reconfigure 后需 reapply）。 3. EAF format magic 是 0x89 'EAF'（gfx_eaf_dec.h），不是 0x5A5A （那是 esp_mmap_assets 内部文件分隔符）。 4. SPIFFS 需要在 ai_chat_screen_init 入口自动挂载（不能依赖 bg_gif_demo 的惰性挂载，那个已被 CONFIG 排除）。 依赖增量: - espressif2022/esp_emote_gfx: ~3.0.5 - espressif/esp_mmap_assets: * (仅用于声明依赖，运行时被绕过) 数字人模式核心 UI 范围: - 显示数字人动画 ✅ (hiyori_m06/m07, 居中循环) - 情绪 → GIF 映射 ✅ (23 情绪 → 2 EAF，sad/angry 暂用 m07，m03 待补) - 字幕/状态文字: stub ⏳（字体接驳留待后续，需打包 .bin 字体到资源） - 触摸: 不支持（PoC 阶段不需要） Co-Authored-By: Claude Opus 4.7 (1M context) --- dependencies.lock | 17 +- main/CMakeLists.txt | 18 +- main/dzbj/ai_chat_ui_eaf.c | 404 +++++++++++++++ main/dzbj/dzbj_init.c | 13 +- main/dzbj/lcd.c | 4 +- main/dzbj/lcd.h | 4 + main/idf_component.yml | 1 + .../src/core/runtime/gfx_touch.c | 463 ++++++++++++++++++ 8 files changed, 916 insertions(+), 8 deletions(-) create mode 100644 main/dzbj/ai_chat_ui_eaf.c create mode 100644 managed_components/espressif2022__esp_emote_gfx/src/core/runtime/gfx_touch.c diff --git a/dependencies.lock b/dependencies.lock index 4e9b0cb..438517d 100644 --- a/dependencies.lock +++ b/dependencies.lock @@ -169,6 +169,20 @@ dependencies: registry_url: https://components.espressif.com/ type: service version: 2.5.0 + espressif/esp_mmap_assets: + component_hash: b7c559238d9f4c11048b1d7302f5474e4f4f590902433efd792bd0cbf5324f2a + dependencies: + - name: espressif/cmake_utilities + registry_url: https://components.espressif.com + require: private + version: 0.* + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com/ + type: service + version: 2.0.0 espressif/esp_new_jpeg: component_hash: 98823384f51ca298e2c9bebacd1c629148e528ed0902d18b16556df151519e68 dependencies: [] @@ -290,11 +304,12 @@ direct_dependencies: - espressif/esp_lcd_touch - espressif/esp_lcd_touch_cst816s - espressif/esp_lvgl_port +- espressif/esp_mmap_assets - espressif/knob - espressif/led_strip - espressif2022/esp_emote_gfx - idf - lvgl/lvgl -manifest_hash: 90544e3d787e63c288feeb33cf16100755d3ed90c47270526fd2fb5754eba469 +manifest_hash: 56465d60ff0a813df7f9be998612a4c2bc61e6d560c2f56fd585445d05b25456 target: esp32s3 version: 2.0.0 diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 67e68b7..f8ca527 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -28,10 +28,10 @@ set(SOURCES "audio_codecs/audio_codec.cc" "dzbj/pages_pwm.c" "dzbj/dzbj_init.c" # 含 dzbj_hw_display_init（公共硬件初始化）；dzbj_display_init 函数体内部 #ifdef 包裹 "dzbj/fatfs.c" # DecodeImg 公共（AI 模式 BG GIF PoC 也用）；吧唧专用函数 fatfs_init/list 等无副作用 - "dzbj/ai_chat_ui.c" # AI 对话模式 LVGL 屏幕 - "dzbj/sprite_demo.c" # Sprite Sheet PoC（RGB565 raw 替代 GIF） - "dzbj/dual_gif_demo.c" # 双 GIF 循环播放 PoC - "dzbj/bg_gif_demo.c" # 背景 + 透明 GIF 叠加（方案 C） + # Phase 10: ai_chat_ui 双轨编译 + # CONFIG_BAJI_BADGE_MODE=y → ai_chat_ui.c (LVGL 版) + # CONFIG_BAJI_BADGE_MODE=n → ai_chat_ui_eaf.c (esp_emote_gfx 版) + # 这两个在下方 if(CONFIG_BAJI_BADGE_MODE) 块中条件编译 "fonts/font_puhui_20_4.c" # 阿里巴巴普惠体 20px 4bpp（GB2312 简体中文） # SquareLine Studio UI 公共文件（AI 模式也使用） "ui/ui.c" @@ -221,6 +221,11 @@ if(CONFIG_BAJI_BADGE_MODE) "dzbj/dzbj_button.c" "dzbj/dzbj_battery.c" "dzbj/ble_transfer.c" + # 吧唧模式 LVGL UI（AI 对话屏幕 + 各种 PoC） + "dzbj/ai_chat_ui.c" + "dzbj/sprite_demo.c" + "dzbj/dual_gif_demo.c" + "dzbj/bg_gif_demo.c" # SquareLine Studio 吧唧专属 UI 屏幕（9 个） "ui/screens/ui_ScreenHome.c" "ui/screens/ui_ScreenImg.c" @@ -232,6 +237,11 @@ if(CONFIG_BAJI_BADGE_MODE) "ui/screens/ui_ScreenSharing.c" "ui/screens/ui_ScreenReceiving.c" ) +else() + # Phase 10: 数字人模式 EAF UI（替代 LVGL 版 ai_chat_ui.c + bg_gif_demo.c） + list(APPEND SOURCES + "dzbj/ai_chat_ui_eaf.c" + ) endif() if(CONFIG_CONNECTION_TYPE_MQTT_UDP) diff --git a/main/dzbj/ai_chat_ui_eaf.c b/main/dzbj/ai_chat_ui_eaf.c new file mode 100644 index 0000000..b626466 --- /dev/null +++ b/main/dzbj/ai_chat_ui_eaf.c @@ -0,0 +1,404 @@ +/* + * Phase 10: 数字人 RTC 模式 EAF UI 实现 + * + * 完全替代 ai_chat_ui.c (LVGL 版)，提供相同的 C API 签名让 AiChatDisplay 桥接层无需改动。 + * + * 架构: + * esp_emote_gfx (gfx_emote_init + gfx_disp_add + gfx_anim) + * ↓ + * mmap_assets (use_fs 模式，从 /spiflash/hiyori-assets.bin 加载) + * ↓ + * panel_handle (lcd.c 暴露，已由 lcd_init 完成硬件初始化) + * + * PoC 阶段说明: + * - 只显示数字人动画（核心目的：验证显示 + 听感效果） + * - 字幕/状态文字: 仅日志输出（字体接驳留待后续，需要打包 .bin 字体到 mmap_assets） + * - 资源限制: 当前 hiyori-assets.bin 只含 m06 + m07（用户在线工具未导入 m03） + * sad/angry 等负面情绪暂时降级到 m07 + */ + +#include "ai_chat_ui.h" +#include "lcd.h" // 引用 panel_handle / lcd_io_handle + +#include "gfx.h" +#include "esp_mmap_assets.h" +#include "esp_log.h" +#include "esp_lcd_panel_io.h" +#include "esp_lcd_panel_ops.h" +#include "esp_spiffs.h" // Phase 10: SPIFFS 自动挂载 +#include "esp_heap_caps.h" + +#include +#include +#include + +// Phase 10 v2 修复: +// esp_mmap_assets v2.0.0 在 use_fs=true 模式下，mmap_assets_get_mem() 返回的是文件内偏移量 +// 而不是 RAM 指针（看 esp_mmap_assets.c line 523 + line 353 的 fseek 用法）。 +// 把 offset 当指针 dereference 会导致 LoadProhibited panic。 +// 修复：开机时用 mmap_assets_copy_by_index 把所有 EAF 数据 fread 到 PSRAM buffer， +// 运行时直接用 buffer 指针给 gfx_anim 使用。 +typedef struct { + uint8_t *data; // EAF 数据 PSRAM 指针（malloc 出来） + size_t size; // EAF 大小 + char name[40]; // 文件名 +} eaf_cache_entry_t; + +static eaf_cache_entry_t s_eaf_cache[8]; // 预留 8 个表情槽位 +static int s_eaf_cache_count = 0; + +static const char *TAG = "AI_CHAT_EAF"; + +// ========================================================== +// 配置常量 +// ========================================================== +#define EAF_ASSETS_PATH "/spiflash/hiyori-assets.bin" +#define EAF_MAX_FILES 3 // index.json + 2 个 EAF (m06 + m07) +#define EAF_DEFAULT_FPS 14 // 与工具配置一致 +#define LCD_W 360 +#define LCD_H 360 + +// ========================================================== +// 全局 EAF 上下文 +// ========================================================== +static gfx_handle_t s_emote_handle = NULL; +static gfx_disp_t *s_disp = NULL; +static gfx_obj_t *s_anim_obj = NULL; +static mmap_assets_handle_t s_assets = NULL; +static int s_current_emotion_idx = -1; +static bool s_initialized = false; + +// ========================================================== +// 情绪 → asset 名字 映射表 +// ========================================================== +typedef struct { + const char *emotion; + const char *asset_name; +} eaf_emotion_map_t; + +static const eaf_emotion_map_t s_emotion_map[] = { + // 默认/积极 → m06 + {"neutral", "hiyori_m06.eaf"}, + {"happy", "hiyori_m06.eaf"}, + {"laughing", "hiyori_m06.eaf"}, + {"funny", "hiyori_m06.eaf"}, + {"loving", "hiyori_m06.eaf"}, + {"relaxed", "hiyori_m06.eaf"}, + {"delicious", "hiyori_m06.eaf"}, + {"kissy", "hiyori_m06.eaf"}, + {"confident", "hiyori_m06.eaf"}, + {"silly", "hiyori_m06.eaf"}, + {"blink", "hiyori_m06.eaf"}, + {"curious", "hiyori_m06.eaf"}, + // 思考/疲倦 → m07 + {"sleepy", "hiyori_m07.eaf"}, + {"thinking", "hiyori_m07.eaf"}, + {"confused", "hiyori_m07.eaf"}, + {"embarrassed", "hiyori_m07.eaf"}, + {"dizzy", "hiyori_m07.eaf"}, + // 负面/严肃 → 暂用 m07（m03 未导入） + {"sad", "hiyori_m07.eaf"}, + {"crying", "hiyori_m07.eaf"}, + {"angry", "hiyori_m07.eaf"}, + {"surprised", "hiyori_m07.eaf"}, + {"shocked", "hiyori_m07.eaf"}, +}; +#define EMOTION_MAP_SIZE (sizeof(s_emotion_map) / sizeof(s_emotion_map[0])) + +// ========================================================== +// LCD flush 回调 (gfx → esp_lcd_panel_draw_bitmap) +// ========================================================== +static void eaf_disp_flush_cb(gfx_disp_t *disp, int x1, int y1, int x2, int y2, const void *data) { + esp_lcd_panel_handle_t panel = (esp_lcd_panel_handle_t)gfx_disp_get_user_data(disp); + esp_lcd_panel_draw_bitmap(panel, x1, y1, x2, y2, data); +} + +// panel IO 完成回调，通知 gfx flush 完毕 +static bool eaf_flush_io_ready(esp_lcd_panel_io_handle_t panel_io, + esp_lcd_panel_io_event_data_t *edata, + void *user_ctx) { + gfx_disp_t *disp = (gfx_disp_t *)user_ctx; + if (disp) { + gfx_disp_flush_ready(disp, true); + } + return true; +} + +// ========================================================== +// 按名字查找 cache 中的 EAF entry +// ========================================================== +static int find_cache_index_by_name(const char *name) { + if (!name) return -1; + for (int i = 0; i < s_eaf_cache_count; i++) { + if (strcmp(s_eaf_cache[i].name, name) == 0 && s_eaf_cache[i].data) { + return i; + } + } + return -1; +} + +// ========================================================== +// 切换表情到指定 asset (用 PSRAM 中 cache 的 EAF 数据) +// ========================================================== +static esp_err_t switch_emotion_by_asset(const char *asset_name) { + if (!s_initialized || !s_anim_obj) return ESP_ERR_INVALID_STATE; + + int idx = find_cache_index_by_name(asset_name); + if (idx < 0) { + ESP_LOGW(TAG, "Asset 未在 cache: %s", asset_name); + return ESP_ERR_NOT_FOUND; + } + if (idx == s_current_emotion_idx) { + return ESP_OK; // 已是当前表情 + } + + uint8_t *eaf_data = s_eaf_cache[idx].data; + size_t eaf_size = s_eaf_cache[idx].size; + + esp_err_t ret = gfx_emote_lock(s_emote_handle); + if (ret != ESP_OK) return ret; + + gfx_anim_stop(s_anim_obj); + + gfx_anim_src_t src = { + .type = GFX_ANIM_SRC_TYPE_MEMORY, + .data = eaf_data, + .data_len = eaf_size, + }; + gfx_anim_set_src_desc(s_anim_obj, &src); + + // 居中显示，hiyori 209×360 居中放 360×360 屏 + gfx_obj_align(s_anim_obj, GFX_ALIGN_CENTER, 0, 0); + + // 全部帧 + EAF_DEFAULT_FPS + 永远循环 + gfx_anim_set_segment(s_anim_obj, 0, 0xFFFFFFFF, EAF_DEFAULT_FPS, true); + gfx_anim_start(s_anim_obj); + + gfx_emote_unlock(s_emote_handle); + + s_current_emotion_idx = idx; + ESP_LOGI(TAG, "切换表情: %s (idx=%d, size=%d)", asset_name, idx, eaf_size); + return ESP_OK; +} + +// ========================================================== +// 公开 C API（与 ai_chat_ui.c 完全相同的签名） +// ========================================================== + +void ai_chat_screen_init(void) { + if (s_initialized) { + ESP_LOGW(TAG, "已初始化，跳过"); + return; + } + + ESP_LOGI(TAG, "============================"); + ESP_LOGI(TAG, "=== EAF 数字人 UI 初始化 ==="); + ESP_LOGI(TAG, "============================"); + + // 0. 确保 SPIFFS 挂载（mmap_assets_new with use_fs=true 需要 vfs 路径可访问） + size_t spiffs_total = 0, spiffs_used = 0; + esp_err_t mount_ret = esp_spiffs_info("storage", &spiffs_total, &spiffs_used); + if (mount_ret != ESP_OK) { + ESP_LOGI(TAG, "SPIFFS 未挂载，自动挂载到 /spiflash..."); + esp_vfs_spiffs_conf_t spiffs_cfg = { + .base_path = "/spiflash", + .partition_label = "storage", + .max_files = 5, + .format_if_mount_failed = false, + }; + mount_ret = esp_vfs_spiffs_register(&spiffs_cfg); + if (mount_ret != ESP_OK) { + ESP_LOGE(TAG, "SPIFFS 挂载失败: %s", esp_err_to_name(mount_ret)); + return; + } + esp_spiffs_info("storage", &spiffs_total, &spiffs_used); + } + ESP_LOGI(TAG, "SPIFFS 已就绪: total=%u KB, used=%u KB", + (unsigned)(spiffs_total / 1024), (unsigned)(spiffs_used / 1024)); + + // 1. 自己解析 hiyori-assets.bin（绕过 esp_mmap_assets v2.0.0 use_fs 模式的严重 offset bug） + // + // MMAP bin 实际 layout（hex 反推得出）： + // [0x00-0x03] "MMAP" magic + // [0x04-0x07] version + checksum (2B + 2B) + // [0x08-0x0B] header_size = 16 + // [0x0C-0x0F] file_count + // [0x10-0x1F] reserved (16B) + // [0x20+] file entry table，每 entry = 28B (16B name + 4B size + 4B offset + 4B pad) + // [data] table 后是数据段。每个文件: 2B 0x5A 0x5A magic prefix + size 字节数据。 + // entry.offset 是相对数据段起点的偏移（指向文件的 magic prefix 起点） + ESP_LOGI(TAG, "解析 hiyori-assets.bin:"); + FILE *f = fopen(EAF_ASSETS_PATH, "rb"); + if (!f) { + ESP_LOGE(TAG, "打开 %s 失败", EAF_ASSETS_PATH); + return; + } + uint8_t header[16]; + if (fread(header, 1, 16, f) != 16 || memcmp(header, "MMAP", 4) != 0) { + ESP_LOGE(TAG, "MMAP 头解析失败"); + fclose(f); + return; + } + uint32_t file_count = header[12] | (header[13] << 8) | (header[14] << 16) | (header[15] << 24); + ESP_LOGI(TAG, " MMAP file_count=%u", (unsigned)file_count); + // 跳过 reserved 16B 到 entry table 起点 (0x20) + fseek(f, 0x20, SEEK_SET); + const size_t ENTRY_SIZE = 28; // 16 + 4 + 4 + 4 + const size_t DATA_START = 0x20 + file_count * ENTRY_SIZE; + + s_eaf_cache_count = 0; + for (uint32_t i = 0; i < file_count; i++) { + uint8_t entry[28]; + fseek(f, 0x20 + i * ENTRY_SIZE, SEEK_SET); + if (fread(entry, 1, ENTRY_SIZE, f) != ENTRY_SIZE) { + ESP_LOGE(TAG, " entry[%u] 读取失败", (unsigned)i); + continue; + } + char name[17] = {0}; + memcpy(name, entry, 16); + name[16] = '\0'; + uint32_t fsize = entry[16] | (entry[17] << 8) | (entry[18] << 16) | (entry[19] << 24); + uint32_t foffset = entry[20] | (entry[21] << 8) | (entry[22] << 16) | (entry[23] << 24); + + // 只缓存 .eaf 文件 + size_t nlen = strlen(name); + if (nlen < 4 || strcmp(name + nlen - 4, ".eaf") != 0) { + ESP_LOGI(TAG, " 跳过非 EAF: %s (size=%u)", name, (unsigned)fsize); + continue; + } + + // 真实文件位置 = data_section_start + entry.offset + 2 (跳过 0x5A 0x5A magic prefix) + size_t real_offset = DATA_START + foffset + 2; + + uint8_t *buf = heap_caps_malloc(fsize, MALLOC_CAP_SPIRAM); + if (!buf) { + ESP_LOGE(TAG, " PSRAM malloc 失败: %s (size=%u)", name, (unsigned)fsize); + continue; + } + if (fseek(f, real_offset, SEEK_SET) != 0 || fread(buf, 1, fsize, f) != fsize) { + ESP_LOGE(TAG, " fread 失败: %s @ offset %zu", name, real_offset); + heap_caps_free(buf); + continue; + } + + // 验证 EAF format magic + if (buf[0] != 0x89 || buf[1] != 'E' || buf[2] != 'A' || buf[3] != 'F') { + ESP_LOGE(TAG, " EAF magic 失败: %s (got %02x %02x %02x %02x)", + name, buf[0], buf[1], buf[2], buf[3]); + heap_caps_free(buf); + continue; + } + + if (s_eaf_cache_count >= (int)(sizeof(s_eaf_cache)/sizeof(s_eaf_cache[0]))) { + ESP_LOGW(TAG, " cache 已满，丢弃: %s", name); + heap_caps_free(buf); + break; + } + s_eaf_cache[s_eaf_cache_count].data = buf; + s_eaf_cache[s_eaf_cache_count].size = fsize; + strncpy(s_eaf_cache[s_eaf_cache_count].name, name, sizeof(s_eaf_cache[0].name) - 1); + s_eaf_cache[s_eaf_cache_count].name[sizeof(s_eaf_cache[0].name) - 1] = '\0'; + ESP_LOGI(TAG, " ✓ Cached [%d] %s (%u bytes) @ %p (file_offset=%zu)", + s_eaf_cache_count, name, (unsigned)fsize, buf, real_offset); + s_eaf_cache_count++; + } + fclose(f); + + if (s_eaf_cache_count == 0) { + ESP_LOGE(TAG, "没有 EAF 资源被加载，初始化中止"); + return; + } + ESP_LOGI(TAG, "EAF 预加载完成，共 %d 个表情可用", s_eaf_cache_count); + + // 2. 初始化 gfx 核心（绑 Core 0，与原 LVGL 一致避免抢音频 Core 1） + gfx_core_config_t gfx_cfg = { + .fps = 25, + .task = GFX_EMOTE_INIT_CONFIG(), + }; + gfx_cfg.task.task_priority = 4; + gfx_cfg.task.task_affinity = 0; // Core 0 + gfx_cfg.task.task_stack = 8 * 1024; + s_emote_handle = gfx_emote_init(&gfx_cfg); + if (!s_emote_handle) { + ESP_LOGE(TAG, "gfx_emote_init 失败"); + return; + } + + // 3. 添加 display（接管 panel_handle） + gfx_disp_config_t disp_cfg = { + .h_res = LCD_W, + .v_res = LCD_H, + .flush_cb = eaf_disp_flush_cb, + .update_cb = NULL, + .user_data = (void *)panel_handle, + .flags = { + .swap = true, // RGB565 字节序（与 LVGL 配置一致） + .buff_dma = true, + .buff_spiram = false, + .double_buffer = true, + }, + .buffers = { .buf1 = NULL, .buf2 = NULL, .buf_pixels = LCD_W * 20 }, + }; + s_disp = gfx_disp_add(s_emote_handle, &disp_cfg); + if (!s_disp) { + ESP_LOGE(TAG, "gfx_disp_add 失败"); + gfx_emote_deinit(s_emote_handle); + s_emote_handle = NULL; + return; + } + + // 注册 panel IO 完成回调 + const esp_lcd_panel_io_callbacks_t cbs = { .on_color_trans_done = eaf_flush_io_ready }; + esp_lcd_panel_io_register_event_callbacks(lcd_io_handle, &cbs, s_disp); + + // 4. 设置背景色 = BG_COLOR (0x000000 黑色，与 LVGL 版一致) + gfx_disp_set_bg_color(s_disp, GFX_COLOR_HEX(0x000000)); + + // 5. 创建动画对象 + 加载默认表情 m06 + s_anim_obj = gfx_anim_create(s_disp); + if (!s_anim_obj) { + ESP_LOGE(TAG, "gfx_anim_create 失败"); + return; + } + + s_initialized = true; + + // 默认表情 = neutral → m06 + switch_emotion_by_asset("hiyori_m06.eaf"); + + ESP_LOGI(TAG, "=== EAF 数字人 UI 初始化完成 ==="); +} + +void ai_chat_set_status(const char* status) { + // PoC 阶段不显示状态文字（gfx_label 需要字体资源接驳，留待后续） + if (status) { + ESP_LOGI(TAG, "状态: %s（PoC 阶段暂不显示）", status); + } +} + +void ai_chat_set_emotion(const char* emotion) { + if (!emotion || !s_initialized) return; + + // 查映射表 + const char *asset_name = "hiyori_m06.eaf"; // 默认 fallback + for (size_t i = 0; i < EMOTION_MAP_SIZE; i++) { + if (strcmp(emotion, s_emotion_map[i].emotion) == 0) { + asset_name = s_emotion_map[i].asset_name; + break; + } + } + switch_emotion_by_asset(asset_name); +} + +void ai_chat_set_chat_message(const char* role, const char* content) { + (void)role; + // PoC 阶段不显示字幕（gfx_label 需要字体资源接驳，留待后续） + if (content && content[0]) { + ESP_LOGI(TAG, "字幕: %s（PoC 阶段暂不显示）", content); + } +} + +void ai_chat_resume_animation(void) { + // EAF 动画由 gfx_anim_start 持续播放，无需手动 resume + ESP_LOGD(TAG, "resume_animation（EAF 模式下自动循环，无需操作）"); +} diff --git a/main/dzbj/dzbj_init.c b/main/dzbj/dzbj_init.c index 40e4a04..e6e2aa6 100644 --- a/main/dzbj/dzbj_init.c +++ b/main/dzbj/dzbj_init.c @@ -13,10 +13,13 @@ #define TAG "DZBJ" // 仅硬件+LVGL 初始化（不加载 SquareLine UI，不点亮背光） +// +// Phase 10: 数字人 EAF 模式（CONFIG_BAJI_BADGE_MODE=n）下跳过 LVGL 初始化 +// 让 esp_emote_gfx 接管 panel_handle，避免双框架冲突 void dzbj_hw_display_init(i2c_master_bus_handle_t i2c_bus) { ESP_LOGI(TAG, "开始初始化显示硬件..."); - // 1. LCD 硬件初始化（QSPI ST77916） + // 1. LCD 硬件初始化（QSPI ST77916）—— 共享 lcd_init(); ESP_LOGI(TAG, "LCD 硬件初始化完成"); @@ -31,9 +34,15 @@ void dzbj_hw_display_init(i2c_master_bus_handle_t i2c_bus) { ESP_LOGI(TAG, "屏幕触摸已禁用 (DZBJ_ENABLE_TOUCH=0)"); #endif - // 4. LVGL 初始化（显示） +#ifdef CONFIG_BAJI_BADGE_MODE + // 4. LVGL 初始化（仅吧唧模式） lvgl_lcd_init(); ESP_LOGI(TAG, "LVGL 初始化完成"); +#else + // Phase 10: 数字人 EAF 模式下不初始化 LVGL + // esp_emote_gfx 会在 ai_chat_screen_init 中接管 panel_handle + ESP_LOGI(TAG, "数字人 EAF 模式: 跳过 LVGL 初始化，等待 esp_emote_gfx 接管"); +#endif } #ifdef CONFIG_BAJI_BADGE_MODE diff --git a/main/dzbj/lcd.c b/main/dzbj/lcd.c index c59fe12..3b7ad10 100644 --- a/main/dzbj/lcd.c +++ b/main/dzbj/lcd.c @@ -209,7 +209,8 @@ static const st77916_lcd_init_cmd_t lcd_init_cmds[] = { static lv_disp_t * disp_handle = NULL; esp_lcd_panel_handle_t panel_handle = NULL; // 暴露给 sprite_demo 等模块直接 DMA 写 LCD -static esp_lcd_panel_io_handle_t io_handle = NULL; +static esp_lcd_panel_io_handle_t io_handle = NULL; // 仅文件内使用 +esp_lcd_panel_io_handle_t lcd_io_handle = NULL; // Phase 10: 暴露给 EAF UI 注册 IO 完成回调（lcd_init 后赋值） #if DZBJ_ENABLE_TOUCH static esp_lcd_touch_handle_t touch_handle = NULL; static esp_lcd_panel_io_handle_t tp_io_handle = NULL; @@ -239,6 +240,7 @@ void lcd_init(){ io_config.pclk_hz = 80 * 1000 * 1000; io_config.trans_queue_depth = 64; // 默认 10 太小，sprite 分条传输需要更大队列 ESP_ERROR_CHECK(esp_lcd_new_panel_io_spi((esp_lcd_spi_bus_handle_t)SPI_LCD_HOST, &io_config, &io_handle)); + lcd_io_handle = io_handle; // Phase 10: 同步给 EAF UI 使用 const st77916_vendor_config_t vendor_config = { .init_cmds = lcd_init_cmds, .init_cmds_size = sizeof(lcd_init_cmds) / sizeof(st77916_lcd_init_cmd_t), diff --git a/main/dzbj/lcd.h b/main/dzbj/lcd.h index 4d84b2d..32d44db 100644 --- a/main/dzbj/lcd.h +++ b/main/dzbj/lcd.h @@ -6,6 +6,10 @@ #include "esp_lcd_st77916.h" #include +// 全局 LCD 句柄（lcd_init 后可用） +extern esp_lcd_panel_handle_t panel_handle; +extern esp_lcd_panel_io_handle_t lcd_io_handle; // Phase 10: 给 EAF 注册 IO 完成回调 + void lcd_init(void); void lvgl_lcd_init(void); void lcd_clear_screen_black(void); diff --git a/main/idf_component.yml b/main/idf_component.yml index ad7dd16..0764cdb 100644 --- a/main/idf_component.yml +++ b/main/idf_component.yml @@ -19,6 +19,7 @@ dependencies: esp_jpeg: "*" ## Phase 10: 数字人模式 UI 框架（替代 LVGL，仅 CONFIG_BAJI_BADGE_MODE=n 时使用） espressif2022/esp_emote_gfx: "~3.0.5" + espressif/esp_mmap_assets: "*" ## Required IDF version idf: version: ">=5.3" diff --git a/managed_components/espressif2022__esp_emote_gfx/src/core/runtime/gfx_touch.c b/managed_components/espressif2022__esp_emote_gfx/src/core/runtime/gfx_touch.c new file mode 100644 index 0000000..b394350 --- /dev/null +++ b/managed_components/espressif2022__esp_emote_gfx/src/core/runtime/gfx_touch.c @@ -0,0 +1,463 @@ +/* + * SPDX-FileCopyrightText: 2025-2026 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/********************* + * INCLUDES + *********************/ +#include +#include +#include +#include + +#include "driver/gpio.h" +#include "esp_attr.h" +#include "esp_timer.h" +#include "esp_log.h" +#define GFX_LOG_MODULE GFX_LOG_MODULE_TOUCH +#include "common/gfx_log_priv.h" + +#include "core/object/gfx_obj_priv.h" +#include "core/runtime/gfx_core_priv.h" +#include "core/runtime/gfx_touch_priv.h" + +/* ========================================================================= + * Phase 10 LOCAL PATCH: 兼容 esp_lcd_touch v1.1.2（项目使用版本，无 v2 新 API） + * + * esp_emote_gfx v3.0.5 期望 esp_lcd_touch v2.x 的 esp_lcd_touch_point_data_t + * 和 esp_lcd_touch_get_data，本项目用的是 v1.1.2 只有 esp_lcd_touch_get_coordinates。 + * 加 shim 桥接旧 API → 新 API 类型，让编译通过。 + * + * 注意：reconfigure 或 update esp_emote_gfx 后此补丁会丢失，需要 reapply。 + * 数字人 PoC 不使用触摸，运行时这段代码不会被调用。 + * ========================================================================= */ +#include "esp_lcd_touch.h" +typedef struct { + uint16_t x; + uint16_t y; + uint16_t strength; + uint16_t track_id; +} esp_lcd_touch_point_data_t; +static inline esp_err_t esp_lcd_touch_get_data(esp_lcd_touch_handle_t tp, + esp_lcd_touch_point_data_t *points, + uint8_t *count, uint8_t max) { + uint16_t x[4] = {0}, y[4] = {0}, s[4] = {0}; + uint8_t n = 0; + uint8_t mp = max < 4 ? max : 4; + bool ok = esp_lcd_touch_get_coordinates(tp, x, y, s, &n, mp); + if (!ok) { + *count = 0; + return ESP_OK; // 旧 API 无触摸时返回 false，shim 转为成功+0 点 + } + for (uint8_t i = 0; i < n && i < max; i++) { + points[i].x = x[i]; + points[i].y = y[i]; + points[i].strength = s[i]; + points[i].track_id = 0; // v1.1.2 无 track_id + } + *count = n; + return ESP_OK; +} +/* ====================== END Phase 10 LOCAL PATCH ====================== */ + +/********************* + * DEFINES + *********************/ + +/********************** + * STATIC VARIABLES + **********************/ + +static const char *TAG = "touch"; +static const uint32_t DEFAULT_POLL_MS = 15; +static const uint32_t DEFAULT_IRQ_POLL_MS = 5; + +/********************** + * TYPEDEFS + **********************/ + +typedef struct { + gfx_touch_t *touch; + void *original_user_data; + volatile bool unregistering; +} gfx_touch_isr_ctx_t; + +/********************** + * STATIC PROTOTYPES + **********************/ + +static void gfx_touch_poll_cb(void *user_data); + +/********************** + * STATIC FUNCTIONS + **********************/ + +/** Return topmost visible object on disp that contains (x, y), or NULL (same order as render = last in list = front) */ +static gfx_obj_t *gfx_touch_hit_test(gfx_disp_t *disp, uint16_t x, uint16_t y) +{ + gfx_obj_t *hit = NULL; + for (gfx_obj_child_t *n = disp->child_list; n != NULL; n = n->next) { + gfx_obj_t *obj = (gfx_obj_t *)n->src; + if (!obj->state.is_visible) { + continue; + } + if (obj->align.enabled || obj->state.layout_dirty) { + gfx_obj_calc_pos_in_parent(obj); + } + int32_t ox = obj->geometry.x; + int32_t oy = obj->geometry.y; + uint32_t w = obj->geometry.width; + uint32_t h = obj->geometry.height; + if (w == 0 || h == 0) { + continue; + } + if ((int32_t)x >= ox && (int32_t)x < ox + (int32_t)w && (int32_t)y >= oy && (int32_t)y < oy + (int32_t)h) { + hit = obj; + } + } + return hit; +} + +static uint32_t gfx_touch_now_ms(void) +{ + return (uint32_t)(esp_timer_get_time() / 1000); +} + +static void gfx_touch_dispatch(gfx_touch_t *touch, gfx_touch_event_type_t type, const esp_lcd_touch_point_data_t *pt) +{ + void *hit_obj = NULL; + + gfx_touch_event_t evt = { + .type = type, + .x = touch->last_x, + .y = touch->last_y, + .strength = touch->last_strength, + .track_id = touch->last_id, + .timestamp_ms = gfx_touch_now_ms(), + }; + + if (pt) { + evt.x = pt->x; + evt.y = pt->y; + evt.strength = pt->strength; + evt.track_id = pt->track_id; + } + + if (touch->disp) { + if (type == GFX_TOUCH_EVENT_PRESS) { + hit_obj = gfx_touch_hit_test(touch->disp, evt.x, evt.y); + if (hit_obj != NULL) { + touch->pressed_obj = (gfx_obj_t *)hit_obj; + touch->pressed_id = evt.track_id; + } else { + touch->pressed_obj = NULL; + } + } else { + /* MOVE / RELEASE: keep delivering to the object that got PRESS (drag support) */ + if (touch->pressed_obj != NULL && evt.track_id == touch->pressed_id) { + hit_obj = (gfx_obj_t *)touch->pressed_obj; + } else { + hit_obj = NULL; + } + if (type == GFX_TOUCH_EVENT_RELEASE) { + touch->pressed_obj = NULL; + } + } + if (hit_obj != NULL) { + gfx_obj_t *obj = (gfx_obj_t *)hit_obj; + if (obj->vfunc.touch_event) { + obj->vfunc.touch_event(obj, &evt); + } + if (obj->user_touch_cb) { + obj->user_touch_cb(obj, &evt, obj->user_touch_data); + } + } + } + + if (touch->event_cb) { + touch->event_cb((gfx_touch_t *)touch, &evt, touch->user_data); + } +} + +static void IRAM_ATTR gfx_touch_isr(esp_lcd_touch_handle_t tp) +{ + + if (!tp || !tp->config.user_data) { + return; + } + + gfx_touch_isr_ctx_t *isr_ctx = (gfx_touch_isr_ctx_t *)tp->config.user_data; + if (!isr_ctx || isr_ctx->unregistering || !isr_ctx->touch) { + return; + } + + isr_ctx->touch->irq_pending = true; +} + +static esp_err_t gfx_touch_enable_interrupt(gfx_touch_t *touch) +{ + if (!touch || !touch->handle || touch->int_gpio_num == GPIO_NUM_NC) { + return ESP_ERR_INVALID_ARG; + } + + gfx_touch_isr_ctx_t *isr_ctx = calloc(1, sizeof(gfx_touch_isr_ctx_t)); + if (!isr_ctx) { + return ESP_ERR_NO_MEM; + } + + isr_ctx->touch = touch; + isr_ctx->original_user_data = touch->handle->config.user_data; + touch->isr_ctx = isr_ctx; + + esp_err_t ret = esp_lcd_touch_register_interrupt_callback_with_data(touch->handle, gfx_touch_isr, isr_ctx); + if (ret != ESP_OK) { + touch->isr_ctx = NULL; + free(isr_ctx); + return ret; + } + + touch->irq_enabled = true; + touch->irq_pending = false; + GFX_LOGI(TAG, "init touch: interrupt enabled on gpio %d", touch->int_gpio_num); + return ESP_OK; +} + +static void gfx_touch_disable_interrupt(gfx_touch_t *touch) +{ + if (!touch) { + return; + } + + if (touch->irq_enabled && touch->int_gpio_num != GPIO_NUM_NC && GPIO_IS_VALID_GPIO(touch->int_gpio_num)) { + esp_err_t gpio_ret = gpio_intr_disable(touch->int_gpio_num); + if (gpio_ret != ESP_OK) { + GFX_LOGW(TAG, "delete touch: disable gpio interrupt failed on pin %d (%d)", touch->int_gpio_num, gpio_ret); + } + } + + if (touch->isr_ctx) { + gfx_touch_isr_ctx_t *isr_ctx = (gfx_touch_isr_ctx_t *)touch->isr_ctx; + isr_ctx->unregistering = true; + esp_lcd_touch_register_interrupt_callback(touch->handle, NULL); + if (touch->handle && touch->handle->config.user_data != isr_ctx->original_user_data) { + touch->handle->config.user_data = isr_ctx->original_user_data; + } + free(isr_ctx); + touch->isr_ctx = NULL; + } + + touch->irq_enabled = false; + touch->irq_pending = false; +} + +static void gfx_touch_poll_cb(void *user_data) +{ + gfx_touch_t *touch = (gfx_touch_t *)user_data; + if (!touch || !touch->handle) { + return; + } + + if (touch->irq_enabled) { + if (!touch->irq_pending) { + return; + } + touch->irq_pending = false; + } + + esp_err_t ret = esp_lcd_touch_read_data(touch->handle); + if (ret != ESP_OK) { + GFX_LOGW(TAG, "poll touch: read failed (%d)", ret); + return; + } + + esp_lcd_touch_point_data_t points[1] = {0}; + uint8_t count = 0; + + ret = esp_lcd_touch_get_data(touch->handle, points, &count, 1); + if (ret != ESP_OK) { + GFX_LOGW(TAG, "poll touch: get data failed (%d)", ret); + return; + } + + bool pressed_now = (count > 0); + + if (pressed_now) { + uint16_t new_x = points[0].x; + uint16_t new_y = points[0].y; + + if (pressed_now && !touch->pressed) { + gfx_touch_dispatch(touch, GFX_TOUCH_EVENT_PRESS, &points[0]); + } else if (touch->pressed && (new_x != touch->last_x || new_y != touch->last_y)) { + gfx_touch_dispatch(touch, GFX_TOUCH_EVENT_MOVE, &points[0]); + } + + touch->last_x = new_x; + touch->last_y = new_y; + touch->last_strength = points[0].strength; + touch->last_id = points[0].track_id; + } else { + if (touch->pressed) { + gfx_touch_dispatch(touch, GFX_TOUCH_EVENT_RELEASE, NULL); + } + } + + touch->pressed = pressed_now; +} + +/********************** + * PUBLIC FUNCTIONS + **********************/ + +esp_err_t gfx_touch_start(gfx_touch_t *touch, const gfx_touch_config_t *cfg) +{ + if (!touch || !touch->ctx || !cfg) { + return ESP_ERR_INVALID_ARG; + } + + if (!cfg->handle) { + return ESP_OK; + } + + touch->handle = cfg->handle; + touch->disp = cfg->disp; + touch->event_cb = cfg->event_cb; + touch->user_data = cfg->user_data; + touch->int_gpio_num = GPIO_NUM_NC; + touch->irq_enabled = false; + touch->irq_pending = false; + touch->isr_ctx = NULL; + + bool irq_requested = false; + gpio_num_t selected_gpio = GPIO_NUM_NC; + + if (touch->handle->config.int_gpio_num != GPIO_NUM_NC && + GPIO_IS_VALID_GPIO(touch->handle->config.int_gpio_num)) { + selected_gpio = touch->handle->config.int_gpio_num; + } + + if (selected_gpio != GPIO_NUM_NC) { + touch->int_gpio_num = selected_gpio; + irq_requested = true; + } else { + touch->int_gpio_num = GPIO_NUM_NC; + } + + uint32_t default_poll = irq_requested ? DEFAULT_IRQ_POLL_MS : DEFAULT_POLL_MS; + touch->poll_ms = cfg->poll_ms ? cfg->poll_ms : default_poll; + touch->pressed = false; + touch->last_x = 0; + touch->last_y = 0; + touch->last_strength = 0; + touch->last_id = 0; + touch->pressed_obj = NULL; + + if (irq_requested) { + esp_err_t irq_ret = gfx_touch_enable_interrupt(touch); + if (irq_ret != ESP_OK) { + GFX_LOGW(TAG, "init touch: enable gpio interrupt failed on %d (%d), using polling mode", touch->int_gpio_num, irq_ret); + touch->int_gpio_num = GPIO_NUM_NC; + touch->irq_enabled = false; + touch->irq_pending = false; + if (!cfg->poll_ms) { + touch->poll_ms = DEFAULT_POLL_MS; + } + } + } + + touch->poll_timer = gfx_timer_create(touch->ctx, gfx_touch_poll_cb, touch->poll_ms, touch); + if (!touch->poll_timer) { + GFX_LOGE(TAG, "init touch: create polling timer failed"); + if (touch->irq_enabled || touch->isr_ctx) { + gfx_touch_disable_interrupt(touch); + } + return ESP_ERR_NO_MEM; + } + + GFX_LOGD(TAG, "init touch: polling started (%"PRIu32" ms)", touch->poll_ms); + return ESP_OK; +} + +void gfx_touch_del(gfx_touch_t *touch) +{ + if (!touch) { + return; + } + + gfx_core_context_t *ctx = (gfx_core_context_t *)touch->ctx; + if (ctx != NULL) { + if (ctx->touch == touch) { + ctx->touch = touch->next; + } else { + gfx_touch_t *prev = ctx->touch; + while (prev != NULL && prev->next != touch) { + prev = prev->next; + } + if (prev != NULL) { + prev->next = touch->next; + } + } + } + + if (touch->irq_enabled || touch->isr_ctx) { + gfx_touch_disable_interrupt(touch); + } + + if (touch->poll_timer && touch->ctx) { + gfx_timer_delete(touch->ctx, touch->poll_timer); + touch->poll_timer = NULL; + } + + touch->ctx = NULL; + touch->next = NULL; + touch->handle = NULL; + touch->event_cb = NULL; + touch->user_data = NULL; + touch->pressed = false; + touch->pressed_obj = NULL; + touch->int_gpio_num = GPIO_NUM_NC; +} + +gfx_touch_t *gfx_touch_add(gfx_handle_t handle, const gfx_touch_config_t *cfg) +{ + if (!handle || !cfg || !cfg->handle) { + return NULL; + } + + gfx_core_context_t *ctx = (gfx_core_context_t *)handle; + + gfx_touch_t *new_touch = (gfx_touch_t *)calloc(1, sizeof(gfx_touch_t)); + if (!new_touch) { + return NULL; + } + memset(new_touch, 0, sizeof(gfx_touch_t)); + new_touch->ctx = ctx; + + if (gfx_touch_start(new_touch, cfg) != ESP_OK) { + free(new_touch); + return NULL; + } + + if (ctx->touch == NULL) { + ctx->touch = new_touch; + } else { + gfx_touch_t *tail = ctx->touch; + while (tail->next != NULL) { + tail = tail->next; + } + tail->next = new_touch; + } + + return new_touch; +} + +esp_err_t gfx_touch_set_disp(gfx_touch_t *touch, gfx_disp_t *disp) +{ + if (!touch || !disp) { + return ESP_ERR_INVALID_ARG; + } + + touch->disp = disp; + return ESP_OK; +}