/* * Phase 10: 数字人 RTC 模式 EAF UI 实现 * * 完全替代 ai_chat_ui.c (LVGL 版),提供相同的 C API 签名让 AiChatDisplay 桥接层无需改动。 * * 架构: * esp_emote_gfx (gfx_emote_init + gfx_disp_add + gfx_anim) * ↓ * mmap_assets (use_fs 模式,从 /spiflash/hiyori-assets.bin 加载) * ↓ * panel_handle (lcd.c 暴露,已由 lcd_init 完成硬件初始化) * * PoC 阶段说明: * - 只显示数字人动画(核心目的:验证显示 + 听感效果) * - 字幕/状态文字: 仅日志输出(字体接驳留待后续,需要打包 .bin 字体到 mmap_assets) * - 资源限制: 当前 hiyori-assets.bin 只含 m06 + m07(用户在线工具未导入 m03) * sad/angry 等负面情绪暂时降级到 m07 */ #include "ai_chat_ui.h" #include "lcd.h" // 引用 panel_handle / lcd_io_handle #include "gfx.h" #include "esp_mmap_assets.h" #include "esp_log.h" #include "esp_lcd_panel_io.h" #include "esp_lcd_panel_ops.h" #include "esp_spiffs.h" // Phase 10: SPIFFS 自动挂载 #include "esp_heap_caps.h" #include #include #include // Phase 10 v2 修复: // esp_mmap_assets v2.0.0 在 use_fs=true 模式下,mmap_assets_get_mem() 返回的是文件内偏移量 // 而不是 RAM 指针(看 esp_mmap_assets.c line 523 + line 353 的 fseek 用法)。 // 把 offset 当指针 dereference 会导致 LoadProhibited panic。 // 修复:开机时用 mmap_assets_copy_by_index 把所有 EAF 数据 fread 到 PSRAM buffer, // 运行时直接用 buffer 指针给 gfx_anim 使用。 typedef struct { uint8_t *data; // EAF 数据 PSRAM 指针(malloc 出来) size_t size; // EAF 大小 char name[40]; // 文件名 } eaf_cache_entry_t; static eaf_cache_entry_t s_eaf_cache[8]; // 预留 8 个表情槽位 static int s_eaf_cache_count = 0; static const char *TAG = "AI_CHAT_EAF"; // ========================================================== // 配置常量 // ========================================================== #define EAF_ASSETS_PATH "/spiflash/hiyori-assets.bin" #define EAF_MAX_FILES 3 // index.json + 2 个 EAF (m06 + m07) #define EAF_DEFAULT_FPS 14 // 与工具配置一致 #define LCD_W 360 #define LCD_H 360 // ========================================================== // 全局 EAF 上下文 // ========================================================== static gfx_handle_t s_emote_handle = NULL; static gfx_disp_t *s_disp = NULL; static gfx_obj_t *s_anim_obj = NULL; static mmap_assets_handle_t s_assets = NULL; static int s_current_emotion_idx = -1; static bool s_initialized = false; // ========================================================== // 情绪 → asset 名字 映射表 // ========================================================== typedef struct { const char *emotion; const char *asset_name; } eaf_emotion_map_t; static const eaf_emotion_map_t s_emotion_map[] = { // 默认/积极 → m06 {"neutral", "hiyori_m06.eaf"}, {"happy", "hiyori_m06.eaf"}, {"laughing", "hiyori_m06.eaf"}, {"funny", "hiyori_m06.eaf"}, {"loving", "hiyori_m06.eaf"}, {"relaxed", "hiyori_m06.eaf"}, {"delicious", "hiyori_m06.eaf"}, {"kissy", "hiyori_m06.eaf"}, {"confident", "hiyori_m06.eaf"}, {"silly", "hiyori_m06.eaf"}, {"blink", "hiyori_m06.eaf"}, {"curious", "hiyori_m06.eaf"}, // 思考/疲倦 → m07 {"sleepy", "hiyori_m07.eaf"}, {"thinking", "hiyori_m07.eaf"}, {"confused", "hiyori_m07.eaf"}, {"embarrassed", "hiyori_m07.eaf"}, {"dizzy", "hiyori_m07.eaf"}, // 负面/严肃 → 暂用 m07(m03 未导入) {"sad", "hiyori_m07.eaf"}, {"crying", "hiyori_m07.eaf"}, {"angry", "hiyori_m07.eaf"}, {"surprised", "hiyori_m07.eaf"}, {"shocked", "hiyori_m07.eaf"}, }; #define EMOTION_MAP_SIZE (sizeof(s_emotion_map) / sizeof(s_emotion_map[0])) // ========================================================== // LCD flush 回调 (gfx → esp_lcd_panel_draw_bitmap) // ========================================================== static void eaf_disp_flush_cb(gfx_disp_t *disp, int x1, int y1, int x2, int y2, const void *data) { esp_lcd_panel_handle_t panel = (esp_lcd_panel_handle_t)gfx_disp_get_user_data(disp); esp_lcd_panel_draw_bitmap(panel, x1, y1, x2, y2, data); } // panel IO 完成回调,通知 gfx flush 完毕 static bool eaf_flush_io_ready(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_io_event_data_t *edata, void *user_ctx) { gfx_disp_t *disp = (gfx_disp_t *)user_ctx; if (disp) { gfx_disp_flush_ready(disp, true); } return true; } // ========================================================== // 按名字查找 cache 中的 EAF entry // ========================================================== static int find_cache_index_by_name(const char *name) { if (!name) return -1; for (int i = 0; i < s_eaf_cache_count; i++) { if (strcmp(s_eaf_cache[i].name, name) == 0 && s_eaf_cache[i].data) { return i; } } return -1; } // ========================================================== // 切换表情到指定 asset (用 PSRAM 中 cache 的 EAF 数据) // ========================================================== static esp_err_t switch_emotion_by_asset(const char *asset_name) { if (!s_initialized || !s_anim_obj) return ESP_ERR_INVALID_STATE; int idx = find_cache_index_by_name(asset_name); if (idx < 0) { ESP_LOGW(TAG, "Asset 未在 cache: %s", asset_name); return ESP_ERR_NOT_FOUND; } if (idx == s_current_emotion_idx) { return ESP_OK; // 已是当前表情 } uint8_t *eaf_data = s_eaf_cache[idx].data; size_t eaf_size = s_eaf_cache[idx].size; esp_err_t ret = gfx_emote_lock(s_emote_handle); if (ret != ESP_OK) return ret; gfx_anim_stop(s_anim_obj); gfx_anim_src_t src = { .type = GFX_ANIM_SRC_TYPE_MEMORY, .data = eaf_data, .data_len = eaf_size, }; gfx_anim_set_src_desc(s_anim_obj, &src); // 居中显示,hiyori 209×360 居中放 360×360 屏 gfx_obj_align(s_anim_obj, GFX_ALIGN_CENTER, 0, 0); // 全部帧 + EAF_DEFAULT_FPS + 永远循环 gfx_anim_set_segment(s_anim_obj, 0, 0xFFFFFFFF, EAF_DEFAULT_FPS, true); gfx_anim_start(s_anim_obj); gfx_emote_unlock(s_emote_handle); s_current_emotion_idx = idx; ESP_LOGI(TAG, "切换表情: %s (idx=%d, size=%d)", asset_name, idx, eaf_size); return ESP_OK; } // ========================================================== // 公开 C API(与 ai_chat_ui.c 完全相同的签名) // ========================================================== void ai_chat_screen_init(void) { if (s_initialized) { ESP_LOGW(TAG, "已初始化,跳过"); return; } ESP_LOGI(TAG, "============================"); ESP_LOGI(TAG, "=== EAF 数字人 UI 初始化 ==="); ESP_LOGI(TAG, "============================"); // 0. 确保 SPIFFS 挂载(mmap_assets_new with use_fs=true 需要 vfs 路径可访问) size_t spiffs_total = 0, spiffs_used = 0; esp_err_t mount_ret = esp_spiffs_info("storage", &spiffs_total, &spiffs_used); if (mount_ret != ESP_OK) { ESP_LOGI(TAG, "SPIFFS 未挂载,自动挂载到 /spiflash..."); esp_vfs_spiffs_conf_t spiffs_cfg = { .base_path = "/spiflash", .partition_label = "storage", .max_files = 5, .format_if_mount_failed = false, }; mount_ret = esp_vfs_spiffs_register(&spiffs_cfg); if (mount_ret != ESP_OK) { ESP_LOGE(TAG, "SPIFFS 挂载失败: %s", esp_err_to_name(mount_ret)); return; } esp_spiffs_info("storage", &spiffs_total, &spiffs_used); } ESP_LOGI(TAG, "SPIFFS 已就绪: total=%u KB, used=%u KB", (unsigned)(spiffs_total / 1024), (unsigned)(spiffs_used / 1024)); // 1. 自己解析 hiyori-assets.bin(绕过 esp_mmap_assets v2.0.0 use_fs 模式的严重 offset bug) // // MMAP bin 实际 layout(hex 反推得出): // [0x00-0x03] "MMAP" magic // [0x04-0x07] version + checksum (2B + 2B) // [0x08-0x0B] header_size = 16 // [0x0C-0x0F] file_count // [0x10-0x1F] reserved (16B) // [0x20+] file entry table,每 entry = 28B (16B name + 4B size + 4B offset + 4B pad) // [data] table 后是数据段。每个文件: 2B 0x5A 0x5A magic prefix + size 字节数据。 // entry.offset 是相对数据段起点的偏移(指向文件的 magic prefix 起点) ESP_LOGI(TAG, "解析 hiyori-assets.bin:"); FILE *f = fopen(EAF_ASSETS_PATH, "rb"); if (!f) { ESP_LOGE(TAG, "打开 %s 失败", EAF_ASSETS_PATH); return; } uint8_t header[16]; if (fread(header, 1, 16, f) != 16 || memcmp(header, "MMAP", 4) != 0) { ESP_LOGE(TAG, "MMAP 头解析失败"); fclose(f); return; } uint32_t file_count = header[12] | (header[13] << 8) | (header[14] << 16) | (header[15] << 24); ESP_LOGI(TAG, " MMAP file_count=%u", (unsigned)file_count); // 跳过 reserved 16B 到 entry table 起点 (0x20) fseek(f, 0x20, SEEK_SET); const size_t ENTRY_SIZE = 28; // 16 + 4 + 4 + 4 const size_t DATA_START = 0x20 + file_count * ENTRY_SIZE; s_eaf_cache_count = 0; for (uint32_t i = 0; i < file_count; i++) { uint8_t entry[28]; fseek(f, 0x20 + i * ENTRY_SIZE, SEEK_SET); if (fread(entry, 1, ENTRY_SIZE, f) != ENTRY_SIZE) { ESP_LOGE(TAG, " entry[%u] 读取失败", (unsigned)i); continue; } char name[17] = {0}; memcpy(name, entry, 16); name[16] = '\0'; uint32_t fsize = entry[16] | (entry[17] << 8) | (entry[18] << 16) | (entry[19] << 24); uint32_t foffset = entry[20] | (entry[21] << 8) | (entry[22] << 16) | (entry[23] << 24); // 只缓存 .eaf 文件 size_t nlen = strlen(name); if (nlen < 4 || strcmp(name + nlen - 4, ".eaf") != 0) { ESP_LOGI(TAG, " 跳过非 EAF: %s (size=%u)", name, (unsigned)fsize); continue; } // 真实文件位置 = data_section_start + entry.offset + 2 (跳过 0x5A 0x5A magic prefix) size_t real_offset = DATA_START + foffset + 2; uint8_t *buf = heap_caps_malloc(fsize, MALLOC_CAP_SPIRAM); if (!buf) { ESP_LOGE(TAG, " PSRAM malloc 失败: %s (size=%u)", name, (unsigned)fsize); continue; } if (fseek(f, real_offset, SEEK_SET) != 0 || fread(buf, 1, fsize, f) != fsize) { ESP_LOGE(TAG, " fread 失败: %s @ offset %zu", name, real_offset); heap_caps_free(buf); continue; } // 验证 EAF format magic if (buf[0] != 0x89 || buf[1] != 'E' || buf[2] != 'A' || buf[3] != 'F') { ESP_LOGE(TAG, " EAF magic 失败: %s (got %02x %02x %02x %02x)", name, buf[0], buf[1], buf[2], buf[3]); heap_caps_free(buf); continue; } if (s_eaf_cache_count >= (int)(sizeof(s_eaf_cache)/sizeof(s_eaf_cache[0]))) { ESP_LOGW(TAG, " cache 已满,丢弃: %s", name); heap_caps_free(buf); break; } s_eaf_cache[s_eaf_cache_count].data = buf; s_eaf_cache[s_eaf_cache_count].size = fsize; strncpy(s_eaf_cache[s_eaf_cache_count].name, name, sizeof(s_eaf_cache[0].name) - 1); s_eaf_cache[s_eaf_cache_count].name[sizeof(s_eaf_cache[0].name) - 1] = '\0'; ESP_LOGI(TAG, " ✓ Cached [%d] %s (%u bytes) @ %p (file_offset=%zu)", s_eaf_cache_count, name, (unsigned)fsize, buf, real_offset); s_eaf_cache_count++; } fclose(f); if (s_eaf_cache_count == 0) { ESP_LOGE(TAG, "没有 EAF 资源被加载,初始化中止"); return; } ESP_LOGI(TAG, "EAF 预加载完成,共 %d 个表情可用", s_eaf_cache_count); // 2. 初始化 gfx 核心(绑 Core 0,与原 LVGL 一致避免抢音频 Core 1) gfx_core_config_t gfx_cfg = { .fps = 25, .task = GFX_EMOTE_INIT_CONFIG(), }; gfx_cfg.task.task_priority = 4; gfx_cfg.task.task_affinity = 0; // Core 0 gfx_cfg.task.task_stack = 8 * 1024; s_emote_handle = gfx_emote_init(&gfx_cfg); if (!s_emote_handle) { ESP_LOGE(TAG, "gfx_emote_init 失败"); return; } // 3. 添加 display(接管 panel_handle) gfx_disp_config_t disp_cfg = { .h_res = LCD_W, .v_res = LCD_H, .flush_cb = eaf_disp_flush_cb, .update_cb = NULL, .user_data = (void *)panel_handle, .flags = { .swap = true, // RGB565 字节序(与 LVGL 配置一致) .buff_dma = true, .buff_spiram = false, .double_buffer = true, }, .buffers = { .buf1 = NULL, .buf2 = NULL, .buf_pixels = LCD_W * 20 }, }; s_disp = gfx_disp_add(s_emote_handle, &disp_cfg); if (!s_disp) { ESP_LOGE(TAG, "gfx_disp_add 失败"); gfx_emote_deinit(s_emote_handle); s_emote_handle = NULL; return; } // 注册 panel IO 完成回调 const esp_lcd_panel_io_callbacks_t cbs = { .on_color_trans_done = eaf_flush_io_ready }; esp_lcd_panel_io_register_event_callbacks(lcd_io_handle, &cbs, s_disp); // 4. 设置背景色 = BG_COLOR (0x000000 黑色,与 LVGL 版一致) gfx_disp_set_bg_color(s_disp, GFX_COLOR_HEX(0x000000)); // 5. 创建动画对象 + 加载默认表情 m06 s_anim_obj = gfx_anim_create(s_disp); if (!s_anim_obj) { ESP_LOGE(TAG, "gfx_anim_create 失败"); return; } s_initialized = true; // 默认表情 = neutral → m06 switch_emotion_by_asset("hiyori_m06.eaf"); ESP_LOGI(TAG, "=== EAF 数字人 UI 初始化完成 ==="); } void ai_chat_set_status(const char* status) { // PoC 阶段不显示状态文字(gfx_label 需要字体资源接驳,留待后续) if (status) { ESP_LOGI(TAG, "状态: %s(PoC 阶段暂不显示)", status); } } void ai_chat_set_emotion(const char* emotion) { if (!emotion || !s_initialized) return; // 查映射表 const char *asset_name = "hiyori_m06.eaf"; // 默认 fallback for (size_t i = 0; i < EMOTION_MAP_SIZE; i++) { if (strcmp(emotion, s_emotion_map[i].emotion) == 0) { asset_name = s_emotion_map[i].asset_name; break; } } switch_emotion_by_asset(asset_name); } void ai_chat_set_chat_message(const char* role, const char* content) { (void)role; // PoC 阶段不显示字幕(gfx_label 需要字体资源接驳,留待后续) if (content && content[0]) { ESP_LOGI(TAG, "字幕: %s(PoC 阶段暂不显示)", content); } } void ai_chat_resume_animation(void) { // EAF 动画由 gfx_anim_start 持续播放,无需手动 resume ESP_LOGD(TAG, "resume_animation(EAF 模式下自动循环,无需操作)"); }