Rdzleo/Baji_Rtc_Toy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 火山RTC字幕文本实时显示 + 字幕日志精简

Browse Source 
1. DataCallback 新增 subv（字幕）二进制消息前缀识别，字幕数据正确解析并转发到应用层
2. OnIncomingJson 处理 type:"subtitle"，通过 userId 区分用户语音识别(STT)和 AI 回复文本
3. 字幕日志简化：subv 消息仅打印"接收下行二进制消息(字幕)"，不展开 JSON 内容
4. 新增postman请求的参数

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Rdzleo 2026-03-02 15:11:34 +08:00
parent 0a6b35d979
commit 98e40ac163
3 changed files with 38 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
main/application.cc
View File

@ -1208,6 +1208,34 @@ void Application::Start() {
}
});
}
} else if (strcmp(type->valuestring, "subtitle") == 0) {
// 火山 RTC 字幕消息：data 数组中包含 text、userId、definite
auto data_arr = cJSON_GetObjectItem(root, "data");
if (data_arr && cJSON_IsArray(data_arr)) {
for (int i = 0; i < cJSON_GetArraySize(data_arr); ++i) {
auto item = cJSON_GetArrayItem(data_arr, i);
auto text = cJSON_GetObjectItem(item, "text");
auto user_id = cJSON_GetObjectItem(item, "userId");
auto definite = cJSON_GetObjectItem(item, "definite");
if (!text || !cJSON_IsString(text) || !text->valuestring[0]) continue;
bool is_final = definite && cJSON_IsTrue(definite);
// 通过 userId 判断是用户说的还是 AI 回复
// device_name 对应本设备 UID，匹配则为用户语音识别
bool is_user = false;
if (user_id && cJSON_IsString(user_id)) {
const char* dev_name = CONFIG_VOLC_DEVICE_NAME;
is_user = (strcmp(user_id->valuestring, dev_name) == 0);
}
std::string msg = text->valuestring;
const char* role = is_user ? "user" : "assistant";
ESP_LOGI(TAG, "%s %s: %s", is_final ? ">>" : "..", role, msg.c_str());
Schedule([this, display, msg, role_str = std::string(role)]() {
display->SetChatMessage(role_str.c_str(), msg.c_str());
});
}
}
} else if (strcmp(type->valuestring, "iot") == 0) {
auto commands = cJSON_GetObjectItem(root, "commands");
if (commands != NULL) {

10
main/protocols/volc_rtc_protocol.cc
View File

@ -591,12 +591,18 @@ void VolcRtcProtocol::DataCallback(void* context, const void* data, size_t len,
bool is_ctrl = (memcmp(buf, "ctrl", 4) == 0);
bool is_conv = (memcmp(buf, "conv", 4) == 0);
bool is_tool = (memcmp(buf, "tool", 4) == 0);
if (is_ctrl || is_conv || is_tool) {
bool is_subv = (memcmp(buf, "subv", 4) == 0);
if (is_ctrl || is_conv || is_tool || is_subv) {
uint32_t json_len = (uint32_t)((buf[4] << 24) | (buf[5] << 16) | (buf[6] << 8) | (buf[7]));
if (json_len > 0 && (size_t)(8 + json_len) <= len) {
json_text.assign(reinterpret_cast<const char*>(buf + 8), json_len);
if (!protocol->suppress_incoming_message_log_) {
ESP_LOGI(TAG, "接收下行二进制消息(%s): %.*s", is_ctrl ? "ctrl" : (is_conv ? "conv" : "tool"), (int)json_text.size(), json_text.c_str());
if (is_subv) {
ESP_LOGI(TAG, "接收下行二进制消息(字幕)");
} else {
const char* prefix = is_ctrl ? "ctrl" : (is_conv ? "conv" : "tool");
ESP_LOGI(TAG, "接收下行二进制消息(%s): %.*s", prefix, (int)json_text.size(), json_text.c_str());
}
}
}
}

21
sdkconfig
View File

@ -14,7 +14,6 @@ CONFIG_SOC_GDMA_SUPPORTED=y
CONFIG_SOC_AHB_GDMA_SUPPORTED=y
CONFIG_SOC_GPTIMER_SUPPORTED=y
CONFIG_SOC_LCDCAM_SUPPORTED=y
CONFIG_SOC_LCDCAM_CAM_SUPPORTED=y
CONFIG_SOC_LCDCAM_I80_LCD_SUPPORTED=y
CONFIG_SOC_LCDCAM_RGB_LCD_SUPPORTED=y
CONFIG_SOC_MCPWM_SUPPORTED=y
@ -102,7 +101,7 @@ CONFIG_SOC_CPU_HAS_FPU=y
CONFIG_SOC_HP_CPU_HAS_MULTIPLE_CORES=y
CONFIG_SOC_CPU_BREAKPOINTS_NUM=2
CONFIG_SOC_CPU_WATCHPOINTS_NUM=2
CONFIG_SOC_CPU_WATCHPOINT_MAX_REGION_SIZE=0x40
CONFIG_SOC_CPU_WATCHPOINT_MAX_REGION_SIZE=64
CONFIG_SOC_SIMD_PREFERRED_DATA_ALIGNMENT=16
CONFIG_SOC_DS_SIGNATURE_MAX_BIT_LEN=4096
CONFIG_SOC_DS_KEY_PARAM_MD_IV_LENGTH=16
@ -209,7 +208,7 @@ CONFIG_SOC_RTCIO_INPUT_OUTPUT_SUPPORTED=y
CONFIG_SOC_RTCIO_HOLD_SUPPORTED=y
CONFIG_SOC_RTCIO_WAKE_SUPPORTED=y
CONFIG_SOC_LP_IO_CLOCK_IS_INDEPENDENT=y
CONFIG_SOC_SDM_GROUPS=1
CONFIG_SOC_SDM_GROUPS=y
CONFIG_SOC_SDM_CHANNELS_PER_GROUP=8
CONFIG_SOC_SDM_CLK_SUPPORT_APB=y
CONFIG_SOC_SPI_PERIPH_NUM=3
@ -370,9 +369,6 @@ CONFIG_SOC_BLE_DEVICE_PRIVACY_SUPPORTED=y
CONFIG_SOC_BLUFI_SUPPORTED=y
CONFIG_SOC_ULP_HAS_ADC=y
CONFIG_SOC_PHY_COMBO_MODULE=y
CONFIG_SOC_LCDCAM_CAM_SUPPORT_RGB_YUV_CONV=y
CONFIG_SOC_LCDCAM_CAM_PERIPH_NUM=1
CONFIG_SOC_LCDCAM_CAM_DATA_WIDTH_MAX=16
CONFIG_IDF_CMAKE=y
CONFIG_IDF_TOOLCHAIN="gcc"
CONFIG_IDF_TOOLCHAIN_GCC=y
@ -1035,7 +1031,6 @@ CONFIG_BT_BLE_42_FEATURES_SUPPORTED=y
CONFIG_BT_BLE_42_DTM_TEST_EN=y
CONFIG_BT_BLE_42_ADV_EN=y
CONFIG_BT_BLE_42_SCAN_EN=y
CONFIG_BT_BLE_VENDOR_HCI_EN=y
# CONFIG_BT_BLE_HIGH_DUTY_ADV_INTERVAL is not set
# CONFIG_BT_ABORT_WHEN_ALLOCATION_FAILS is not set
# end of Bluedroid Options
@ -1257,7 +1252,6 @@ CONFIG_ESP_TLS_USE_DS_PERIPHERAL=y
# CONFIG_ESP_TLS_SERVER_MIN_AUTH_MODE_OPTIONAL is not set
# CONFIG_ESP_TLS_PSK_VERIFICATION is not set
# CONFIG_ESP_TLS_INSECURE is not set
CONFIG_ESP_TLS_DYN_BUF_STRATEGY_SUPPORTED=y
# end of ESP-TLS
#
@ -1285,12 +1279,6 @@ CONFIG_ESP_ERR_TO_NAME_LOOKUP=y
CONFIG_ESP_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y
# end of Common ESP-related
#
# ESP-Driver:Camera Controller Configurations
#
# CONFIG_CAM_CTLR_DVP_CAM_ISR_CACHE_SAFE is not set
# end of ESP-Driver:Camera Controller Configurations
#
# ESP-Driver:GPIO Configurations
#
@ -1608,11 +1596,8 @@ CONFIG_ESP_PHY_RF_CAL_PARTIAL=y
# CONFIG_ESP_PHY_RF_CAL_NONE is not set
# CONFIG_ESP_PHY_RF_CAL_FULL is not set
CONFIG_ESP_PHY_CALIBRATION_MODE=0
CONFIG_ESP_PHY_PLL_TRACK_PERIOD_MS=1000
# CONFIG_ESP_PHY_PLL_TRACK_DEBUG is not set
# CONFIG_ESP_PHY_RECORD_USED_TIME is not set
CONFIG_ESP_PHY_IRAM_OPT=y
# CONFIG_ESP_PHY_DEBUG is not set
# end of PHY
#
@ -2283,7 +2268,6 @@ CONFIG_MBEDTLS_DYNAMIC_BUFFER=y
# CONFIG_MBEDTLS_X509_TRUSTED_CERT_CALLBACK is not set
# CONFIG_MBEDTLS_SSL_CONTEXT_SERIALIZATION is not set
# CONFIG_MBEDTLS_SSL_KEEP_PEER_CERTIFICATE is not set
# CONFIG_MBEDTLS_SSL_KEYING_MATERIAL_EXPORT is not set
CONFIG_MBEDTLS_PKCS7_C=y
# end of mbedTLS v3.x related
@ -3314,7 +3298,6 @@ CONFIG_BT_NIMBLE_COEX_PHY_CODED_TX_RX_TLIM_DIS=y
CONFIG_SW_COEXIST_ENABLE=y
CONFIG_ESP32_WIFI_SW_COEXIST_ENABLE=y
CONFIG_ESP_WIFI_SW_COEXIST_ENABLE=y
# CONFIG_CAM_CTLR_DVP_CAM_ISR_IRAM_SAFE is not set
# CONFIG_MCPWM_ISR_IN_IRAM is not set
# CONFIG_EVENT_LOOP_PROFILING is not set
CONFIG_POST_EVENTS_FROM_ISR=y