feat: 火山RTC字幕文本实时显示 + 字幕日志精简
1. DataCallback 新增 subv(字幕)二进制消息前缀识别,字幕数据正确解析并转发到应用层 2. OnIncomingJson 处理 type:"subtitle",通过 userId 区分用户语音识别(STT)和 AI 回复文本 3. 字幕日志简化:subv 消息仅打印"接收下行二进制消息(字幕)",不展开 JSON 内容 4. 新增postman请求的参数 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
0a6b35d979
commit
98e40ac163
@ -1208,6 +1208,34 @@ void Application::Start() {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
} else if (strcmp(type->valuestring, "subtitle") == 0) {
|
||||||
|
// 火山 RTC 字幕消息:data 数组中包含 text、userId、definite
|
||||||
|
auto data_arr = cJSON_GetObjectItem(root, "data");
|
||||||
|
if (data_arr && cJSON_IsArray(data_arr)) {
|
||||||
|
for (int i = 0; i < cJSON_GetArraySize(data_arr); ++i) {
|
||||||
|
auto item = cJSON_GetArrayItem(data_arr, i);
|
||||||
|
auto text = cJSON_GetObjectItem(item, "text");
|
||||||
|
auto user_id = cJSON_GetObjectItem(item, "userId");
|
||||||
|
auto definite = cJSON_GetObjectItem(item, "definite");
|
||||||
|
if (!text || !cJSON_IsString(text) || !text->valuestring[0]) continue;
|
||||||
|
|
||||||
|
bool is_final = definite && cJSON_IsTrue(definite);
|
||||||
|
// 通过 userId 判断是用户说的还是 AI 回复
|
||||||
|
// device_name 对应本设备 UID,匹配则为用户语音识别
|
||||||
|
bool is_user = false;
|
||||||
|
if (user_id && cJSON_IsString(user_id)) {
|
||||||
|
const char* dev_name = CONFIG_VOLC_DEVICE_NAME;
|
||||||
|
is_user = (strcmp(user_id->valuestring, dev_name) == 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string msg = text->valuestring;
|
||||||
|
const char* role = is_user ? "user" : "assistant";
|
||||||
|
ESP_LOGI(TAG, "%s %s: %s", is_final ? ">>" : "..", role, msg.c_str());
|
||||||
|
Schedule([this, display, msg, role_str = std::string(role)]() {
|
||||||
|
display->SetChatMessage(role_str.c_str(), msg.c_str());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
} else if (strcmp(type->valuestring, "iot") == 0) {
|
} else if (strcmp(type->valuestring, "iot") == 0) {
|
||||||
auto commands = cJSON_GetObjectItem(root, "commands");
|
auto commands = cJSON_GetObjectItem(root, "commands");
|
||||||
if (commands != NULL) {
|
if (commands != NULL) {
|
||||||
|
|||||||
@ -591,12 +591,18 @@ void VolcRtcProtocol::DataCallback(void* context, const void* data, size_t len,
|
|||||||
bool is_ctrl = (memcmp(buf, "ctrl", 4) == 0);
|
bool is_ctrl = (memcmp(buf, "ctrl", 4) == 0);
|
||||||
bool is_conv = (memcmp(buf, "conv", 4) == 0);
|
bool is_conv = (memcmp(buf, "conv", 4) == 0);
|
||||||
bool is_tool = (memcmp(buf, "tool", 4) == 0);
|
bool is_tool = (memcmp(buf, "tool", 4) == 0);
|
||||||
if (is_ctrl || is_conv || is_tool) {
|
bool is_subv = (memcmp(buf, "subv", 4) == 0);
|
||||||
|
if (is_ctrl || is_conv || is_tool || is_subv) {
|
||||||
uint32_t json_len = (uint32_t)((buf[4] << 24) | (buf[5] << 16) | (buf[6] << 8) | (buf[7]));
|
uint32_t json_len = (uint32_t)((buf[4] << 24) | (buf[5] << 16) | (buf[6] << 8) | (buf[7]));
|
||||||
if (json_len > 0 && (size_t)(8 + json_len) <= len) {
|
if (json_len > 0 && (size_t)(8 + json_len) <= len) {
|
||||||
json_text.assign(reinterpret_cast<const char*>(buf + 8), json_len);
|
json_text.assign(reinterpret_cast<const char*>(buf + 8), json_len);
|
||||||
if (!protocol->suppress_incoming_message_log_) {
|
if (!protocol->suppress_incoming_message_log_) {
|
||||||
ESP_LOGI(TAG, "接收下行二进制消息(%s): %.*s", is_ctrl ? "ctrl" : (is_conv ? "conv" : "tool"), (int)json_text.size(), json_text.c_str());
|
if (is_subv) {
|
||||||
|
ESP_LOGI(TAG, "接收下行二进制消息(字幕)");
|
||||||
|
} else {
|
||||||
|
const char* prefix = is_ctrl ? "ctrl" : (is_conv ? "conv" : "tool");
|
||||||
|
ESP_LOGI(TAG, "接收下行二进制消息(%s): %.*s", prefix, (int)json_text.size(), json_text.c_str());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
21
sdkconfig
21
sdkconfig
@ -14,7 +14,6 @@ CONFIG_SOC_GDMA_SUPPORTED=y
|
|||||||
CONFIG_SOC_AHB_GDMA_SUPPORTED=y
|
CONFIG_SOC_AHB_GDMA_SUPPORTED=y
|
||||||
CONFIG_SOC_GPTIMER_SUPPORTED=y
|
CONFIG_SOC_GPTIMER_SUPPORTED=y
|
||||||
CONFIG_SOC_LCDCAM_SUPPORTED=y
|
CONFIG_SOC_LCDCAM_SUPPORTED=y
|
||||||
CONFIG_SOC_LCDCAM_CAM_SUPPORTED=y
|
|
||||||
CONFIG_SOC_LCDCAM_I80_LCD_SUPPORTED=y
|
CONFIG_SOC_LCDCAM_I80_LCD_SUPPORTED=y
|
||||||
CONFIG_SOC_LCDCAM_RGB_LCD_SUPPORTED=y
|
CONFIG_SOC_LCDCAM_RGB_LCD_SUPPORTED=y
|
||||||
CONFIG_SOC_MCPWM_SUPPORTED=y
|
CONFIG_SOC_MCPWM_SUPPORTED=y
|
||||||
@ -102,7 +101,7 @@ CONFIG_SOC_CPU_HAS_FPU=y
|
|||||||
CONFIG_SOC_HP_CPU_HAS_MULTIPLE_CORES=y
|
CONFIG_SOC_HP_CPU_HAS_MULTIPLE_CORES=y
|
||||||
CONFIG_SOC_CPU_BREAKPOINTS_NUM=2
|
CONFIG_SOC_CPU_BREAKPOINTS_NUM=2
|
||||||
CONFIG_SOC_CPU_WATCHPOINTS_NUM=2
|
CONFIG_SOC_CPU_WATCHPOINTS_NUM=2
|
||||||
CONFIG_SOC_CPU_WATCHPOINT_MAX_REGION_SIZE=0x40
|
CONFIG_SOC_CPU_WATCHPOINT_MAX_REGION_SIZE=64
|
||||||
CONFIG_SOC_SIMD_PREFERRED_DATA_ALIGNMENT=16
|
CONFIG_SOC_SIMD_PREFERRED_DATA_ALIGNMENT=16
|
||||||
CONFIG_SOC_DS_SIGNATURE_MAX_BIT_LEN=4096
|
CONFIG_SOC_DS_SIGNATURE_MAX_BIT_LEN=4096
|
||||||
CONFIG_SOC_DS_KEY_PARAM_MD_IV_LENGTH=16
|
CONFIG_SOC_DS_KEY_PARAM_MD_IV_LENGTH=16
|
||||||
@ -209,7 +208,7 @@ CONFIG_SOC_RTCIO_INPUT_OUTPUT_SUPPORTED=y
|
|||||||
CONFIG_SOC_RTCIO_HOLD_SUPPORTED=y
|
CONFIG_SOC_RTCIO_HOLD_SUPPORTED=y
|
||||||
CONFIG_SOC_RTCIO_WAKE_SUPPORTED=y
|
CONFIG_SOC_RTCIO_WAKE_SUPPORTED=y
|
||||||
CONFIG_SOC_LP_IO_CLOCK_IS_INDEPENDENT=y
|
CONFIG_SOC_LP_IO_CLOCK_IS_INDEPENDENT=y
|
||||||
CONFIG_SOC_SDM_GROUPS=1
|
CONFIG_SOC_SDM_GROUPS=y
|
||||||
CONFIG_SOC_SDM_CHANNELS_PER_GROUP=8
|
CONFIG_SOC_SDM_CHANNELS_PER_GROUP=8
|
||||||
CONFIG_SOC_SDM_CLK_SUPPORT_APB=y
|
CONFIG_SOC_SDM_CLK_SUPPORT_APB=y
|
||||||
CONFIG_SOC_SPI_PERIPH_NUM=3
|
CONFIG_SOC_SPI_PERIPH_NUM=3
|
||||||
@ -370,9 +369,6 @@ CONFIG_SOC_BLE_DEVICE_PRIVACY_SUPPORTED=y
|
|||||||
CONFIG_SOC_BLUFI_SUPPORTED=y
|
CONFIG_SOC_BLUFI_SUPPORTED=y
|
||||||
CONFIG_SOC_ULP_HAS_ADC=y
|
CONFIG_SOC_ULP_HAS_ADC=y
|
||||||
CONFIG_SOC_PHY_COMBO_MODULE=y
|
CONFIG_SOC_PHY_COMBO_MODULE=y
|
||||||
CONFIG_SOC_LCDCAM_CAM_SUPPORT_RGB_YUV_CONV=y
|
|
||||||
CONFIG_SOC_LCDCAM_CAM_PERIPH_NUM=1
|
|
||||||
CONFIG_SOC_LCDCAM_CAM_DATA_WIDTH_MAX=16
|
|
||||||
CONFIG_IDF_CMAKE=y
|
CONFIG_IDF_CMAKE=y
|
||||||
CONFIG_IDF_TOOLCHAIN="gcc"
|
CONFIG_IDF_TOOLCHAIN="gcc"
|
||||||
CONFIG_IDF_TOOLCHAIN_GCC=y
|
CONFIG_IDF_TOOLCHAIN_GCC=y
|
||||||
@ -1035,7 +1031,6 @@ CONFIG_BT_BLE_42_FEATURES_SUPPORTED=y
|
|||||||
CONFIG_BT_BLE_42_DTM_TEST_EN=y
|
CONFIG_BT_BLE_42_DTM_TEST_EN=y
|
||||||
CONFIG_BT_BLE_42_ADV_EN=y
|
CONFIG_BT_BLE_42_ADV_EN=y
|
||||||
CONFIG_BT_BLE_42_SCAN_EN=y
|
CONFIG_BT_BLE_42_SCAN_EN=y
|
||||||
CONFIG_BT_BLE_VENDOR_HCI_EN=y
|
|
||||||
# CONFIG_BT_BLE_HIGH_DUTY_ADV_INTERVAL is not set
|
# CONFIG_BT_BLE_HIGH_DUTY_ADV_INTERVAL is not set
|
||||||
# CONFIG_BT_ABORT_WHEN_ALLOCATION_FAILS is not set
|
# CONFIG_BT_ABORT_WHEN_ALLOCATION_FAILS is not set
|
||||||
# end of Bluedroid Options
|
# end of Bluedroid Options
|
||||||
@ -1257,7 +1252,6 @@ CONFIG_ESP_TLS_USE_DS_PERIPHERAL=y
|
|||||||
# CONFIG_ESP_TLS_SERVER_MIN_AUTH_MODE_OPTIONAL is not set
|
# CONFIG_ESP_TLS_SERVER_MIN_AUTH_MODE_OPTIONAL is not set
|
||||||
# CONFIG_ESP_TLS_PSK_VERIFICATION is not set
|
# CONFIG_ESP_TLS_PSK_VERIFICATION is not set
|
||||||
# CONFIG_ESP_TLS_INSECURE is not set
|
# CONFIG_ESP_TLS_INSECURE is not set
|
||||||
CONFIG_ESP_TLS_DYN_BUF_STRATEGY_SUPPORTED=y
|
|
||||||
# end of ESP-TLS
|
# end of ESP-TLS
|
||||||
|
|
||||||
#
|
#
|
||||||
@ -1285,12 +1279,6 @@ CONFIG_ESP_ERR_TO_NAME_LOOKUP=y
|
|||||||
CONFIG_ESP_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y
|
CONFIG_ESP_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y
|
||||||
# end of Common ESP-related
|
# end of Common ESP-related
|
||||||
|
|
||||||
#
|
|
||||||
# ESP-Driver:Camera Controller Configurations
|
|
||||||
#
|
|
||||||
# CONFIG_CAM_CTLR_DVP_CAM_ISR_CACHE_SAFE is not set
|
|
||||||
# end of ESP-Driver:Camera Controller Configurations
|
|
||||||
|
|
||||||
#
|
#
|
||||||
# ESP-Driver:GPIO Configurations
|
# ESP-Driver:GPIO Configurations
|
||||||
#
|
#
|
||||||
@ -1608,11 +1596,8 @@ CONFIG_ESP_PHY_RF_CAL_PARTIAL=y
|
|||||||
# CONFIG_ESP_PHY_RF_CAL_NONE is not set
|
# CONFIG_ESP_PHY_RF_CAL_NONE is not set
|
||||||
# CONFIG_ESP_PHY_RF_CAL_FULL is not set
|
# CONFIG_ESP_PHY_RF_CAL_FULL is not set
|
||||||
CONFIG_ESP_PHY_CALIBRATION_MODE=0
|
CONFIG_ESP_PHY_CALIBRATION_MODE=0
|
||||||
CONFIG_ESP_PHY_PLL_TRACK_PERIOD_MS=1000
|
|
||||||
# CONFIG_ESP_PHY_PLL_TRACK_DEBUG is not set
|
# CONFIG_ESP_PHY_PLL_TRACK_DEBUG is not set
|
||||||
# CONFIG_ESP_PHY_RECORD_USED_TIME is not set
|
# CONFIG_ESP_PHY_RECORD_USED_TIME is not set
|
||||||
CONFIG_ESP_PHY_IRAM_OPT=y
|
|
||||||
# CONFIG_ESP_PHY_DEBUG is not set
|
|
||||||
# end of PHY
|
# end of PHY
|
||||||
|
|
||||||
#
|
#
|
||||||
@ -2283,7 +2268,6 @@ CONFIG_MBEDTLS_DYNAMIC_BUFFER=y
|
|||||||
# CONFIG_MBEDTLS_X509_TRUSTED_CERT_CALLBACK is not set
|
# CONFIG_MBEDTLS_X509_TRUSTED_CERT_CALLBACK is not set
|
||||||
# CONFIG_MBEDTLS_SSL_CONTEXT_SERIALIZATION is not set
|
# CONFIG_MBEDTLS_SSL_CONTEXT_SERIALIZATION is not set
|
||||||
# CONFIG_MBEDTLS_SSL_KEEP_PEER_CERTIFICATE is not set
|
# CONFIG_MBEDTLS_SSL_KEEP_PEER_CERTIFICATE is not set
|
||||||
# CONFIG_MBEDTLS_SSL_KEYING_MATERIAL_EXPORT is not set
|
|
||||||
CONFIG_MBEDTLS_PKCS7_C=y
|
CONFIG_MBEDTLS_PKCS7_C=y
|
||||||
# end of mbedTLS v3.x related
|
# end of mbedTLS v3.x related
|
||||||
|
|
||||||
@ -3314,7 +3298,6 @@ CONFIG_BT_NIMBLE_COEX_PHY_CODED_TX_RX_TLIM_DIS=y
|
|||||||
CONFIG_SW_COEXIST_ENABLE=y
|
CONFIG_SW_COEXIST_ENABLE=y
|
||||||
CONFIG_ESP32_WIFI_SW_COEXIST_ENABLE=y
|
CONFIG_ESP32_WIFI_SW_COEXIST_ENABLE=y
|
||||||
CONFIG_ESP_WIFI_SW_COEXIST_ENABLE=y
|
CONFIG_ESP_WIFI_SW_COEXIST_ENABLE=y
|
||||||
# CONFIG_CAM_CTLR_DVP_CAM_ISR_IRAM_SAFE is not set
|
|
||||||
# CONFIG_MCPWM_ISR_IN_IRAM is not set
|
# CONFIG_MCPWM_ISR_IN_IRAM is not set
|
||||||
# CONFIG_EVENT_LOOP_PROFILING is not set
|
# CONFIG_EVENT_LOOP_PROFILING is not set
|
||||||
CONFIG_POST_EVENTS_FROM_ISR=y
|
CONFIG_POST_EVENTS_FROM_ISR=y
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user