2008 lines
83 KiB
C++
2008 lines
83 KiB
C++
#include "application.h"
|
||
#include "board.h"
|
||
#include "wifi_board.h"
|
||
#include "display.h"
|
||
#include "system_info.h"
|
||
#include "ml307_ssl_transport.h"
|
||
#include "audio_codec.h"
|
||
#include "settings.h"
|
||
#include "mqtt_protocol.h"
|
||
#include "websocket_protocol.h"
|
||
#include "font_awesome_symbols.h"
|
||
#include "iot/thing_manager.h"
|
||
#include "assets/lang_config.h"
|
||
#include "boards/common/qmi8658a.h" // 添加qmi8658a_data_t类型的头文件
|
||
#include "boards/movecall-moji-esp32s3/movecall_moji_esp32s3.h" // 添加MovecallMojiESP32S3类的头文件
|
||
|
||
#include <cstring>
|
||
#include <esp_log.h>
|
||
#include <cJSON.h>
|
||
#include <driver/gpio.h>
|
||
#include <arpa/inet.h>
|
||
#include <esp_app_desc.h>
|
||
#include <cmath> // 用于sqrt函数
|
||
|
||
#define TAG "Application"
|
||
#define MAC_TAG "WiFiMAC"
|
||
|
||
|
||
static const char* const STATE_STRINGS[] = {
|
||
"unknown",
|
||
"starting",
|
||
"configuring",
|
||
"idle",
|
||
"connecting",
|
||
"listening",
|
||
"speaking",
|
||
"upgrading",
|
||
"activating",
|
||
"fatal_error",
|
||
"invalid_state"
|
||
};
|
||
|
||
Application::Application() {
|
||
event_group_ = xEventGroupCreate();
|
||
background_task_ = new BackgroundTask(4096 * 8);
|
||
|
||
esp_timer_create_args_t clock_timer_args = {
|
||
.callback = [](void* arg) {
|
||
Application* app = (Application*)arg;
|
||
app->OnClockTimer();
|
||
},
|
||
.arg = this,
|
||
.dispatch_method = ESP_TIMER_TASK,
|
||
.name = "clock_timer",
|
||
.skip_unhandled_events = true
|
||
};
|
||
esp_timer_create(&clock_timer_args, &clock_timer_handle_);
|
||
}
|
||
|
||
Application::~Application() {
|
||
if (clock_timer_handle_ != nullptr) {
|
||
esp_timer_stop(clock_timer_handle_);
|
||
esp_timer_delete(clock_timer_handle_);
|
||
}
|
||
if (background_task_ != nullptr) {
|
||
delete background_task_;
|
||
}
|
||
vEventGroupDelete(event_group_);
|
||
}
|
||
|
||
void Application::CheckNewVersion() {
|
||
auto& board = Board::GetInstance();
|
||
auto display = board.GetDisplay();
|
||
// Check if there is a new firmware version available
|
||
ota_.SetPostData(board.GetJson());// 发送当前设备的JSON数据到OTA服务器，用于检查是否有新的固件版本 包办板载信息 BOARD_TYPE
|
||
|
||
const int MAX_RETRY = 10;
|
||
int retry_count = 0;
|
||
|
||
while (true) {
|
||
if (!ota_.CheckVersion()) {
|
||
retry_count++;
|
||
if (retry_count >= MAX_RETRY) {
|
||
ESP_LOGE(TAG, "Too many retries, exit version check");
|
||
return;
|
||
}
|
||
ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", 60, retry_count, MAX_RETRY);
|
||
vTaskDelay(pdMS_TO_TICKS(60000));
|
||
continue;
|
||
}
|
||
retry_count = 0;
|
||
|
||
if (ota_.HasNewVersion()) {
|
||
Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "happy", Lang::Sounds::P3_UPGRADE);
|
||
// Wait for the chat state to be idle
|
||
do {
|
||
vTaskDelay(pdMS_TO_TICKS(3000));
|
||
} while (GetDeviceState() != kDeviceStateIdle);
|
||
|
||
// Use main task to do the upgrade, not cancelable
|
||
Schedule([this, display]() {
|
||
SetDeviceState(kDeviceStateUpgrading);
|
||
|
||
display->SetIcon(FONT_AWESOME_DOWNLOAD);
|
||
std::string message = std::string(Lang::Strings::NEW_VERSION) + ota_.GetFirmwareVersion();
|
||
display->SetChatMessage("system", message.c_str());
|
||
|
||
auto& board = Board::GetInstance();
|
||
board.SetPowerSaveMode(false);// 关闭低功耗模式
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Stop();
|
||
#endif
|
||
// 预先关闭音频输出，避免升级过程有音频操作
|
||
auto codec = board.GetAudioCodec();
|
||
codec->EnableInput(false);
|
||
codec->EnableOutput(false);
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.clear();
|
||
}
|
||
background_task_->WaitForCompletion();
|
||
delete background_task_;
|
||
background_task_ = nullptr;
|
||
vTaskDelay(pdMS_TO_TICKS(1000));
|
||
|
||
ota_.StartUpgrade([display](int progress, size_t speed) {
|
||
char buffer[64];
|
||
snprintf(buffer, sizeof(buffer), "%d%% %zuKB/s", progress, speed / 1024);
|
||
display->SetChatMessage("system", buffer);
|
||
});
|
||
|
||
// If upgrade success, the device will reboot and never reach here
|
||
display->SetStatus(Lang::Strings::UPGRADE_FAILED);
|
||
ESP_LOGI(TAG, "Firmware upgrade failed...");
|
||
vTaskDelay(pdMS_TO_TICKS(3000));
|
||
Reboot();
|
||
});
|
||
|
||
return;
|
||
}
|
||
|
||
// No new version, mark the current version as valid
|
||
ota_.MarkCurrentVersionValid();
|
||
std::string message = std::string(Lang::Strings::VERSION) + ota_.GetCurrentVersion();
|
||
display->ShowNotification(message.c_str());
|
||
|
||
// 检查是否有设备激活码
|
||
// if (ota_.HasActivationCode()) {
|
||
// // Activation code is valid
|
||
// SetDeviceState(kDeviceStateActivating);//设置设备状态为激活中
|
||
// // ShowActivationCode();//显示设备激活码
|
||
|
||
// // Check again in 60 seconds or until the device is idle
|
||
// for (int i = 0; i < 60; ++i) {
|
||
// if (device_state_ == kDeviceStateIdle) {
|
||
// break;
|
||
// }
|
||
// vTaskDelay(pdMS_TO_TICKS(1000));
|
||
// }
|
||
// continue;
|
||
// }
|
||
|
||
SetDeviceState(kDeviceStateIdle);
|
||
display->SetChatMessage("system", "");
|
||
ResetDecoder();
|
||
PlaySound(Lang::Sounds::P3_SUCCESS);
|
||
// Exit the loop if upgrade or idle
|
||
break;
|
||
}
|
||
}
|
||
|
||
// 取消设备激活码播报,当前设备绑定使用Wi-Fi的Mac地址进行绑定
|
||
// void Application::ShowActivationCode() {
|
||
// auto& message = ota_.GetActivationMessage();
|
||
// auto& code = ota_.GetActivationCode();
|
||
|
||
// struct digit_sound {
|
||
// char digit;
|
||
// const std::string_view& sound;
|
||
// };
|
||
// static const std::array<digit_sound, 10> digit_sounds{{
|
||
// digit_sound{'0', Lang::Sounds::P3_0},
|
||
// digit_sound{'1', Lang::Sounds::P3_1},
|
||
// digit_sound{'2', Lang::Sounds::P3_2},
|
||
// digit_sound{'3', Lang::Sounds::P3_3},
|
||
// digit_sound{'4', Lang::Sounds::P3_4},
|
||
// digit_sound{'5', Lang::Sounds::P3_5},
|
||
// digit_sound{'6', Lang::Sounds::P3_6},
|
||
// digit_sound{'7', Lang::Sounds::P3_7},
|
||
// digit_sound{'8', Lang::Sounds::P3_8},
|
||
// digit_sound{'9', Lang::Sounds::P3_9}
|
||
// }};
|
||
|
||
// // This sentence uses 9KB of SRAM, so we need to wait for it to finish
|
||
// Alert(Lang::Strings::ACTIVATION, message.c_str(), "happy", Lang::Sounds::P3_ACTIVATION);
|
||
// vTaskDelay(pdMS_TO_TICKS(1000));
|
||
// background_task_->WaitForCompletion();
|
||
|
||
// for (const auto& digit : code) {
|
||
// auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(),
|
||
// [digit](const digit_sound& ds) { return ds.digit == digit; });
|
||
// if (it != digit_sounds.end()) {
|
||
// PlaySound(it->sound);
|
||
// }
|
||
// }
|
||
// }
|
||
|
||
// 新增代码(小程序控制 暂停播放 音频)
|
||
// =========================================================
|
||
void Application::PauseAudioPlayback() {
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
if (!audio_paused_) {
|
||
audio_paused_ = true;// 暂停播放(更新标志位)
|
||
ESP_LOGI(TAG, "🔇 从服务器接收到暂停播放指令");
|
||
|
||
// 恢复原始处理方式：立即停止音频输出
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(false);// 暂停时立即停止音频输出
|
||
ESP_LOGI(TAG, "⏸️ 音频编解码器输出已禁用，实现立即暂停");
|
||
}
|
||
ESP_LOGI(TAG, "⏸️ 音频播放已暂停");
|
||
}
|
||
}
|
||
// 新增代码(小程序控制 继续播放 音频)
|
||
void Application::ResumeAudioPlayback() {
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
if (audio_paused_) {
|
||
audio_paused_ = false;// 恢复播放(更新标志位)
|
||
ESP_LOGI(TAG, "<EFBFBD> 从服务器接收到继续播放指令");
|
||
|
||
// 恢复原始处理方式：重新启用音频输出
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(true);// 恢复时重新启用音频输出
|
||
ESP_LOGI(TAG, "▶️ 音频编解码器输出已启用");
|
||
}
|
||
ESP_LOGI(TAG, "▶️ 音频播放已恢复");
|
||
}
|
||
}
|
||
// =========================================================
|
||
|
||
void Application::Alert(const char* status, const char* message, const char* emotion, const std::string_view& sound) {
|
||
ESP_LOGW(TAG, "Alert %s: %s [%s]", status, message, emotion);
|
||
auto display = Board::GetInstance().GetDisplay();
|
||
display->SetStatus(status);
|
||
display->SetEmotion(emotion);
|
||
display->SetChatMessage("system", message);
|
||
if (!sound.empty()) {
|
||
ResetDecoder();
|
||
PlaySound(sound);
|
||
}
|
||
}
|
||
|
||
void Application::DismissAlert() {
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
auto display = Board::GetInstance().GetDisplay();
|
||
display->SetStatus(Lang::Strings::STANDBY);
|
||
display->SetEmotion("neutral");
|
||
display->SetChatMessage("system", "");
|
||
}
|
||
}
|
||
|
||
void Application::PlaySound(const std::string_view& sound) {
|
||
// The assets are encoded at 16000Hz, 60ms frame duration
|
||
SetDecodeSampleRate(16000, 60);
|
||
const char* data = sound.data();
|
||
size_t size = sound.size();
|
||
for (const char* p = data; p < data + size; ) {
|
||
auto p3 = (BinaryProtocol3*)p;
|
||
p += sizeof(BinaryProtocol3);
|
||
|
||
auto payload_size = ntohs(p3->payload_size);
|
||
std::vector<uint8_t> opus;
|
||
opus.resize(payload_size);
|
||
memcpy(opus.data(), p3->payload, payload_size);
|
||
p += payload_size;
|
||
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.emplace_back(std::move(opus));
|
||
}
|
||
}
|
||
|
||
// 切换聊天状态的函数，用于在不同的设备状态之间进行切换
|
||
void Application::ToggleChatState() {
|
||
// 如果当前设备状态是激活中，则将状态设置为空闲并返回
|
||
if (device_state_ == kDeviceStateActivating) {
|
||
SetDeviceState(kDeviceStateIdle); // 设置设备状态为空闲
|
||
return; // 直接返回，不执行后续逻辑
|
||
}
|
||
|
||
// 检查协议对象是否已初始化
|
||
if (!protocol_) {
|
||
ESP_LOGE(TAG, "Protocol not initialized"); // 记录错误日志：协议未初始化
|
||
return; // 协议未初始化则直接返回
|
||
}
|
||
|
||
// 如果当前设备状态是idle空闲，则尝试开始聊天
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
// 使用Schedule函数异步执行以下操作
|
||
Schedule([this]() {
|
||
SetDeviceState(kDeviceStateConnecting); // 设置设备状态为连接中
|
||
ESP_LOGI(TAG, "Attempting to open audio channel"); // 记录信息日志：尝试打开音频通道
|
||
|
||
// 尝试打开音频通道
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
ESP_LOGW(TAG, "Failed to open audio channel, will retry in 2 seconds"); // 记录警告日志：打开音频通道失败
|
||
SetDeviceState(kDeviceStateIdle); // 将设备状态重新设置为空闲
|
||
|
||
// 2秒后自动重试
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(2000)); // 延迟2000毫秒（2秒）
|
||
ESP_LOGI(TAG, "Retrying audio channel connection"); // 记录信息日志：重试音频通道连接
|
||
ToggleChatState(); // 递归调用自身，重新尝试切换聊天状态
|
||
});
|
||
return; // 返回，不执行后续的SetListeningMode
|
||
}
|
||
|
||
// 音频通道打开成功，根据实时聊天是否启用来设置监听模式
|
||
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeAutoStop);
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) { // 如果当前设备状态是说话中
|
||
// 异步执行中止说话操作
|
||
Schedule([this]() {
|
||
AbortSpeaking(kAbortReasonNone); // 中止说话，原因为无特定原因
|
||
});
|
||
} else if (device_state_ == kDeviceStateListening) { // 如果当前设备状态是监听中
|
||
// 异步执行关闭音频通道操作
|
||
Schedule([this]() {
|
||
protocol_->CloseAudioChannel(); // 关闭音频通道
|
||
});
|
||
}
|
||
}
|
||
|
||
void Application::ToggleListeningState() {
|
||
if (device_state_ == kDeviceStateActivating) {
|
||
SetDeviceState(kDeviceStateIdle);
|
||
return;
|
||
}
|
||
|
||
if (!protocol_) {
|
||
ESP_LOGE(TAG, "Protocol not initialized");
|
||
return;
|
||
}
|
||
|
||
// 简单的状态切换：idle <-> listening
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
// 从待命状态进入聆听状态
|
||
Schedule([this]() {
|
||
SetDeviceState(kDeviceStateConnecting);
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
return;
|
||
}
|
||
SetListeningMode(kListeningModeManualStop);
|
||
ESP_LOGI(TAG, "Interrupt button: Entering listening state");
|
||
});
|
||
} else if (device_state_ == kDeviceStateListening) {
|
||
// 从聆听状态返回待命状态
|
||
Schedule([this]() {
|
||
protocol_->CloseAudioChannel();
|
||
ESP_LOGI(TAG, "Interrupt button: Returning to idle state");
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
// 如果正在说话，中止说话并返回待命状态
|
||
Schedule([this]() {
|
||
AbortSpeaking(kAbortReasonNone);
|
||
ESP_LOGI(TAG, "Interrupt button: Stopping speech and returning to idle state");
|
||
});
|
||
} else if (device_state_ == kDeviceStateConnecting) {
|
||
// 如果正在连接，直接返回待命状态
|
||
Schedule([this]() {
|
||
SetDeviceState(kDeviceStateIdle);
|
||
ESP_LOGI(TAG, "Interrupt button: Canceling connection and returning to idle state");
|
||
});
|
||
}
|
||
}
|
||
|
||
void Application::StartListening() {
|
||
if (device_state_ == kDeviceStateActivating) {
|
||
SetDeviceState(kDeviceStateIdle);
|
||
return;
|
||
}
|
||
|
||
if (!protocol_) {
|
||
ESP_LOGE(TAG, "Protocol not initialized");
|
||
return;
|
||
}
|
||
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
Schedule([this]() {
|
||
if (!protocol_->IsAudioChannelOpened()) {
|
||
SetDeviceState(kDeviceStateConnecting);
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
return;
|
||
}
|
||
}
|
||
|
||
SetListeningMode(kListeningModeManualStop);
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
Schedule([this]() {
|
||
AbortSpeaking(kAbortReasonNone);
|
||
SetListeningMode(kListeningModeManualStop);
|
||
});
|
||
}
|
||
}
|
||
|
||
void Application::StopListening() {
|
||
Schedule([this]() {
|
||
if (device_state_ == kDeviceStateListening) {
|
||
protocol_->SendStopListening();
|
||
SetDeviceState(kDeviceStateIdle);
|
||
}
|
||
});
|
||
}
|
||
|
||
void Application::SendTextMessage(const std::string& text) {
|
||
if (!protocol_) {
|
||
ESP_LOGE(TAG, "Protocol not initialized");
|
||
return;
|
||
}
|
||
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
Schedule([this, text]() {
|
||
SetDeviceState(kDeviceStateConnecting);
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
return;
|
||
}
|
||
|
||
// 发送文本消息
|
||
protocol_->SendTextMessage(text);
|
||
ESP_LOGI(TAG, "Sent text message: %s", text.c_str());
|
||
|
||
// 立即启动监听模式以接收语音回复
|
||
ESP_LOGI(TAG, "realtime_chat_enabled_=%s", realtime_chat_enabled_ ? "true" : "false");
|
||
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeManualStop);
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
Schedule([this, text]() {
|
||
AbortSpeaking(kAbortReasonNone);
|
||
protocol_->SendTextMessage(text);
|
||
ESP_LOGI(TAG, "Sent text message: %s", text.c_str());
|
||
|
||
// 启动监听模式以接收语音回复
|
||
ESP_LOGI(TAG, "realtime_chat_enabled_=%s", realtime_chat_enabled_ ? "true" : "false");
|
||
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeManualStop);
|
||
});
|
||
} else if (device_state_ == kDeviceStateListening) {
|
||
Schedule([this, text]() {
|
||
protocol_->SendTextMessage(text);
|
||
ESP_LOGI(TAG, "Sent text message: %s", text.c_str());
|
||
});
|
||
}
|
||
}
|
||
|
||
void Application::Start() {
|
||
auto& board = Board::GetInstance();
|
||
SetDeviceState(kDeviceStateStarting);
|
||
|
||
/* Setup the display */
|
||
auto display = board.GetDisplay();
|
||
|
||
/* Setup the audio codec */
|
||
auto codec = board.GetAudioCodec();
|
||
opus_decoder_ = std::make_unique<OpusDecoderWrapper>(codec->output_sample_rate(), 1, OPUS_FRAME_DURATION_MS);
|
||
opus_encoder_ = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
|
||
if (realtime_chat_enabled_) {
|
||
ESP_LOGI(TAG, "Realtime chat enabled, setting opus encoder complexity to 0");
|
||
opus_encoder_->SetComplexity(0);
|
||
} else if (board.GetBoardType() == "ml307") {
|
||
ESP_LOGI(TAG, "ML307 board detected, setting opus encoder complexity to 5");
|
||
opus_encoder_->SetComplexity(5);
|
||
} else {
|
||
ESP_LOGI(TAG, "WiFi board detected, setting opus encoder complexity to 3");
|
||
opus_encoder_->SetComplexity(3);
|
||
}
|
||
|
||
if (codec->input_sample_rate() != 16000) {
|
||
input_resampler_.Configure(codec->input_sample_rate(), 16000);
|
||
reference_resampler_.Configure(codec->input_sample_rate(), 16000);
|
||
}
|
||
codec->Start();
|
||
{
|
||
int battery_level = 0;
|
||
bool charging = false;
|
||
bool discharging = false;
|
||
if (board.GetBatteryLevel(battery_level, charging, discharging)) {
|
||
// 如果电池电量低于25%，则将输出音量设置为0（静音）
|
||
if (battery_level <= 25) {
|
||
codec->SetOutputVolumeRuntime(0);
|
||
} else {
|
||
Settings s("audio", false);
|
||
int vol = s.GetInt("output_volume", AudioCodec::default_output_volume());
|
||
if (vol <= 0) {
|
||
vol = AudioCodec::default_output_volume();
|
||
}
|
||
codec->SetOutputVolumeRuntime(vol);// 设置运行时输出音量
|
||
}
|
||
}
|
||
}
|
||
|
||
xTaskCreatePinnedToCore([](void* arg) {
|
||
Application* app = (Application*)arg;
|
||
app->AudioLoop();
|
||
vTaskDelete(NULL);
|
||
}, "audio_loop", 4096 * 2, this, 8, &audio_loop_task_handle_, realtime_chat_enabled_ ? 1 : 0);
|
||
|
||
/* Start the main loop */
|
||
xTaskCreatePinnedToCore([](void* arg) {
|
||
Application* app = (Application*)arg;
|
||
app->MainLoop();
|
||
vTaskDelete(NULL);
|
||
}, "main_loop", 4096 * 2, this, 4, &main_loop_task_handle_, 0);
|
||
|
||
// 播放开机播报语音 - 在网络连接之前
|
||
ESP_LOGI(TAG, "设备启动完成，正在播放开机音效！");
|
||
//PlaySound(Lang::Sounds::P3_KAIJIBOBAO); 原有蜡笔小新音色
|
||
|
||
if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){
|
||
PlaySound(Lang::Sounds::P3_KAKA_KAIJIBOBAO);
|
||
}
|
||
else if(strcmp(CONFIG_DEVICE_ROLE, "LALA") == 0){
|
||
PlaySound(Lang::Sounds::P3_LALA_KAIJIBOBAO);
|
||
}
|
||
|
||
/* Wait for the network to be ready */
|
||
board.StartNetwork();
|
||
|
||
// Initialize the protocol
|
||
display->SetStatus(Lang::Strings::LOADING_PROTOCOL);
|
||
#ifdef CONFIG_CONNECTION_TYPE_WEBSOCKET
|
||
protocol_ = std::make_unique<WebsocketProtocol>();
|
||
#else
|
||
protocol_ = std::make_unique<MqttProtocol>();
|
||
#endif
|
||
protocol_->OnNetworkError([this](const std::string& message) {
|
||
// SetDeviceState(kDeviceStateIdle);
|
||
// Alert(Lang::Strings::ERROR, message.c_str(), "sad", Lang::Sounds::P3_EXCLAMATION);
|
||
|
||
ESP_LOGW(TAG, "Network error occurred: %s", message.c_str());
|
||
// 检查是否是TLS连接重置错误
|
||
if (message.find("TLS") != std::string::npos || message.find("-76") != std::string::npos) {
|
||
ESP_LOGI(TAG, "TLS connection reset detected, will retry connection");
|
||
SetDeviceState(kDeviceStateIdle);
|
||
|
||
// 3秒后自动重试连接
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(3000));
|
||
if (GetDeviceState() == kDeviceStateIdle) {
|
||
ESP_LOGI(TAG, "Auto-retrying connection after TLS error");
|
||
ToggleChatState();
|
||
}
|
||
});
|
||
} else {
|
||
// 其他网络错误正常处理
|
||
SetDeviceState(kDeviceStateIdle);
|
||
Alert(Lang::Strings::ERROR, message.c_str(), "sad", Lang::Sounds::P3_EXCLAMATION);
|
||
}
|
||
});
|
||
protocol_->OnIncomingAudio([this](std::vector<uint8_t>&& data) {
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.emplace_back(std::move(data));
|
||
});
|
||
protocol_->OnAudioChannelOpened([this, codec, &board]() {
|
||
ESP_LOGI(TAG, "🟢 音频通道已打开");
|
||
|
||
// 🔧 关键修复：立即取消所有待执行的电源管理任务
|
||
static TaskHandle_t power_save_task = nullptr;
|
||
if (power_save_task != nullptr) {
|
||
vTaskDelete(power_save_task);
|
||
power_save_task = nullptr;
|
||
ESP_LOGI(TAG, "🔧 取消了待执行的电源管理任务");
|
||
}
|
||
|
||
// 唤醒PowerSaveTimer，从低功耗模式恢复到正常模式
|
||
board.WakeUp();
|
||
|
||
// 立即禁用电源管理，确保连接稳定
|
||
ESP_LOGI(TAG, "🔄 禁用电源低功耗管理模式");
|
||
board.SetPowerSaveMode(false);
|
||
|
||
if (protocol_->server_sample_rate() != codec->output_sample_rate()) {
|
||
ESP_LOGW(TAG, "⚠️ 服务器采样率 %d 与设备输出采样率 %d 不匹配，重采样可能导致失真",
|
||
protocol_->server_sample_rate(), codec->output_sample_rate());
|
||
}
|
||
|
||
SetDecodeSampleRate(protocol_->server_sample_rate(), protocol_->server_frame_duration());
|
||
|
||
// 发送IoT状态信息
|
||
auto& thing_manager = iot::ThingManager::GetInstance();
|
||
protocol_->SendIotDescriptors(thing_manager.GetDescriptorsJson());
|
||
std::string states;
|
||
if (thing_manager.GetStatesJson(states, false)) {
|
||
protocol_->SendIotStates(states);
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🟢 音频通道初始化完成");
|
||
});
|
||
protocol_->OnAudioChannelClosed([this, &board]() {
|
||
ESP_LOGI(TAG, "🔴 音频通道关闭，开始清理任务");
|
||
|
||
// 🔧 关键修复：取消所有待执行的电源管理任务，防止时序冲突
|
||
static TaskHandle_t power_save_task = nullptr;
|
||
if (power_save_task != nullptr) {
|
||
vTaskDelete(power_save_task);
|
||
power_save_task = nullptr;
|
||
ESP_LOGI(TAG, "🔧 取消了之前的电源管理任务，防止时序冲突");
|
||
}
|
||
|
||
// 音频处理器已经在WebSocket断开时停止了
|
||
// 等待所有后台任务完成
|
||
background_task_->WaitForCompletion();
|
||
ESP_LOGI(TAG, "🔴 后台任务完成");
|
||
|
||
// 🔧 方案2：先设置设备状态，再启用电源管理，避免时序问题
|
||
Schedule([this, &board]() {
|
||
ESP_LOGI(TAG, "🔄 设置设备为空闲状态");
|
||
auto display = Board::GetInstance().GetDisplay();
|
||
display->SetChatMessage("system", "");
|
||
SetDeviceState(kDeviceStateIdle);
|
||
|
||
// 状态设置完成后，再启用电源管理
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
ESP_LOGI(TAG, "🔄 设备已稳定在idle状态，启用电源低功耗管理");
|
||
try {
|
||
board.SetPowerSaveMode(true);
|
||
} catch (...) {
|
||
ESP_LOGE(TAG, "❌ 设置电源管理模式失败");
|
||
}
|
||
});
|
||
});
|
||
protocol_->OnIncomingJson([this, display](const cJSON* root) {
|
||
// Parse JSON data
|
||
auto type = cJSON_GetObjectItem(root, "type");
|
||
if (strcmp(type->valuestring, "tts") == 0) {
|
||
auto state = cJSON_GetObjectItem(root, "state");
|
||
if (strcmp(state->valuestring, "start") == 0) {
|
||
Schedule([this]() {
|
||
aborted_ = false;
|
||
if (device_state_ == kDeviceStateIdle || device_state_ == kDeviceStateListening) {
|
||
SetDeviceState(kDeviceStateSpeaking);
|
||
}
|
||
});
|
||
} else if (strcmp(state->valuestring, "stop") == 0) {
|
||
Schedule([this]() {
|
||
background_task_->WaitForCompletion();
|
||
if (device_state_ == kDeviceStateSpeaking) {
|
||
if (listening_mode_ == kListeningModeManualStop) {
|
||
SetDeviceState(kDeviceStateIdle);
|
||
} else {
|
||
SetDeviceState(kDeviceStateListening);
|
||
}
|
||
}
|
||
});
|
||
} else if (strcmp(state->valuestring, "sentence_start") == 0) {
|
||
auto text = cJSON_GetObjectItem(root, "text");
|
||
if (text != NULL) {
|
||
ESP_LOGI(TAG, "<< %s", text->valuestring);
|
||
Schedule([this, display, message = std::string(text->valuestring)]() {
|
||
display->SetChatMessage("assistant", message.c_str());
|
||
});
|
||
}
|
||
}
|
||
} else if (strcmp(type->valuestring, "stt") == 0) {
|
||
auto text = cJSON_GetObjectItem(root, "text");
|
||
if (text != NULL) {
|
||
ESP_LOGI(TAG, ">> %s", text->valuestring);
|
||
Schedule([this, display, message = std::string(text->valuestring)]() {
|
||
display->SetChatMessage("user", message.c_str());
|
||
});
|
||
}
|
||
} else if (strcmp(type->valuestring, "llm") == 0) {
|
||
auto emotion = cJSON_GetObjectItem(root, "emotion");
|
||
if (emotion != NULL) {
|
||
Schedule([this, display, emotion_str = std::string(emotion->valuestring)]() {
|
||
display->SetEmotion(emotion_str.c_str());
|
||
});
|
||
}
|
||
} else if (strcmp(type->valuestring, "iot") == 0) {
|
||
auto commands = cJSON_GetObjectItem(root, "commands");
|
||
if (commands != NULL) {
|
||
auto& thing_manager = iot::ThingManager::GetInstance();
|
||
for (int i = 0; i < cJSON_GetArraySize(commands); ++i) {
|
||
auto command = cJSON_GetArrayItem(commands, i);
|
||
thing_manager.Invoke(command);
|
||
}
|
||
}
|
||
// 新增代码(小程序控制 暂停/继续播放 音频)
|
||
// ====================================================================
|
||
}
|
||
else if (strcmp(type->valuestring, "music_control") == 0) {
|
||
auto action = cJSON_GetObjectItem(root, "action");
|
||
if (action && cJSON_IsString(action) && strcmp(action->valuestring, "pause") == 0) {
|
||
// 只有在speaking状态下才响应暂停指令
|
||
if (device_state_ == kDeviceStateSpeaking) {
|
||
ESP_LOGI(TAG, "🔇 从服务器接收到暂停播放指令 (speaking状态)");
|
||
Schedule([this]() {
|
||
PauseAudioPlayback();// 暂停播放
|
||
});
|
||
} else {
|
||
ESP_LOGI(TAG, "🔇 收到暂停指令但设备不在speaking状态，忽略指令 (当前状态: %s)", STATE_STRINGS[device_state_]);
|
||
}
|
||
} else if (action && cJSON_IsString(action) && strcmp(action->valuestring, "resume") == 0) {
|
||
// 只有在speaking状态下才响应恢复播放指令
|
||
if (device_state_ == kDeviceStateSpeaking) {
|
||
ESP_LOGI(TAG, "🔊 从服务器接收到继续播放指令 (speaking状态)");
|
||
Schedule([this]() {
|
||
ResumeAudioPlayback();// 恢复播放
|
||
});
|
||
} else {
|
||
ESP_LOGI(TAG, "🔊 收到恢复播放指令但设备不在speaking状态，忽略指令 (当前状态: %s)", STATE_STRINGS[device_state_]);
|
||
}
|
||
} else if (action && cJSON_IsString(action) && strcmp(action->valuestring, "play") == 0) {
|
||
// 处理新故事推送 - 确保在音频暂停状态和播放状态下都能正常播放
|
||
ESP_LOGI(TAG, "🎵 从服务器接收到新故事推送指令 (action: play)");
|
||
Schedule([this]() {
|
||
// 参考 AbortSpeakingAndReturnToListening 第1583-1651行的逻辑
|
||
// 检查并处理音频暂停状态，确保新故事能正常播放
|
||
if (audio_paused_) {
|
||
ESP_LOGI(TAG, "🔵 检测到音频暂停状态，为新故事推送清除暂停状态");
|
||
audio_paused_ = false;
|
||
ESP_LOGI(TAG, "✅ 音频暂停状态已清除");
|
||
|
||
// 清空音频播放队列，避免播放暂停时残留的音频
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.clear();
|
||
lock.unlock();
|
||
ESP_LOGI(TAG, "🧹 已清空音频播放队列，避免播放残留音频");
|
||
|
||
// 重新启用音频编解码器输出
|
||
auto& board = Board::GetInstance();
|
||
auto codec = board.GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(true);
|
||
ESP_LOGI(TAG, "🔧 为新故事推送重新启用音频编解码器输出");
|
||
}
|
||
}
|
||
|
||
// 如果当前在播放状态，也需要清空队列确保新故事能正常播放
|
||
if (device_state_ == kDeviceStateSpeaking) {
|
||
ESP_LOGI(TAG, "🔵 当前在播放状态，为新故事推送清空音频队列");
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.clear();
|
||
lock.unlock();
|
||
ESP_LOGI(TAG, "🧹 已清空音频播放队列，准备播放新故事");
|
||
|
||
// 确保音频编解码器输出已启用
|
||
auto& board = Board::GetInstance();
|
||
auto codec = board.GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(true);
|
||
ESP_LOGI(TAG, "🔧 确保音频编解码器输出已启用");
|
||
}
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🎵 新故事推送处理完成，音频系统已准备就绪");
|
||
});
|
||
}
|
||
}
|
||
// ====================================================================
|
||
});
|
||
protocol_->Start();
|
||
|
||
// Check for new firmware version or get the MQTT broker address
|
||
ota_.SetCheckVersionUrl(CONFIG_OTA_VERSION_URL);
|
||
ota_.SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str());
|
||
ota_.SetHeader("Client-Id", board.GetUuid());
|
||
ota_.SetHeader("Accept-Language", Lang::CODE);
|
||
auto app_desc = esp_app_get_description();
|
||
ota_.SetHeader("User-Agent", std::string(BOARD_NAME "/") + app_desc->version);
|
||
|
||
// 禁用自动OTA升级更新 - 注释掉下面的任务创建
|
||
xTaskCreate([](void* arg) {
|
||
Application* app = (Application*)arg;
|
||
app->CheckNewVersion();
|
||
vTaskDelete(NULL);
|
||
}, "check_new_version", 4096 * 2, this, 2, nullptr);
|
||
|
||
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.Initialize(codec, realtime_chat_enabled_);
|
||
|
||
// 🎯 根据语音打断功能启用状态配置VAD参数
|
||
EchoAwareVadParams enhanced_params;
|
||
if (realtime_chat_enabled_) {
|
||
// 语音打断功能启用：配置增强的回声感知参数 - 基于小智AI官方优化方案
|
||
// 🎯 平衡配置 - 防误触发同时保证音频流畅
|
||
enhanced_params.snr_threshold = 60.0f; // 平衡基础阈值：足够严格但不过度
|
||
enhanced_params.min_silence_ms = 2000; // 平衡静音要求：2秒
|
||
enhanced_params.interrupt_cooldown_ms = 10000; // 平衡冷却时间：10秒
|
||
enhanced_params.adaptive_threshold = true; // 启用自适应阈值
|
||
|
||
// 🔊 平衡噪声抑制参数 - 优化性能与效果
|
||
enhanced_params.adaptive_noise_suppression = true; // 启用自适应噪声抑制
|
||
enhanced_params.noise_suppression_base = 5.0f; // 平衡基础抑制强度
|
||
enhanced_params.volume_sensitivity = 3.0f; // 平衡音量敏感度：适度的音量影响
|
||
enhanced_params.echo_detection_threshold = 0.15f; // 平衡回声检测阈值
|
||
enhanced_params.distance_estimation_factor = 3.0f; // 平衡距离估算因子
|
||
ESP_LOGI(TAG, "🎯 Adaptive noise suppression enabled for realtime chat - smart volume/distance adjustment");
|
||
} else {
|
||
// 🔧 语音打断功能禁用：关闭复杂VAD，只使用简单VAD
|
||
enhanced_params.adaptive_threshold = false; // 禁用自适应阈值
|
||
enhanced_params.adaptive_noise_suppression = false; // 禁用自适应噪声抑制
|
||
ESP_LOGI(TAG, "🔧 Using simple VAD for basic voice detection - complex echo-aware VAD disabled");
|
||
}
|
||
audio_processor_.SetEchoAwareParams(enhanced_params);
|
||
|
||
audio_processor_.OnOutput([this](std::vector<int16_t>&& data) {
|
||
background_task_->Schedule([this, data = std::move(data)]() mutable {
|
||
opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
|
||
Schedule([this, opus = std::move(opus)]() {
|
||
protocol_->SendAudio(opus);
|
||
});
|
||
});
|
||
});
|
||
});
|
||
// 🎯 根据语音打断功能启用状态选择VAD类型
|
||
if (realtime_chat_enabled_) {
|
||
// 语音打断功能启用：使用复杂的回声感知VAD
|
||
audio_processor_.OnVadStateChange([this](bool speaking) {
|
||
ESP_LOGI(TAG, "Complex VAD state change: speaking=%s, device_state=%d, listening_mode=%d",
|
||
speaking ? "true" : "false", (int)device_state_, (int)listening_mode_);
|
||
|
||
if (device_state_ == kDeviceStateListening) {
|
||
Schedule([this, speaking]() {
|
||
if (speaking) {
|
||
voice_detected_ = true;
|
||
} else {
|
||
voice_detected_ = false;
|
||
}
|
||
auto led = Board::GetInstance().GetLed();
|
||
led->OnStateChanged();
|
||
});
|
||
}
|
||
});
|
||
}
|
||
|
||
// 🔧 简单VAD：用于普通业务（触摸忽略、LED状态等）
|
||
audio_processor_.OnSimpleVadStateChange([this](bool speaking) {
|
||
ESP_LOGI(TAG, "Simple VAD state change: speaking=%s, device_state=%d",
|
||
speaking ? "true" : "false", (int)device_state_);
|
||
|
||
if (device_state_ == kDeviceStateListening) {
|
||
Schedule([this, speaking]() {
|
||
if (speaking) {
|
||
voice_detected_ = true;
|
||
} else {
|
||
voice_detected_ = false;
|
||
}
|
||
auto led = Board::GetInstance().GetLed();
|
||
led->OnStateChanged();
|
||
});
|
||
}
|
||
|
||
// 🔊 语音打断逻辑：只在简单VAD中处理，因为复杂VAD可能过于严格
|
||
if (device_state_ == kDeviceStateSpeaking && listening_mode_ == kListeningModeRealtime) {
|
||
Schedule([this, speaking]() {
|
||
static auto speech_start_time = std::chrono::steady_clock::now();
|
||
static bool speech_confirmation_pending = false;
|
||
auto now = std::chrono::steady_clock::now();
|
||
|
||
if (speaking) {
|
||
// 小智AI方案：检测到人声开始，启动确认流程
|
||
speech_start_time = now;
|
||
speech_confirmation_pending = true;
|
||
ESP_LOGD(TAG, "Human voice detected during playback, starting interrupt evaluation");
|
||
} else if (speech_confirmation_pending) {
|
||
// 小智AI方案：人声结束，评估是否触发打断
|
||
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now - speech_start_time);
|
||
|
||
// 🎯 平衡自适应打断策略：防误触发同时保证响应性
|
||
// 基础持续时间：3秒，平衡根据干扰情况调整
|
||
int required_duration = 3000; // 基础要求3秒
|
||
|
||
// 🔊 根据当前音量动态调整持续时间要求（平衡策略）
|
||
if (current_speaker_volume_ > 0.4f) {
|
||
required_duration = 5000; // 高音量：5秒
|
||
} else if (current_speaker_volume_ > 0.1f) {
|
||
required_duration = 4000; // 中音量：4秒
|
||
}
|
||
// 低音量或静音：保持3秒
|
||
|
||
if (duration.count() >= required_duration) {
|
||
static auto last_interrupt_time = std::chrono::steady_clock::now();
|
||
auto interrupt_duration = std::chrono::duration_cast<std::chrono::milliseconds>(now - last_interrupt_time);
|
||
|
||
// 🎯 平衡自适应多重保护机制：防误触发同时保证性能
|
||
bool volume_protection = (current_speaker_volume_ > 0.01f); // 平衡音量保护：1%阈值
|
||
bool cooldown_protection = (interrupt_duration.count() <= 10000); // 平衡冷却：10秒
|
||
static int false_positive_count = 0; // 误触发计数器
|
||
|
||
// 🎯 平衡连续误触发保护：适度学习机制
|
||
if (interrupt_duration.count() <= 20000) { // 20秒内的误触发相关
|
||
false_positive_count++;
|
||
} else if (interrupt_duration.count() > 60000) { // 60秒后开始衰减
|
||
false_positive_count = std::max(false_positive_count - 1, 0); // 适度衰减
|
||
}
|
||
|
||
bool consecutive_protection = (false_positive_count >= 2); // 2次误触发后保护
|
||
|
||
if (!volume_protection && !cooldown_protection && !consecutive_protection) {
|
||
// 小智AI核心逻辑：StopPlayback -> SetDeviceState(Listening)
|
||
ESP_LOGI(TAG, "🎯 Adaptive voice interrupt triggered (duration: %.0fms/%dms, vol: %.3f) - stopping playback",
|
||
(float)duration.count(), required_duration, current_speaker_volume_);
|
||
AbortSpeaking(kAbortReasonVoiceInterrupt);
|
||
SetDeviceState(kDeviceStateListening);
|
||
last_interrupt_time = now;
|
||
} else {
|
||
ESP_LOGI(TAG, "🎯 Adaptive interrupt suppressed - vol_protection: %s (%.3f), cooldown: %.0fms, consecutive: %d",
|
||
volume_protection ? "true" : "false", current_speaker_volume_,
|
||
10000.0f - (float)interrupt_duration.count(), false_positive_count);
|
||
}
|
||
} else {
|
||
ESP_LOGI(TAG, "🎯 Voice too brief (%.0fms), likely echo or noise - adaptive threshold: %dms",
|
||
(float)duration.count(), required_duration);
|
||
}
|
||
speech_confirmation_pending = false;
|
||
}
|
||
});
|
||
}
|
||
});
|
||
#endif
|
||
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
if (!wake_word_detect_.Initialize(codec)) {
|
||
ESP_LOGE(TAG, "Failed to initialize wake word detection");
|
||
return;
|
||
}
|
||
wake_word_detect_.OnWakeWordDetected([this](const std::string& wake_word) {
|
||
Schedule([this, &wake_word]() {
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
SetDeviceState(kDeviceStateConnecting);
|
||
wake_word_detect_.EncodeWakeWordData();
|
||
// 打开音频通道并发送唤醒词数据到服务器
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
wake_word_detect_.Start();
|
||
return;
|
||
}
|
||
// 编码并发送唤醒词音频数据
|
||
std::vector<uint8_t> opus;
|
||
// Encode and send the wake word data to the server
|
||
while (wake_word_detect_.GetWakeWordOpus(opus)) {
|
||
protocol_->SendAudio(opus);
|
||
}
|
||
// Set the chat state to wake word detected
|
||
protocol_->SendWakeWordDetected(wake_word);
|
||
ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str());
|
||
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeAutoStop);
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
AbortSpeaking(kAbortReasonWakeWordDetected);
|
||
} else if (device_state_ == kDeviceStateActivating) {
|
||
SetDeviceState(kDeviceStateIdle);
|
||
}
|
||
});
|
||
});
|
||
wake_word_detect_.Start();
|
||
#endif
|
||
|
||
SetDeviceState(kDeviceStateIdle);
|
||
esp_timer_start_periodic(clock_timer_handle_, 1000000);
|
||
|
||
#if 0
|
||
while (true) {
|
||
SystemInfo::PrintRealTimeStats(pdMS_TO_TICKS(1000));
|
||
vTaskDelay(pdMS_TO_TICKS(10000));
|
||
}
|
||
#endif
|
||
}
|
||
// 时钟定时器回调函数
|
||
void Application::OnClockTimer() {
|
||
clock_ticks_++;
|
||
|
||
// Print the debug info every 10 seconds
|
||
if (clock_ticks_ % 10 == 0) {
|
||
int free_sram = heap_caps_get_free_size(MALLOC_CAP_INTERNAL);
|
||
int min_free_sram = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL);
|
||
ESP_LOGI(TAG, "Free internal: %u minimal internal: %u", free_sram, min_free_sram);
|
||
// // 打印Wi-Fi的Mac地址
|
||
// ESP_LOGI(MAC_TAG, "Wi-Fi MAC Address: %s", SystemInfo::GetMacAddress().c_str());// 生产测试打印
|
||
|
||
//ESP_LOGI(TAG, "此设备角色为: %s",CONFIG_DEVICE_ROLE);
|
||
// ESP_LOGI(TAG, "此设备角色为: KAKA 1028 升级成功!");
|
||
|
||
// If we have synchronized server time, set the status to clock "HH:MM" if the device is idle
|
||
if (ota_.HasServerTime()) {
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
Schedule([this]() {
|
||
// Set status to clock "HH:MM"
|
||
time_t now = time(NULL);
|
||
char time_str[64];
|
||
strftime(time_str, sizeof(time_str), "%H:%M ", localtime(&now));
|
||
Board::GetInstance().GetDisplay()->SetStatus(time_str);
|
||
});
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 添加任务到主循环
|
||
void Application::Schedule(std::function<void()> callback) {
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);// 加锁保护任务队列
|
||
main_tasks_.push_back(std::move(callback));// 添加任务到队列
|
||
}
|
||
xEventGroupSetBits(event_group_, SCHEDULE_EVENT);// 通知主循环有任务需要执行
|
||
}
|
||
|
||
// The Main Loop controls the chat state and websocket connection
|
||
// If other tasks need to access the websocket or chat state,
|
||
// they should use Schedule to call this function
|
||
void Application::MainLoop() {
|
||
while (true) {
|
||
auto bits = xEventGroupWaitBits(event_group_, SCHEDULE_EVENT, pdTRUE, pdFALSE, portMAX_DELAY);
|
||
|
||
if (bits & SCHEDULE_EVENT) {
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
std::list<std::function<void()>> tasks = std::move(main_tasks_);
|
||
lock.unlock();
|
||
for (auto& task : tasks) {
|
||
task();
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// The Audio Loop is used to input and output audio data
|
||
void Application::AudioLoop() {
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
while (true) {
|
||
OnAudioInput();
|
||
if (codec->output_enabled()) {
|
||
OnAudioOutput();
|
||
}
|
||
}
|
||
}
|
||
|
||
// 音频输出函数
|
||
void Application::OnAudioOutput() {
|
||
auto now = std::chrono::steady_clock::now();
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
const int max_silence_seconds = 10;
|
||
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
|
||
// 新增代码(小程序控制 暂停/继续播放 音频)
|
||
// =========================================================
|
||
// 🔧 暂停状态下停止从队列取数据，但保留队列状态
|
||
if (audio_paused_) {
|
||
// 暂停时重置音量状态，避免误判
|
||
if (current_speaker_volume_ > 0.0f) {
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
}
|
||
return;
|
||
}
|
||
// =========================================================
|
||
|
||
if (audio_decode_queue_.empty()) {
|
||
// 重要：没有音频数据时立即重置音量状态
|
||
if (current_speaker_volume_ > 0.0f) {
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
}
|
||
|
||
// Disable the output if there is no audio data for a long time
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
auto duration = std::chrono::duration_cast<std::chrono::seconds>(now - last_output_time_).count();
|
||
if (duration > max_silence_seconds) {
|
||
codec->EnableOutput(false);
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (device_state_ == kDeviceStateListening) {
|
||
audio_decode_queue_.clear();
|
||
// 重要：清空播放队列时重置音量状态，避免误判
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
return;
|
||
}
|
||
|
||
auto opus = std::move(audio_decode_queue_.front());
|
||
audio_decode_queue_.pop_front();
|
||
lock.unlock();
|
||
|
||
background_task_->Schedule([this, codec, opus = std::move(opus)]() mutable {
|
||
if (aborted_) {
|
||
return;
|
||
}
|
||
|
||
std::vector<int16_t> pcm;
|
||
if (!opus_decoder_->Decode(std::move(opus), pcm)) {
|
||
return;
|
||
}
|
||
// Resample if the sample rate is different
|
||
if (opus_decoder_->sample_rate() != codec->output_sample_rate()) {
|
||
int target_size = output_resampler_.GetOutputSamples(pcm.size());
|
||
std::vector<int16_t> resampled(target_size);
|
||
output_resampler_.Process(pcm.data(), pcm.size(), resampled.data());
|
||
pcm = std::move(resampled);
|
||
}
|
||
// 计算并同步音频输出音量到音频处理器，用于动态VAD阈值调整
|
||
if (realtime_chat_enabled_ && !pcm.empty()) {
|
||
// 计算音频块的RMS音量 (0.0 - 1.0)
|
||
float sum_squares = 0.0f;
|
||
for (const auto& sample : pcm) {
|
||
float normalized = (float)sample / 32768.0f;
|
||
sum_squares += normalized * normalized;
|
||
}
|
||
float rms_volume = std::sqrt(sum_squares / pcm.size());
|
||
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
// 同步音量到音频处理器，用于动态阈值调整
|
||
current_speaker_volume_ = rms_volume; // 保存当前音量供打断逻辑使用
|
||
audio_processor_.SetSpeakerVolume(rms_volume);
|
||
#endif
|
||
}
|
||
|
||
codec->OutputData(pcm);
|
||
last_output_time_ = std::chrono::steady_clock::now();
|
||
});
|
||
}
|
||
|
||
void Application::OnAudioInput() {
|
||
std::vector<int16_t> data;
|
||
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
if (wake_word_detect_.IsRunning()) {
|
||
ReadAudio(data, 16000, wake_word_detect_.GetFeedSize());
|
||
wake_word_detect_.Feed(data);// 将音频数据喂给唤醒词检测器
|
||
return;
|
||
}
|
||
#endif
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (audio_processor_.IsRunning()) {
|
||
ReadAudio(data, 16000, audio_processor_.GetFeedSize());
|
||
audio_processor_.Feed(data);
|
||
return;
|
||
}
|
||
#else
|
||
if (device_state_ == kDeviceStateListening) {
|
||
ReadAudio(data, 16000, 30 * 16000 / 1000);
|
||
background_task_->Schedule([this, data = std::move(data)]() mutable {
|
||
opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
|
||
Schedule([this, opus = std::move(opus)]() {
|
||
protocol_->SendAudio(opus);
|
||
});
|
||
});
|
||
});
|
||
return;
|
||
}
|
||
#endif
|
||
vTaskDelay(pdMS_TO_TICKS(30));
|
||
}
|
||
|
||
void Application::ReadAudio(std::vector<int16_t>& data, int sample_rate, int samples) {
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
if (codec->input_sample_rate() != sample_rate) {
|
||
data.resize(samples * codec->input_sample_rate() / sample_rate);
|
||
if (!codec->InputData(data)) {
|
||
return;
|
||
}
|
||
if (codec->input_channels() == 2) {
|
||
auto mic_channel = std::vector<int16_t>(data.size() / 2);
|
||
auto reference_channel = std::vector<int16_t>(data.size() / 2);
|
||
for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) {
|
||
mic_channel[i] = data[j];
|
||
reference_channel[i] = data[j + 1];
|
||
}
|
||
auto resampled_mic = std::vector<int16_t>(input_resampler_.GetOutputSamples(mic_channel.size()));
|
||
auto resampled_reference = std::vector<int16_t>(reference_resampler_.GetOutputSamples(reference_channel.size()));
|
||
input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data());
|
||
reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data());
|
||
data.resize(resampled_mic.size() + resampled_reference.size());
|
||
for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) {
|
||
data[j] = resampled_mic[i];
|
||
data[j + 1] = resampled_reference[i];
|
||
}
|
||
} else {
|
||
auto resampled = std::vector<int16_t>(input_resampler_.GetOutputSamples(data.size()));
|
||
input_resampler_.Process(data.data(), data.size(), resampled.data());
|
||
data = std::move(resampled);
|
||
}
|
||
} else {
|
||
data.resize(samples);
|
||
if (!codec->InputData(data)) {
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
// 打断语音播报(终止播放)
|
||
void Application::AbortSpeaking(AbortReason reason) {
|
||
// 🔧 防止重复中止操作，避免竞态条件
|
||
bool expected = false;
|
||
if (!is_aborting_.compare_exchange_strong(expected, true)) {
|
||
ESP_LOGD(TAG, "AbortSpeaking already in progress, ignoring duplicate call");
|
||
return;
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🔴 Abort speaking - immediate stop");
|
||
aborted_ = true;
|
||
|
||
// 🔧 更新安全操作时间戳
|
||
last_safe_operation_.store(std::chrono::steady_clock::now());
|
||
|
||
// 🔧 修复：立即清空音频队列，确保音频播放立即停止
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
if (!audio_decode_queue_.empty()) {
|
||
ESP_LOGI(TAG, "🔴 Clearing %zu audio frames from queue", audio_decode_queue_.size());
|
||
audio_decode_queue_.clear();
|
||
|
||
// 重置音量状态
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
}
|
||
}
|
||
|
||
// ⚠️ 移除WaitForCompletion避免死锁，让后台任务通过aborted_标志自然结束
|
||
ESP_LOGI(TAG, "🔴 Audio queue cleared, background tasks will stop on next iteration");
|
||
|
||
// 🔧 修复：安全地发送中止消息，避免WebSocket崩溃
|
||
if (protocol_ && device_state_ == kDeviceStateSpeaking && IsSafeToOperate()) {
|
||
try {
|
||
// 🔧 额外的连接状态验证
|
||
if (protocol_->IsAudioChannelOpened()) {
|
||
protocol_->SendAbortSpeaking(reason);
|
||
ESP_LOGI(TAG, "AbortSpeaking message sent successfully");
|
||
// 更新安全操作时间戳
|
||
last_safe_operation_.store(std::chrono::steady_clock::now());
|
||
} else {
|
||
ESP_LOGW(TAG, "Audio channel not properly opened, skipping AbortSpeaking");
|
||
}
|
||
} catch (const std::exception& e) {
|
||
ESP_LOGW(TAG, "Failed to send AbortSpeaking message: %s", e.what());
|
||
}
|
||
} else {
|
||
ESP_LOGD(TAG, "Skipping AbortSpeaking message - conditions not safe");
|
||
}
|
||
|
||
// 🔧 重置中止标志，允许后续操作
|
||
is_aborting_.store(false);
|
||
}
|
||
|
||
// 发送讲故事请求 【新增】
|
||
void Application::SendStoryRequest() {
|
||
if (!protocol_) {
|
||
ESP_LOGE(TAG, "Protocol not initialized");
|
||
return;
|
||
}
|
||
|
||
if (device_state_ == kDeviceStateIdle) { // 设备状态为待机
|
||
Schedule([this]() {
|
||
SetDeviceState(kDeviceStateConnecting);
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
return;
|
||
}
|
||
protocol_->SendStoryRequest(); // 发送讲故事请求
|
||
ESP_LOGI(TAG, "Sent story request");
|
||
|
||
// 立即启动监听模式以接收语音回复
|
||
ESP_LOGI(TAG, "SendStoryRequest: realtime_chat_enabled_ = %s", realtime_chat_enabled_ ? "true" : "false");
|
||
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeManualStop);
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) { // 设备状态为说话
|
||
Schedule([this]() {
|
||
AbortSpeaking(kAbortReasonNone);
|
||
protocol_->SendStoryRequest(); // 发送讲故事请求
|
||
ESP_LOGI(TAG, "Sent story request");
|
||
|
||
// 启动监听模式以接收语音回复
|
||
ESP_LOGI(TAG, "SendStoryRequest: realtime_chat_enabled_ = %s", realtime_chat_enabled_ ? "true" : "false");
|
||
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeManualStop);
|
||
});
|
||
} else if (device_state_ == kDeviceStateListening) { // 设备状态为监听
|
||
Schedule([this]() {
|
||
protocol_->SendStoryRequest(); // 发送讲故事请求（调用协议层）
|
||
ESP_LOGI(TAG, "Sent story request");
|
||
});
|
||
}
|
||
}
|
||
|
||
void Application::SetListeningMode(ListeningMode mode) {
|
||
ESP_LOGI(TAG, "Setting listening mode to %d", (int)mode);
|
||
listening_mode_ = mode;
|
||
SetDeviceState(kDeviceStateListening);
|
||
}
|
||
|
||
void Application::SetDeviceState(DeviceState state) {
|
||
if (device_state_ == state) {
|
||
return;
|
||
}
|
||
|
||
clock_ticks_ = 0;
|
||
auto previous_state = device_state_;
|
||
device_state_ = state;
|
||
ESP_LOGI(TAG, "STATE: %s", STATE_STRINGS[device_state_]);
|
||
// The state is changed, wait for all background tasks to finish
|
||
background_task_->WaitForCompletion();
|
||
|
||
auto& board = Board::GetInstance();
|
||
auto display = board.GetDisplay();
|
||
auto led = board.GetLed();
|
||
led->OnStateChanged();
|
||
|
||
// 检查是否正在进行BluFi配网，配网时禁止播放待命音效(新增代码)
|
||
// =================================================================
|
||
bool is_blufi_provisioning = false;
|
||
if (Board::GetInstance().GetBoardType() == "wifi") {
|
||
auto& wifi_board = static_cast<WifiBoard&>(Board::GetInstance());
|
||
is_blufi_provisioning = wifi_board.IsBluFiProvisioningActive();
|
||
}
|
||
// =================================================================
|
||
|
||
switch (state) {
|
||
case kDeviceStateUnknown:
|
||
case kDeviceStateIdle:
|
||
display->SetStatus(Lang::Strings::STANDBY);
|
||
display->SetEmotion("neutral");
|
||
|
||
|
||
// // 只有从非待命状态进入待命状态时才播放待命音效，避免重复播放(原来的代码)
|
||
// if (previous_state != kDeviceStateIdle &&
|
||
// previous_state != kDeviceStateUnknown &&
|
||
// previous_state != kDeviceStateWifiConfiguring) {
|
||
// ESP_LOGI(TAG, "Entering idle state, playing standby sound");
|
||
// PlaySound(Lang::Sounds::P3_DAIMING);
|
||
// }
|
||
// 开机后 进入待命状态 播报 卡卡正在待命(配网模式下不播报“卡卡正在待命”)-新增代码
|
||
//=====================================================================================
|
||
if (previous_state != kDeviceStateIdle && previous_state != kDeviceStateUnknown &&
|
||
previous_state != kDeviceStateWifiConfiguring && !is_blufi_provisioning && !IsLowBatteryTransition()) {
|
||
ESP_LOGI(TAG, "Entering idle state, playing standby sound");
|
||
// PlaySound(Lang::Sounds::P3_DAIMING); 原有 待命 播报
|
||
if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){
|
||
PlaySound(Lang::Sounds::P3_KAKA_DAIMING);
|
||
}
|
||
else if(strcmp(CONFIG_DEVICE_ROLE, "LALA") == 0){
|
||
PlaySound(Lang::Sounds::P3_LALA_DAIMING);
|
||
}
|
||
}
|
||
//=====================================================================================
|
||
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.Stop();
|
||
#endif
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Start();
|
||
#endif
|
||
break;
|
||
case kDeviceStateConnecting:
|
||
display->SetStatus(Lang::Strings::CONNECTING);
|
||
display->SetEmotion("neutral");
|
||
display->SetChatMessage("system", "");
|
||
break;
|
||
case kDeviceStateListening:
|
||
display->SetStatus(Lang::Strings::LISTENING);
|
||
display->SetEmotion("neutral");
|
||
|
||
// 关键修复：只有在非音效播放状态下才重置音量，避免中断正在播放的音效
|
||
// 检查是否有音频正在播放，如果有则延迟重置音量
|
||
if (IsAudioQueueEmpty()) {
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
} else {
|
||
// 如果有音频正在播放，延迟重置音量
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(500)); // 等待音效播放完成
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
});
|
||
}
|
||
|
||
// Update the IoT states before sending the start listening command
|
||
UpdateIotStates();
|
||
|
||
// Make sure the audio processor is running
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (!audio_processor_.IsRunning()) {
|
||
#else
|
||
if (true) {
|
||
#endif
|
||
// 🔧 关键修复：检查协议连接状态，防止发送到无效连接
|
||
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
||
// Send the start listening command
|
||
protocol_->SendStartListening(listening_mode_);
|
||
if (listening_mode_ == kListeningModeAutoStop && previous_state == kDeviceStateSpeaking) {
|
||
// FIXME: Wait for the speaker to empty the buffer
|
||
vTaskDelay(pdMS_TO_TICKS(120));
|
||
}
|
||
opus_encoder_->ResetState();
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Stop();
|
||
#endif
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.Start();
|
||
#endif
|
||
} else {
|
||
ESP_LOGW(TAG, "Audio channel not available, skipping SendStartListening");
|
||
// 保持在聆听状态，不自动回退到idle状态
|
||
ESP_LOGI(TAG, "🔵 Staying in listening state despite audio channel unavailable");
|
||
}
|
||
}
|
||
break;
|
||
case kDeviceStateSpeaking:
|
||
display->SetStatus(Lang::Strings::SPEAKING);
|
||
|
||
if (listening_mode_ != kListeningModeRealtime) {
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.Stop();
|
||
#endif
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Start();
|
||
#endif
|
||
} else {
|
||
// 在实时模式下，保持audio_processor运行以检测语音打断
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (!audio_processor_.IsRunning()) {
|
||
audio_processor_.Start();
|
||
}
|
||
#endif
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Stop();
|
||
#endif
|
||
}
|
||
ResetDecoder();
|
||
break;
|
||
default:
|
||
// Do nothing
|
||
break;
|
||
}
|
||
}
|
||
|
||
void Application::ResetDecoder() {
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
opus_decoder_->ResetState();
|
||
audio_decode_queue_.clear();
|
||
last_output_time_ = std::chrono::steady_clock::now();
|
||
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
codec->EnableOutput(true);
|
||
}
|
||
|
||
void Application::SetDecodeSampleRate(int sample_rate, int frame_duration) {
|
||
if (opus_decoder_->sample_rate() == sample_rate && opus_decoder_->duration_ms() == frame_duration) {
|
||
return;
|
||
}
|
||
|
||
opus_decoder_.reset();
|
||
opus_decoder_ = std::make_unique<OpusDecoderWrapper>(sample_rate, 1, frame_duration);
|
||
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
if (opus_decoder_->sample_rate() != codec->output_sample_rate()) {
|
||
ESP_LOGI(TAG, "Resampling audio from %d to %d", opus_decoder_->sample_rate(), codec->output_sample_rate());
|
||
output_resampler_.Configure(opus_decoder_->sample_rate(), codec->output_sample_rate());
|
||
}
|
||
}
|
||
|
||
void Application::UpdateIotStates() {
|
||
auto& thing_manager = iot::ThingManager::GetInstance();
|
||
std::string states;
|
||
if (thing_manager.GetStatesJson(states, true)) {
|
||
protocol_->SendIotStates(states);
|
||
}
|
||
}
|
||
|
||
void Application::Reboot() {
|
||
ESP_LOGI(TAG, "Rebooting...");
|
||
esp_restart();
|
||
}
|
||
|
||
// 唤醒词触发函数
|
||
void Application::WakeWordInvoke(const std::string& wake_word) {
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
ToggleChatState();
|
||
Schedule([this, wake_word]() {
|
||
if (protocol_) {
|
||
protocol_->SendWakeWordDetected(wake_word);
|
||
}
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
//AbortSpeakingAndReturnToListening();// 使用唤醒词打断时立即切换到聆听状态
|
||
Schedule([this]() {
|
||
AbortSpeaking(kAbortReasonNone);
|
||
});
|
||
} else if (device_state_ == kDeviceStateListening) {
|
||
Schedule([this]() {
|
||
if (protocol_) {
|
||
protocol_->CloseAudioChannel();
|
||
}
|
||
});
|
||
}
|
||
}
|
||
|
||
bool Application::CanEnterSleepMode() {
|
||
if (device_state_ != kDeviceStateIdle) {
|
||
return false;
|
||
}
|
||
|
||
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
||
return false;
|
||
}
|
||
|
||
// Now it is safe to enter sleep mode
|
||
return true;
|
||
}
|
||
void Application::WaitForAudioPlayback() {
|
||
// 等待 audio_decode_queue_ 清空且音频输出完成
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
int timeout_count = 0;
|
||
const int max_timeout = 150; // 3秒超时 (150 * 20ms = 3000ms)
|
||
|
||
while (timeout_count < max_timeout) {
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
if (audio_decode_queue_.empty()) {
|
||
// 检查音频输出是否已关闭或静音
|
||
if (!codec->output_enabled() || device_state_ != kDeviceStateSpeaking) {
|
||
ESP_LOGI(TAG, "Audio playback completed");
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
vTaskDelay(pdMS_TO_TICKS(20));
|
||
timeout_count++;
|
||
}
|
||
|
||
if (timeout_count >= max_timeout) {
|
||
ESP_LOGW(TAG, "WaitForAudioPlayback timeout after 3 seconds");
|
||
}
|
||
}
|
||
|
||
bool Application::IsAudioQueueEmpty() {
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
return audio_decode_queue_.empty();
|
||
}
|
||
|
||
void Application::ClearAudioQueue() {
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.clear();
|
||
audio_paused_ = false; // 清除暂停状态
|
||
// ESP_LOGI(TAG, "🧹 音频播放队列已清空，暂停状态已清除");
|
||
ESP_LOGI(TAG, "🎵 测试模式：音频开始播放，等待播放完成"); // 生产测试打印
|
||
|
||
|
||
// 重新启用音频编解码器输出，确保后续音频能正常播放
|
||
auto& board = Board::GetInstance();
|
||
auto codec = board.GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(true);
|
||
// ESP_LOGI(TAG, "🔧 音频编解码器输出已重新启用");
|
||
ESP_LOGI(TAG, "✅ 测试模式：音频播放完成"); // 生产测试打印
|
||
}
|
||
}
|
||
|
||
// 🔧 检查当前是否可以安全执行操作
|
||
bool Application::IsSafeToOperate() {
|
||
// 检查是否正在执行中止操作
|
||
if (is_aborting_.load()) {
|
||
return false;
|
||
}
|
||
|
||
// 检查最近是否有操作过于频繁
|
||
auto now = std::chrono::steady_clock::now();
|
||
auto last_op = last_safe_operation_.load();
|
||
auto time_diff = std::chrono::duration_cast<std::chrono::milliseconds>(now - last_op);
|
||
|
||
// 如果距离上次操作少于50ms，认为可能存在竞态风险
|
||
if (time_diff.count() < 50) {
|
||
ESP_LOGD(TAG, "Operation too frequent, waiting for safety");
|
||
return false;
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
void Application::StopAudioProcessor() {
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.Stop();
|
||
#endif
|
||
}
|
||
|
||
// 🔴 专门处理从说话状态到空闲状态的切换
|
||
void Application::AbortSpeakingAndReturnToIdle() {
|
||
ESP_LOGI(TAG, "🔴 AbortSpeakingAndReturnToIdle: Starting transition from speaking to idle state");
|
||
ESP_LOGI(TAG, "📊 当前设备状态: %s", STATE_STRINGS[device_state_]);
|
||
ESP_LOGI(TAG, "🎯 目标状态: idle (空闲状态)");
|
||
|
||
// 检查当前状态是否为说话状态
|
||
if (device_state_ != kDeviceStateSpeaking) {
|
||
ESP_LOGW(TAG, "🔴 AbortSpeakingAndReturnToIdle: Device not in speaking state, current state: %s", STATE_STRINGS[device_state_]);
|
||
return;
|
||
}
|
||
|
||
ESP_LOGI(TAG, "✅ 状态检查通过，当前处于说话状态");
|
||
|
||
// 检查操作安全性
|
||
if (!IsSafeToOperate()) {
|
||
ESP_LOGW(TAG, "🔴 AbortSpeakingAndReturnToIdle: Operation not safe, scheduling retry");
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
AbortSpeakingAndReturnToIdle();
|
||
});
|
||
return;
|
||
}
|
||
|
||
// 更新安全操作时间戳
|
||
last_safe_operation_.store(std::chrono::steady_clock::now());
|
||
ESP_LOGI(TAG, "⏰ 安全操作时间戳已更新");
|
||
|
||
// 立即停止音频处理
|
||
ESP_LOGI(TAG, "🔇 开始停止音频处理");
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
if (!audio_decode_queue_.empty()) {
|
||
ESP_LOGI(TAG, "🗑️ 清空音频队列，当前队列大小: %zu", audio_decode_queue_.size());
|
||
audio_decode_queue_.clear();
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
ESP_LOGI(TAG, "✅ 音频队列已清空，音量已重置为0");
|
||
} else {
|
||
ESP_LOGI(TAG, "ℹ️ 音频队列已为空，无需清空");
|
||
}
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🔴 AbortSpeakingAndReturnToIdle: Sending abort message to server");
|
||
|
||
// 发送中止消息给服务器
|
||
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
||
ESP_LOGI(TAG, "📡 WebSocket连接正常，发送中止消息");
|
||
try {
|
||
protocol_->SendAbortSpeaking(kAbortReasonNone);
|
||
ESP_LOGI(TAG, "✅ 中止消息发送成功");
|
||
} catch (const std::exception& e) {
|
||
ESP_LOGW(TAG, "❌ 发送中止消息失败: %s", e.what());
|
||
}
|
||
|
||
// 延迟100ms后主动关闭连接，确保服务器有时间处理中止消息
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
ESP_LOGI(TAG, "⏳ 延迟100ms后开始主动关闭WebSocket连接");
|
||
ESP_LOGI(TAG, "🔌 执行主动断开WebSocket连接");
|
||
if (protocol_) {
|
||
protocol_->CloseAudioChannel();
|
||
ESP_LOGI(TAG, "✅ CloseAudioChannel调用完成");
|
||
} else {
|
||
ESP_LOGW(TAG, "⚠️ protocol_为空，无法关闭音频通道");
|
||
}
|
||
});
|
||
} else {
|
||
ESP_LOGW(TAG, "⚠️ WebSocket连接不可用，强制关闭连接");
|
||
if (protocol_) {
|
||
ESP_LOGI(TAG, "🔌 强制执行WebSocket断开");
|
||
protocol_->CloseAudioChannel();
|
||
ESP_LOGI(TAG, "✅ 强制断开完成");
|
||
} else {
|
||
ESP_LOGW(TAG, "❌ protocol_为空，无法执行断开操作");
|
||
}
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🎯 主动断开流程已启动，等待OnAudioChannelClosed回调触发状态转换");
|
||
ESP_LOGI(TAG, "📋 预期流程: WebSocket断开 → 回调触发 → 转换到idle状态 → 播放待机音");
|
||
}
|
||
|
||
// 🔵 专门处理从说话状态到聆听状态的切换
|
||
void Application::AbortSpeakingAndReturnToListening() {
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Starting transition from speaking to listening state (断开连接方案)");
|
||
|
||
// 检查当前状态是否为说话状态或可切换状态
|
||
// =========================================================================================
|
||
if (device_state_ != kDeviceStateSpeaking && device_state_ != kDeviceStateListening && device_state_ != kDeviceStateIdle) {
|
||
ESP_LOGW(TAG, "🔵 AbortSpeakingAndReturnToListening: Device not in valid state for transition, current state: %s", STATE_STRINGS[device_state_]);
|
||
return;
|
||
}
|
||
// 如果已经在listening状态，直接返回避免重复切换
|
||
if (device_state_ == kDeviceStateListening) {
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Already in listening state, skipping transition");
|
||
return;
|
||
}
|
||
// 🔧 检查并处理音频播放状态(BOOT按键优化方案)
|
||
if (!audio_paused_ && device_state_ == kDeviceStateSpeaking) {
|
||
ESP_LOGI(TAG, "🔵 检测到播放状态，一次按键完成暂停和状态切换");
|
||
|
||
// 第一步：禁用音频输出(立即停止播放)
|
||
auto& board = Board::GetInstance();// 获取音频编解码器
|
||
auto codec = board.GetAudioCodec();// 获取音频编解码器
|
||
if (codec) {
|
||
codec->EnableOutput(false);// 暂停时禁用音频编解码器输出
|
||
ESP_LOGI(TAG, "🔧 暂停时禁用音频编解码器输出");
|
||
}
|
||
// 第二步：切换到暂停状态
|
||
audio_paused_ = true;
|
||
ESP_LOGI(TAG, "✅ 已切换到暂停状态");
|
||
// 第三步：立即执行状态切换逻辑（不返回，继续执行下面的代码）
|
||
ESP_LOGI(TAG, "🔵 继续执行状态切换到聆听状态");
|
||
}
|
||
|
||
// 🔧 检查并处理音频暂停状态(BOOT按键优化方案)
|
||
if (audio_paused_) {
|
||
ESP_LOGI(TAG, "🔵 检测到音频暂停状态，应用BOOT按键优化方案");
|
||
audio_paused_ = false;
|
||
ESP_LOGI(TAG, "✅ 音频暂停状态已清除");
|
||
|
||
// 🔧 关键优化：清空音频播放队列，避免播放暂停时残留的音频
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.clear();
|
||
lock.unlock();
|
||
ESP_LOGI(TAG, "🧹 已清空音频播放队列，避免播放残留音频");
|
||
|
||
// BOOT按键切换时的优化方案：确保音频系统能正常响应状态切换
|
||
auto& board = Board::GetInstance();
|
||
auto codec = board.GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(true);
|
||
ESP_LOGI(TAG, "🔧 为状态切换重新启用音频编解码器输出");// 重新启用输出,后续可以播放
|
||
}
|
||
// 🔧 关键修复：强制停止音频处理器，确保后续状态切换时能重新启动
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (audio_processor_.IsRunning()) {
|
||
ESP_LOGI(TAG, "🔧 强制停止音频处理器以确保状态切换成功");
|
||
audio_processor_.Stop();
|
||
}
|
||
#endif
|
||
|
||
// 🔧 音频暂停状态下直接切换，避免复杂的异步处理
|
||
ESP_LOGI(TAG, "🔵 音频暂停状态下直接执行状态切换");
|
||
|
||
// 播放提示音
|
||
if (codec && codec->output_enabled()) {
|
||
ESP_LOGI(TAG, "播放提示音：卡卡在呢");
|
||
// PlaySound(Lang::Sounds::P3_KAKAZAINNE); 原有蜡笔小新 音色播报
|
||
if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){
|
||
PlaySound(Lang::Sounds::P3_KAKA_ZAINNE);
|
||
}
|
||
else if(strcmp(CONFIG_DEVICE_ROLE, "LALA") == 0){
|
||
PlaySound(Lang::Sounds::P3_LALA_ZAINNE);
|
||
}
|
||
|
||
|
||
// 简化等待逻辑
|
||
vTaskDelay(pdMS_TO_TICKS(620)); // 等待音效播放完成
|
||
ESP_LOGI(TAG, "音频播放完成");
|
||
}
|
||
|
||
// 直接切换到聆听状态
|
||
SetDeviceState(kDeviceStateListening);
|
||
ESP_LOGI(TAG, "🔵 音频暂停状态下状态切换完成");
|
||
return;
|
||
}
|
||
// =========================================================================================
|
||
|
||
// 检查操作安全性
|
||
if (!IsSafeToOperate()) {
|
||
ESP_LOGW(TAG, "🔵 AbortSpeakingAndReturnToListening: Operation not safe, scheduling retry");
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
AbortSpeakingAndReturnToListening();
|
||
});
|
||
return;
|
||
}
|
||
|
||
// 更新安全操作时间戳
|
||
last_safe_operation_.store(std::chrono::steady_clock::now());
|
||
|
||
// 立即停止音频处理器和清空音频队列
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (audio_processor_.IsRunning()) {
|
||
ESP_LOGI(TAG, "🔵 停止音频处理器");
|
||
audio_processor_.Stop();
|
||
}
|
||
|
||
// 清空音频队列并重置音量
|
||
if (!IsAudioQueueEmpty()) {
|
||
ESP_LOGI(TAG, "🔵 清空音频队列并重置音量");
|
||
while (!IsAudioQueueEmpty()) {
|
||
vTaskDelay(pdMS_TO_TICKS(10));
|
||
}
|
||
current_speaker_volume_ = 0.0f;
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
ESP_LOGI(TAG, "✅ 音频队列已清空，音量已重置为0");
|
||
} else {
|
||
ESP_LOGI(TAG, "ℹ️ 音频队列已为空，无需清空");
|
||
}
|
||
#endif
|
||
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Sending abort message to server");
|
||
|
||
// 发送中止消息给服务器
|
||
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
||
ESP_LOGI(TAG, "📡 WebSocket连接正常，发送中止消息");
|
||
try {
|
||
protocol_->SendAbortSpeaking(kAbortReasonVoiceInterrupt);
|
||
ESP_LOGI(TAG, "✅ 中止消息发送成功");
|
||
} catch (const std::exception& e) {
|
||
ESP_LOGW(TAG, "❌ 发送中止消息失败: %s", e.what());
|
||
}
|
||
|
||
// 延迟100ms后播放音效并直接切换到聆听状态，不关闭WebSocket连接
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
ESP_LOGI(TAG, "⏳ 延迟100ms后播放音效并切换到聆听状态");
|
||
|
||
// 先播放"卡卡在呢"音效
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Playing KAKAZAINNE sound");
|
||
|
||
// 🔧 修复：强制重新初始化音频输出，确保硬件状态正确
|
||
auto& board = Board::GetInstance();
|
||
auto audio_codec = board.GetAudioCodec();
|
||
ESP_LOGI(TAG, "强制重新初始化音频输出");
|
||
audio_codec->EnableOutput(false); // 先关闭音频输出
|
||
vTaskDelay(pdMS_TO_TICKS(50)); // 短暂延迟让硬件复位
|
||
audio_codec->EnableOutput(true); // 再开启，强制硬件重新初始化
|
||
|
||
// 🔧 检查音频资源是否可用
|
||
if (audio_codec->output_enabled()) {
|
||
ESP_LOGI(TAG, "播放提示音：卡卡在呢");
|
||
ResetDecoder(); // 🔧 关键修复：重置解码器状态，清除残留
|
||
|
||
// 获取当前系统音量并临时设置以确保音效能播放
|
||
float system_volume = audio_codec ? (audio_codec->output_volume() / 100.0f) : 0.7f; // 默认70%
|
||
current_speaker_volume_ = system_volume;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(system_volume);
|
||
ESP_LOGI(TAG, "✅ 音量设置成功: %.2f", system_volume);
|
||
#endif
|
||
|
||
// PlaySound(Lang::Sounds::P3_KAKAZAINNE); 原有蜡笔小新 音色播报
|
||
if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){
|
||
PlaySound(Lang::Sounds::P3_KAKA_ZAINNE);
|
||
}
|
||
else if(strcmp(CONFIG_DEVICE_ROLE, "LALA") == 0){
|
||
PlaySound(Lang::Sounds::P3_LALA_ZAINNE);
|
||
}
|
||
|
||
// 🔧 修复：使用改进的等待逻辑，确保音频真正播放完成
|
||
ESP_LOGI(TAG, "等待音频播放完成...");
|
||
vTaskDelay(pdMS_TO_TICKS(100)); // 给音频足够的时间开始播放
|
||
|
||
// 等待音频队列清空 + 额外缓冲时间确保I2S硬件完成输出
|
||
int timeout_count = 0;
|
||
const int max_timeout = 150; // 3秒超时
|
||
|
||
while (timeout_count < max_timeout) {
|
||
if (IsAudioQueueEmpty()) {
|
||
// 队列清空后，再等待500ms确保I2S硬件完成输出
|
||
ESP_LOGI(TAG, "音频队列已清空，等待硬件输出完成...");
|
||
vTaskDelay(pdMS_TO_TICKS(500));
|
||
ESP_LOGI(TAG, "音频播放完成");
|
||
break;
|
||
}
|
||
vTaskDelay(pdMS_TO_TICKS(20));
|
||
timeout_count++;
|
||
}
|
||
|
||
if (timeout_count >= max_timeout) {
|
||
ESP_LOGW(TAG, "等待音频播放超时，继续状态切换");
|
||
}
|
||
} else {
|
||
ESP_LOGW(TAG, "音频输出无法启用，跳过提示音播放");
|
||
}
|
||
|
||
// 直接切换到聆听状态，音频播放已在上面完成
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Switching to listening state (保持WebSocket连接)");
|
||
SetDeviceState(kDeviceStateListening);
|
||
});
|
||
} else {
|
||
ESP_LOGW(TAG, "⚠️ WebSocket连接不可用，直接切换状态");
|
||
|
||
// 直接播放音效并切换状态
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Playing KAKAZAINNE sound");
|
||
|
||
// 🔧 修复：强制重新初始化音频输出，确保硬件状态正确
|
||
auto& board = Board::GetInstance();
|
||
auto audio_codec = board.GetAudioCodec();
|
||
ESP_LOGI(TAG, "强制重新初始化音频输出");
|
||
audio_codec->EnableOutput(false); // 先关闭音频输出
|
||
vTaskDelay(pdMS_TO_TICKS(50)); // 短暂延迟让硬件复位
|
||
audio_codec->EnableOutput(true); // 再开启，强制硬件重新初始化
|
||
|
||
// 🔧 检查音频资源是否可用
|
||
if (audio_codec->output_enabled()) {
|
||
ESP_LOGI(TAG, "播放提示音：卡卡在呢");
|
||
ResetDecoder(); // 🔧 关键修复：重置解码器状态，清除残留
|
||
|
||
// 获取当前系统音量并临时设置以确保音效能播放
|
||
float system_volume = audio_codec ? (audio_codec->output_volume() / 100.0f) : 0.7f; // 默认70%
|
||
current_speaker_volume_ = system_volume;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(system_volume);
|
||
ESP_LOGI(TAG, "✅ 音量设置成功: %.2f", system_volume);
|
||
#endif
|
||
|
||
// PlaySound(Lang::Sounds::P3_KAKAZAINNE); 原有蜡笔小新 音色播报
|
||
if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){
|
||
PlaySound(Lang::Sounds::P3_KAKA_ZAINNE);
|
||
}
|
||
else if(strcmp(CONFIG_DEVICE_ROLE, "LALA") == 0){
|
||
PlaySound(Lang::Sounds::P3_LALA_ZAINNE);
|
||
}
|
||
|
||
// 🔧 修复：使用改进的等待逻辑，确保音频真正播放完成
|
||
ESP_LOGI(TAG, "等待音频播放完成...");
|
||
vTaskDelay(pdMS_TO_TICKS(100)); // 给音频足够的时间开始播放
|
||
|
||
// 等待音频队列清空 + 额外缓冲时间确保I2S硬件完成输出
|
||
int timeout_count = 0;
|
||
const int max_timeout = 150; // 3秒超时
|
||
|
||
while (timeout_count < max_timeout) {
|
||
if (IsAudioQueueEmpty()) {
|
||
// 队列清空后，再等待500ms确保I2S硬件完成输出
|
||
ESP_LOGI(TAG, "音频队列已清空，等待硬件输出完成...");
|
||
vTaskDelay(pdMS_TO_TICKS(500));
|
||
ESP_LOGI(TAG, "音频播放完成");
|
||
break;
|
||
}
|
||
vTaskDelay(pdMS_TO_TICKS(20));
|
||
timeout_count++;
|
||
}
|
||
|
||
if (timeout_count >= max_timeout) {
|
||
ESP_LOGW(TAG, "等待音频播放超时，继续状态切换");
|
||
}
|
||
} else {
|
||
ESP_LOGW(TAG, "音频输出无法启用，跳过提示音播放");
|
||
}
|
||
|
||
// 直接切换到聆听状态，音频播放已在上面完成
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Switching to listening state");
|
||
SetDeviceState(kDeviceStateListening);
|
||
});
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Transition initiated - keeping WebSocket connection and switching to listening");
|
||
}
|
||
|
||
// 姿态传感器接口实现
|
||
bool Application::IsImuSensorAvailable() {
|
||
auto& board = Board::GetInstance();
|
||
if (board.GetBoardType() == "movecall-moji-esp32s3") {
|
||
auto& moji_board = static_cast<MovecallMojiESP32S3&>(board);
|
||
return moji_board.IsImuInitialized();
|
||
}
|
||
return false;
|
||
}
|
||
|
||
bool Application::GetImuData(float* acc_x, float* acc_y, float* acc_z,
|
||
float* gyro_x, float* gyro_y, float* gyro_z,
|
||
float* temperature) {
|
||
auto& board = Board::GetInstance();
|
||
if (board.GetBoardType() == "movecall-moji-esp32s3") {
|
||
auto& moji_board = static_cast<MovecallMojiESP32S3&>(board);
|
||
qmi8658a_data_t imu_data;
|
||
if (moji_board.GetImuData(&imu_data)) {
|
||
if (acc_x) *acc_x = imu_data.acc_x;
|
||
if (acc_y) *acc_y = imu_data.acc_y;
|
||
if (acc_z) *acc_z = imu_data.acc_z;
|
||
if (gyro_x) *gyro_x = imu_data.gyro_x;
|
||
if (gyro_y) *gyro_y = imu_data.gyro_y;
|
||
if (gyro_z) *gyro_z = imu_data.gyro_z;
|
||
if (temperature) *temperature = imu_data.temperature;
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
void Application::OnMotionDetected() {
|
||
ESP_LOGI(TAG, "Motion detected by IMU sensor");
|
||
|
||
// 如果设备处于空闲状态，可以触发一些动作
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
// 例如：显示运动检测提示
|
||
auto& board = Board::GetInstance();
|
||
auto display = board.GetDisplay();
|
||
display->SetChatMessage("system", "检测到运动");
|
||
|
||
// 可以在这里添加更多的运动检测处理逻辑
|
||
// 比如：唤醒设备、记录运动数据等
|
||
}
|
||
}
|
||
|
||
void Application::SetLowBatteryTransition(bool value) {
|
||
is_low_battery_transition_.store(value);// 设置低电量过渡状态
|
||
}
|
||
|
||
bool Application::IsLowBatteryTransition() const {
|
||
return is_low_battery_transition_.load();// 获取低电量过渡状态
|
||
}
|