3037 lines
145 KiB
C++
3037 lines
145 KiB
C++
#include "application.h"
|
||
#include "board.h"
|
||
#include "wifi_board.h"
|
||
#include "display.h"
|
||
#include "system_info.h"
|
||
#include "ml307_ssl_transport.h"
|
||
#include "audio_codec.h"
|
||
#include "settings.h"
|
||
#include "mqtt_protocol.h"
|
||
#include "websocket_protocol.h"
|
||
#include "volc_rtc_protocol.h"
|
||
#include "font_awesome_symbols.h"
|
||
#include "iot/thing_manager.h"
|
||
#include "assets/lang_config.h"
|
||
#include "volume_config.h"
|
||
#include "boards/common/qmi8658a.h" // 添加qmi8658a_data_t类型的头文件
|
||
#include "boards/movecall-moji-esp32s3/movecall_moji_esp32s3.h" // 添加MovecallMojiESP32S3类的头文件
|
||
#include "weather_api.h"
|
||
|
||
#include <cstring>
|
||
#include <esp_log.h>
|
||
#include <cJSON.h>
|
||
#include <driver/gpio.h>
|
||
#include <arpa/inet.h>
|
||
#include <esp_app_desc.h>
|
||
#include <cmath>
|
||
#include <chrono>
|
||
#include <esp_wifi.h>
|
||
#include <nvs.h>
|
||
|
||
#define TAG "Application"
|
||
#define MAC_TAG "WiFiMAC"
|
||
|
||
#define DIALOG_IDLE_COUNTDOWN_SECONDS 40
|
||
|
||
|
||
// 定义设备状态字符串
|
||
static const char* const STATE_STRINGS[] = {
|
||
"unknown",
|
||
"starting",
|
||
"configuring",
|
||
"idle",
|
||
"connecting",
|
||
"listening",
|
||
"speaking",
|
||
"dialog",
|
||
"upgrading",
|
||
"activating",
|
||
"fatal_error"
|
||
};
|
||
|
||
Application::Application() {
|
||
event_group_ = xEventGroupCreate();
|
||
background_task_ = new BackgroundTask(4096 * 8);
|
||
last_audible_output_time_ = std::chrono::steady_clock::now(); // 初始化最后一次有声音输出的时间点
|
||
skip_dialog_idle_session_ = false; // 初始化跳过对话待机会话标志为false
|
||
dialog_watchdog_running_ = false; // 初始化对话看门狗运行标志
|
||
dialog_watchdog_last_logged_ = -1; // 初始化对话看门狗日志记录
|
||
dialog_watchdog_task_handle_ = nullptr; // 初始化对话看门狗任务句柄
|
||
clock_ticks_ = 0; // 初始化时钟计数
|
||
main_loop_task_handle_ = nullptr; // 初始化主循环任务句柄
|
||
check_new_version_task_handle_ = nullptr; // 初始化版本检查任务句柄
|
||
audio_loop_task_handle_ = nullptr; // 初始化音频循环任务句柄
|
||
|
||
esp_timer_create_args_t clock_timer_args = {
|
||
.callback = [](void* arg) {
|
||
Application* app = (Application*)arg;
|
||
app->OnClockTimer();
|
||
},
|
||
.arg = this,
|
||
.dispatch_method = ESP_TIMER_TASK,
|
||
.name = "clock_timer",
|
||
.skip_unhandled_events = true
|
||
};
|
||
esp_timer_create(&clock_timer_args, &clock_timer_handle_);
|
||
}
|
||
|
||
Application::~Application() {
|
||
// 停止并清理对话看门狗
|
||
StopDialogWatchdog();
|
||
|
||
if (clock_timer_handle_ != nullptr) {
|
||
esp_timer_stop(clock_timer_handle_);
|
||
esp_timer_delete(clock_timer_handle_);
|
||
}
|
||
if (background_task_ != nullptr) {
|
||
delete background_task_;
|
||
}
|
||
if (recorder_pipeline_) {
|
||
recorder_pipeline_close(recorder_pipeline_);
|
||
recorder_pipeline_ = nullptr;
|
||
}
|
||
if (player_pipeline_) {
|
||
player_pipeline_close(player_pipeline_);
|
||
player_pipeline_ = nullptr;
|
||
}
|
||
vEventGroupDelete(event_group_);
|
||
}
|
||
|
||
void Application::CheckNewVersion() {
|
||
// ESP_LOGI(TAG, "OTA版本检查已临时禁用");
|
||
// return;
|
||
auto& board = Board::GetInstance();
|
||
auto display = board.GetDisplay();
|
||
// Check if there is a new firmware version available
|
||
ota_.SetPostData(board.GetJson());// 发送当前设备的JSON数据到OTA服务器，用于检查是否有新的固件版本 包办板载信息 BOARD_TYPE
|
||
|
||
const int MAX_RETRY = 10;
|
||
int retry_count = 0;
|
||
|
||
while (true) {
|
||
if (!ota_.CheckVersion()) {
|
||
retry_count++;
|
||
if (retry_count >= MAX_RETRY) {
|
||
ESP_LOGE(TAG, "Too many retries, exit version check");
|
||
return;
|
||
}
|
||
ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", 60, retry_count, MAX_RETRY);
|
||
vTaskDelay(pdMS_TO_TICKS(60000));
|
||
continue;
|
||
}
|
||
retry_count = 0;
|
||
|
||
if (ota_.HasNewVersion()) {
|
||
Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "happy", Lang::Sounds::P3_UPGRADE);
|
||
// Wait for the chat state to be idle
|
||
do {
|
||
vTaskDelay(pdMS_TO_TICKS(3000));
|
||
} while (GetDeviceState() != kDeviceStateIdle);
|
||
|
||
// Use main task to do the upgrade, not cancelable
|
||
Schedule([this, display]() {
|
||
SetDeviceState(kDeviceStateUpgrading);
|
||
|
||
display->SetIcon(FONT_AWESOME_DOWNLOAD);
|
||
std::string message = std::string(Lang::Strings::NEW_VERSION) + ota_.GetFirmwareVersion();
|
||
display->SetChatMessage("system", message.c_str());
|
||
|
||
auto& board = Board::GetInstance();
|
||
board.SetPowerSaveMode(false);// 关闭低功耗模式
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Stop();
|
||
#endif
|
||
// 预先关闭音频输出，避免升级过程有音频操作
|
||
auto codec = board.GetAudioCodec();
|
||
codec->EnableInput(false);
|
||
codec->EnableOutput(false);
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.clear();
|
||
}
|
||
background_task_->WaitForCompletion();
|
||
delete background_task_;
|
||
background_task_ = nullptr;
|
||
vTaskDelay(pdMS_TO_TICKS(1000));
|
||
|
||
ota_.StartUpgrade([display](int progress, size_t speed) {
|
||
char buffer[64];
|
||
snprintf(buffer, sizeof(buffer), "%d%% %zuKB/s", progress, speed / 1024);
|
||
display->SetChatMessage("system", buffer);
|
||
});
|
||
|
||
// If upgrade success, the device will reboot and never reach here
|
||
display->SetStatus(Lang::Strings::UPGRADE_FAILED);
|
||
ESP_LOGI(TAG, "Firmware upgrade failed...");
|
||
vTaskDelay(pdMS_TO_TICKS(3000));
|
||
Reboot();
|
||
});
|
||
|
||
return;
|
||
}
|
||
|
||
// No new version, mark the current version as valid
|
||
ota_.MarkCurrentVersionValid();
|
||
std::string message = std::string(Lang::Strings::VERSION) + ota_.GetCurrentVersion();
|
||
display->ShowNotification(message.c_str());
|
||
|
||
// 检查是否有设备激活码
|
||
// if (ota_.HasActivationCode()) {
|
||
// // Activation code is valid
|
||
// SetDeviceState(kDeviceStateActivating);//设置设备状态为激活中
|
||
// // ShowActivationCode();//显示设备激活码
|
||
|
||
// // Check again in 60 seconds or until the device is idle
|
||
// for (int i = 0; i < 60; ++i) {
|
||
// if (device_state_ == kDeviceStateIdle) {
|
||
// break;
|
||
// }
|
||
// vTaskDelay(pdMS_TO_TICKS(1000));
|
||
// }
|
||
// continue;
|
||
// }
|
||
|
||
SetDeviceState(kDeviceStateIdle);
|
||
display->SetChatMessage("system", "");
|
||
ResetDecoder();
|
||
PlaySound(Lang::Sounds::P3_SUCCESS);
|
||
// Exit the loop if upgrade or idle
|
||
break;
|
||
}
|
||
}
|
||
|
||
// 取消设备激活码播报,当前设备绑定使用Wi-Fi的Mac地址进行绑定
|
||
// void Application::ShowActivationCode() {
|
||
// auto& message = ota_.GetActivationMessage();
|
||
// auto& code = ota_.GetActivationCode();
|
||
|
||
// struct digit_sound {
|
||
// char digit;
|
||
// const std::string_view& sound;
|
||
// };
|
||
// static const std::array<digit_sound, 10> digit_sounds{{
|
||
// digit_sound{'0', Lang::Sounds::P3_0},
|
||
// digit_sound{'1', Lang::Sounds::P3_1},
|
||
// digit_sound{'2', Lang::Sounds::P3_2},
|
||
// digit_sound{'3', Lang::Sounds::P3_3},
|
||
// digit_sound{'4', Lang::Sounds::P3_4},
|
||
// digit_sound{'5', Lang::Sounds::P3_5},
|
||
// digit_sound{'6', Lang::Sounds::P3_6},
|
||
// digit_sound{'7', Lang::Sounds::P3_7},
|
||
// digit_sound{'8', Lang::Sounds::P3_8},
|
||
// digit_sound{'9', Lang::Sounds::P3_9}
|
||
// }};
|
||
|
||
// // This sentence uses 9KB of SRAM, so we need to wait for it to finish
|
||
// Alert(Lang::Strings::ACTIVATION, message.c_str(), "happy", Lang::Sounds::P3_ACTIVATION);
|
||
// vTaskDelay(pdMS_TO_TICKS(1000));
|
||
// background_task_->WaitForCompletion();
|
||
|
||
// for (const auto& digit : code) {
|
||
// auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(),
|
||
// [digit](const digit_sound& ds) { return ds.digit == digit; });
|
||
// if (it != digit_sounds.end()) {
|
||
// PlaySound(it->sound);
|
||
// }
|
||
// }
|
||
// }
|
||
|
||
// 新增代码(小程序控制 暂停播放 音频)
|
||
// =========================================================
|
||
void Application::PauseAudioPlayback() {
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
if (!audio_paused_) {
|
||
audio_paused_ = true;// 暂停播放(更新标志位)
|
||
ESP_LOGI(TAG, "🔇 从服务器接收到暂停播放指令");
|
||
|
||
// 恢复原始处理方式：立即停止音频输出
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(false);// 暂停时立即停止音频输出
|
||
ESP_LOGI(TAG, "⏸️ 音频编解码器输出已禁用，实现立即暂停");
|
||
}
|
||
ESP_LOGI(TAG, "⏸️ 音频播放已暂停");
|
||
}
|
||
}
|
||
// 新增代码(小程序控制 继续播放 音频)
|
||
void Application::ResumeAudioPlayback() {
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
if (audio_paused_) {
|
||
audio_paused_ = false;// 恢复播放(更新标志位)
|
||
ESP_LOGI(TAG, "<EFBFBD> 从服务器接收到继续播放指令");
|
||
|
||
// 恢复原始处理方式：重新启用音频输出
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(true);// 恢复时重新启用音频输出
|
||
ESP_LOGI(TAG, "▶️ 音频编解码器输出已启用");
|
||
}
|
||
ESP_LOGI(TAG, "▶️ 音频播放已恢复");
|
||
}
|
||
}
|
||
// =========================================================
|
||
|
||
void Application::Alert(const char* status, const char* message, const char* emotion, const std::string_view& sound) {
|
||
ESP_LOGW(TAG, "Alert %s: %s [%s]", status, message, emotion);
|
||
auto display = Board::GetInstance().GetDisplay();
|
||
display->SetStatus(status);
|
||
display->SetEmotion(emotion);
|
||
display->SetChatMessage("system", message);
|
||
if (!sound.empty()) {
|
||
ResetDecoder();
|
||
PlaySound(sound);
|
||
}
|
||
}
|
||
|
||
void Application::DismissAlert() {
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
auto display = Board::GetInstance().GetDisplay();
|
||
display->SetStatus(Lang::Strings::STANDBY);
|
||
display->SetEmotion("neutral");
|
||
display->SetChatMessage("system", "");
|
||
}
|
||
}
|
||
|
||
// 播放音频文件的函数，用于播放存储在内存中的音频数据
|
||
void Application::PlaySound(const std::string_view& sound) {
|
||
// The assets are encoded at 16000Hz, 60ms frame duration
|
||
SetDecodeSampleRate(16000, 60);
|
||
const char* data = sound.data();
|
||
size_t size = sound.size();
|
||
for (const char* p = data; p < data + size; ) {
|
||
auto p3 = (BinaryProtocol3*)p;
|
||
p += sizeof(BinaryProtocol3);
|
||
|
||
auto payload_size = ntohs(p3->payload_size);
|
||
std::vector<uint8_t> opus;
|
||
opus.resize(payload_size);
|
||
memcpy(opus.data(), p3->payload, payload_size);
|
||
p += payload_size;
|
||
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.emplace_back(std::move(opus));
|
||
}
|
||
}
|
||
|
||
// 切换聊天状态的函数，用于在不同的设备状态之间进行切换
|
||
void Application::ToggleChatState() {
|
||
// 如果当前设备状态是激活中，则将状态设置为空闲并返回
|
||
if (device_state_ == kDeviceStateActivating) {
|
||
SetDeviceState(kDeviceStateIdle); // 设置设备状态为空闲
|
||
return; // 直接返回，不执行后续逻辑
|
||
}
|
||
// 检查协议对象是否已初始化
|
||
if (!protocol_) {
|
||
ESP_LOGE(TAG, "协议未初始化"); // 记录错误日志：协议未初始化
|
||
return; // 协议未初始化则直接返回
|
||
}
|
||
// 如果当前设备状态是idle空闲，则尝试进入对话模式
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
Schedule([this]() {
|
||
SetDeviceState(kDeviceStateConnecting);
|
||
ESP_LOGI(TAG, "正在尝试打开音频通道");
|
||
Board::GetInstance().SetPowerSaveMode(false);// 关闭低功耗模式
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
auto ac = Board::GetInstance().GetAudioCodec();
|
||
ESP_LOGW(TAG, "打开音频通道失败，将在2秒后重试");
|
||
if (ac) {
|
||
ESP_LOGW(TAG, "Diag: codec out_channels=%d in_channels=%d out_sr=%d in_sr=%d", ac->output_channels(), ac->input_channels(), ac->output_sample_rate(), ac->input_sample_rate());
|
||
}
|
||
SetDeviceState(kDeviceStateIdle);
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(2000));
|
||
ESP_LOGI(TAG, "正在重试音频通道连接");
|
||
ToggleChatState();// 打开音频通道
|
||
});
|
||
return;
|
||
}
|
||
|
||
listening_mode_ = kListeningModeRealtime;// 设置监听模式为实时监听
|
||
SetDeviceState(kDeviceStateDialog);// 设置设备状态为对话模式
|
||
protocol_->SendStartListening(listening_mode_);// 发送开始监听消息
|
||
auto codec = Board::GetInstance().GetAudioCodec();// 获取音频编解码器
|
||
if (codec) {
|
||
codec->EnableOutput(true);// 启用音频输出
|
||
}
|
||
ESP_LOGI(TAG, "进入对话框状态：启用全双工");
|
||
});
|
||
} else if (device_state_ == kDeviceStateDialog) {
|
||
Schedule([this]() {
|
||
// protocol_->CloseAudioChannel();// 关闭音频通道
|
||
// ESP_LOGI(TAG, "关闭音频通道并切换到空闲状态");// 关闭音频通道并切换到空闲状态
|
||
protocol_->SendStartListening(listening_mode_);// 发送开始监听消息
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
Schedule([this]() {
|
||
AbortSpeaking(kAbortReasonNone);
|
||
protocol_->CloseAudioChannel();
|
||
ESP_LOGI(TAG, "关闭音频通道并切换到空闲状态");// 关闭音频通道并切换到空闲状态
|
||
});
|
||
} else if (device_state_ == kDeviceStateListening || (listening_mode_ == kListeningModeRealtime && device_state_ == kDeviceStateSpeaking)) {
|
||
Schedule([this]() {
|
||
protocol_->CloseAudioChannel();
|
||
ESP_LOGI(TAG, "关闭音频通道并切换到空闲状态");// 关闭音频通道并切换到空闲状态
|
||
});
|
||
}
|
||
}
|
||
|
||
void Application::ToggleListeningState() {
|
||
if (device_state_ == kDeviceStateActivating) {
|
||
SetDeviceState(kDeviceStateIdle);
|
||
return;
|
||
}
|
||
|
||
if (!protocol_) {
|
||
ESP_LOGE(TAG, "协议未初始化！");// 记录错误日志：协议未初始化
|
||
return;
|
||
}
|
||
|
||
// 简单的状态切换：idle <-> listening
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
// 从待命状态进入聆听状态
|
||
Schedule([this]() {
|
||
SetDeviceState(kDeviceStateConnecting);
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
return;
|
||
}
|
||
SetListeningMode(kListeningModeManualStop);
|
||
ESP_LOGI(TAG, "中断按钮：进入聆听状态");// 中断按钮：进入聆听状态
|
||
});
|
||
} else if (device_state_ == kDeviceStateListening) {
|
||
// 从聆听状态返回待命状态
|
||
Schedule([this]() {
|
||
protocol_->CloseAudioChannel();
|
||
ESP_LOGI(TAG, "中断按钮：返回待命状态");// 中断按钮：返回待命状态
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
// 如果正在说话，中止说话并返回待命状态
|
||
Schedule([this]() {
|
||
AbortSpeaking(kAbortReasonNone);
|
||
if (protocol_) {
|
||
protocol_->CloseAudioChannel();
|
||
}
|
||
SetDeviceState(kDeviceStateIdle);
|
||
ESP_LOGI(TAG, "中断按钮：停止说话，关闭音频通道并返回待命状态");// 中断按钮：停止说话，关闭音频通道并返回待命状态
|
||
});
|
||
} else if (device_state_ == kDeviceStateConnecting) {
|
||
// 如果正在连接，直接返回待命状态
|
||
Schedule([this]() {
|
||
SetDeviceState(kDeviceStateIdle);
|
||
ESP_LOGI(TAG, "中断按钮：取消连接并返回待命状态");// 中断按钮：取消连接并返回待命状态
|
||
});
|
||
}
|
||
}
|
||
|
||
void Application::StartListening() {
|
||
if (device_state_ == kDeviceStateActivating) {
|
||
SetDeviceState(kDeviceStateIdle);
|
||
return;
|
||
}
|
||
|
||
if (!protocol_) {
|
||
ESP_LOGE(TAG, "协议未初始化！");// 记录错误日志：协议未初始化
|
||
return;
|
||
}
|
||
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
Schedule([this]() {
|
||
if (!protocol_->IsAudioChannelOpened()) {
|
||
SetDeviceState(kDeviceStateConnecting);// 切换到连接状态
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
return;
|
||
}
|
||
}
|
||
|
||
SetListeningMode(kListeningModeManualStop);// 设置监听模式为手动停止
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
Schedule([this]() {
|
||
AbortSpeaking(kAbortReasonNone);// 中止说话
|
||
SetListeningMode(kListeningModeManualStop);// 设置监听模式为手动停止
|
||
});
|
||
}
|
||
}
|
||
|
||
void Application::StopListening() {
|
||
Schedule([this]() {
|
||
if (device_state_ == kDeviceStateListening) {
|
||
protocol_->SendStopListening();
|
||
SetDeviceState(kDeviceStateIdle);
|
||
}
|
||
});
|
||
}
|
||
|
||
// 🔊 发送文本消息到RTC（传入大模型上下文信息）
|
||
void Application::SendTextMessage(const std::string& text) {
|
||
if (!protocol_) {
|
||
ESP_LOGE(TAG, "协议未初始化！");// 记录错误日志：协议未初始化
|
||
return;
|
||
}
|
||
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
Schedule([this, text]() {
|
||
SetDeviceState(kDeviceStateConnecting);// 切换到连接状态
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
return;
|
||
}
|
||
|
||
SetDeviceState(kDeviceStateDialog);
|
||
protocol_->SendStartListening(listening_mode_);
|
||
|
||
// 发送文本消息
|
||
protocol_->SendTextMessage(text);
|
||
ESP_LOGI(TAG, "发送文本消息：%s", text.c_str());// 发送文本消息：%s
|
||
|
||
// 立即启动监听模式以接收语音回复
|
||
ESP_LOGI(TAG, "realtime_chat_enabled_=%s", realtime_chat_enabled_ ? "true" : "false");
|
||
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeManualStop);
|
||
});
|
||
} else if (device_state_ == kDeviceStateDialog) {
|
||
Schedule([this, text]() {
|
||
// if (!protocol_->IsAudioChannelOpened()) {// 如果音频通道未打开
|
||
// if (!protocol_->OpenAudioChannel()) {// 尝试打开音频通道
|
||
// return;
|
||
// }
|
||
// }
|
||
if (!dialog_upload_enabled_) {
|
||
SetDialogUploadEnabled(true);// 启用对话上传
|
||
protocol_->SendStartListening(listening_mode_);// 发送开始监听消息
|
||
}
|
||
protocol_->SendTextMessage(text);// 发送文本消息
|
||
ESP_LOGI(TAG, "发送文本消息：%s", text.c_str());// 发送文本消息：%s
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
Schedule([this, text]() {
|
||
AbortSpeaking(kAbortReasonNone);
|
||
protocol_->SendTextMessage(text);
|
||
ESP_LOGI(TAG, "发送文本消息：%s", text.c_str());// 发送文本消息：%s
|
||
|
||
// 启动监听模式以接收语音回复
|
||
ESP_LOGI(TAG, "realtime_chat_enabled_=%s", realtime_chat_enabled_ ? "true" : "false");
|
||
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeManualStop);
|
||
});
|
||
} else if (device_state_ == kDeviceStateListening) {
|
||
Schedule([this, text]() {
|
||
protocol_->SendTextMessage(text);
|
||
ESP_LOGI(TAG, "发送文本消息：%s", text.c_str());// 发送文本消息：%s
|
||
});
|
||
}
|
||
}
|
||
|
||
void Application::Start() {
|
||
auto& board = Board::GetInstance();
|
||
SetDeviceState(kDeviceStateStarting);
|
||
|
||
// 读取NVS中的重启标志
|
||
Settings sys("system", true);
|
||
int32_t reboot_dlg_idle = sys.GetInt("reboot_dlg_idle", 0);
|
||
int32_t reboot_origin = sys.GetInt("reboot_origin", 0);
|
||
|
||
// 检查是否是因为对话空闲倒计时而重启的
|
||
if (reboot_dlg_idle == 1 && reboot_origin == 1) {
|
||
ESP_LOGI(TAG, "检测到对话空闲倒计时重启标志，将跳过开机播报和网络连接播报");
|
||
skip_dialog_idle_session_ = true;
|
||
Settings sysclr("system", true);
|
||
sysclr.SetInt("reboot_dlg_idle", 0);
|
||
sysclr.SetInt("reboot_origin", 0);
|
||
sysclr.Commit();
|
||
} else {
|
||
ESP_LOGI(TAG, "正常启动流程，将执行开机播报和网络连接播报");
|
||
skip_dialog_idle_session_ = false;
|
||
}
|
||
|
||
/* Setup the display */
|
||
auto display = board.GetDisplay();
|
||
|
||
/* Setup the audio codec */
|
||
auto codec = board.GetAudioCodec();
|
||
opus_decoder_ = std::make_unique<OpusDecoderWrapper>(codec->output_sample_rate(), 1, OPUS_FRAME_DURATION_MS);
|
||
opus_encoder_ = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
|
||
if (realtime_chat_enabled_) {
|
||
ESP_LOGI(TAG, "实时聊天已启用，将opus编码器复杂度设置为0");// 实时聊天已启用，将opus编码器复杂度设置为0
|
||
opus_encoder_->SetComplexity(0);
|
||
} else if (board.GetBoardType() == "ml307") {
|
||
ESP_LOGI(TAG, "检测到ML307板卡，将opus编码器复杂度设置为5");// 检测到ML307板卡，将opus编码器复杂度设置为5
|
||
opus_encoder_->SetComplexity(5);
|
||
} else {
|
||
ESP_LOGI(TAG, "检测到WiFi板卡，将opus编码器复杂度设置为3");// 检测到WiFi板卡，将opus编码器复杂度设置为3
|
||
opus_encoder_->SetComplexity(3);
|
||
}
|
||
|
||
if (codec->input_sample_rate() != 16000) {
|
||
input_resampler_.Configure(codec->input_sample_rate(), 16000);
|
||
reference_resampler_.Configure(codec->input_sample_rate(), 16000);
|
||
}
|
||
uplink_resampler_.Configure(16000, 8000);
|
||
codec->Start();
|
||
{
|
||
int battery_level = 0;
|
||
bool charging = false;
|
||
bool discharging = false;
|
||
if (board.GetBatteryLevel(battery_level, charging, discharging)) {
|
||
// 如果电池电量低于25%，则将输出音量设置为0（静音）
|
||
if (battery_level <= 25) {
|
||
codec->SetOutputVolumeRuntime(0);
|
||
} else {
|
||
Settings s("audio", false);
|
||
int vol = s.GetInt("output_volume", AudioCodec::default_output_volume());
|
||
if (vol <= 0) {
|
||
vol = AudioCodec::default_output_volume();
|
||
}
|
||
codec->SetOutputVolumeRuntime(vol);// 设置运行时输出音量
|
||
}
|
||
}
|
||
}
|
||
|
||
// // 在启动阶段创建并运行播放管道以统一输出（开机启动播放管道）
|
||
// if (!player_pipeline_) {
|
||
// player_pipeline_ = player_pipeline_open();
|
||
// player_pipeline_run(player_pipeline_);
|
||
// }
|
||
|
||
xTaskCreatePinnedToCore([](void* arg) {
|
||
Application* app = (Application*)arg;
|
||
app->AudioLoop();
|
||
vTaskDelete(NULL);
|
||
}, "audio_loop", 4096 * 3, this, 8, &audio_loop_task_handle_, realtime_chat_enabled_ ? 1 : 0);
|
||
|
||
/* Start the main loop */
|
||
xTaskCreatePinnedToCore([](void* arg) {
|
||
Application* app = (Application*)arg;
|
||
app->MainLoop();
|
||
vTaskDelete(NULL);
|
||
}, "main_loop", 4096 * 3, this, 4, &main_loop_task_handle_, 0);
|
||
|
||
// 根据标志决定是否播放开机播报语音
|
||
if (!skip_dialog_idle_session_) {
|
||
ESP_LOGI(TAG, "设备启动完成，播放开机播报语音");// 设备启动完成，播放开机播报语音
|
||
//PlaySound(Lang::Sounds::P3_KAIJIBOBAO); 原有蜡笔小新音色
|
||
|
||
if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){
|
||
PlaySound(Lang::Sounds::P3_KAKA_KAIJIBOBAO);
|
||
}
|
||
else if(strcmp(CONFIG_DEVICE_ROLE, "RTC_Test") == 0){
|
||
PlaySound(Lang::Sounds::P3_LALA_KAIJIBOBAO);
|
||
}
|
||
} else {
|
||
ESP_LOGI(TAG, "跳过开机播报语音");
|
||
}
|
||
|
||
/* Wait for the network to be ready */
|
||
board.StartNetwork();
|
||
|
||
// Initialize the protocol
|
||
display->SetStatus(Lang::Strings::LOADING_PROTOCOL);
|
||
#if CONFIG_CONNECTION_TYPE_VOLC_RTC
|
||
auto volc_protocol = std::make_unique<VolcRtcProtocol>();// 初始化VolcRtc协议
|
||
// 设置AgentConfig: 这里的配置会在RTC入会时透传给云端
|
||
// WelcomeMessage: 设置开场白
|
||
// std::string agent_config = "{\"agent_config\":{\"WelcomeMessage\":\"我是推销员雷军，有什么产品可以帮您介绍的嘛\"}}"; // 已请求成功，配置生效
|
||
// std::string config = "{\"Config\":{\"WebSearchAgentConfig\":{\"ComfortWords\":\"啦啦正在上网查询，等一下哦～\"}}}"; // 已请求成功，配置生效
|
||
// std::string config = "{\"Config\":{\"WebSearchAgentConfig\":{\"ParamsString\":\"{\\\"bot_id\\\":\\\"7585449675889608233\\\",\\\"stream\\\":true,\\\"location_info\\\":{\\\"city\\\":\\\"上海\\\"}}\"}}}";// 已经请求成功，无报错，配置不生效
|
||
std::string city = g_weather_api.GetDefaultLocation();// 获取当前默认城市信息
|
||
wifi_config_t wc{};// 获取当前WiFi配置
|
||
esp_wifi_get_config(WIFI_IF_STA, &wc);// 获取当前WiFi配置
|
||
std::string ssid = std::string(reinterpret_cast<char*>(wc.sta.ssid));// 获取当前WiFi SSID
|
||
wifi_ap_record_t ap{};// 获取当前AP信息
|
||
std::string bssid;// 获取当前AP BSSID
|
||
|
||
if (esp_wifi_sta_get_ap_info(&ap) == ESP_OK) {
|
||
char buf[18];
|
||
snprintf(buf, sizeof(buf), "%02x:%02x:%02x:%02x:%02x:%02x",ap.bssid[0], ap.bssid[1], ap.bssid[2],ap.bssid[3], ap.bssid[4], ap.bssid[5]);
|
||
bssid.assign(buf);
|
||
}
|
||
nvs_handle_t h;
|
||
// 从NVS中读取当前WiFi SSID和BSSID对应的城市信息
|
||
if (nvs_open("wifi_city_map", NVS_READONLY, &h) == ESP_OK) {
|
||
auto try_get = [&](const std::string& key)->std::string{
|
||
size_t len = 0;
|
||
if (nvs_get_str(h, key.c_str(), NULL, &len) == ESP_OK && len > 0) {
|
||
std::vector<char> buf(len);
|
||
if (nvs_get_str(h, key.c_str(), buf.data(), &len) == ESP_OK) {
|
||
return std::string(buf.data());
|
||
}
|
||
}
|
||
return std::string();// 如果NVS中没有对应城市信息，返回空字符串
|
||
};
|
||
// 从NVS中读取当前WiFi SSID和BSSID对应的城市信息
|
||
if (!ssid.empty()) {
|
||
std::string city_hit;// 从NVS中读取当前WiFi SSID和BSSID对应的城市信息
|
||
if (!bssid.empty()) {
|
||
city_hit = try_get(ssid + "|" + bssid);// 从NVS中读取当前WiFi SSID和BSSID对应的城市信息
|
||
}
|
||
if (city_hit.empty()) {
|
||
city_hit = try_get(ssid);// 从NVS中读取当前WiFi SSID对应的城市信息
|
||
}
|
||
if (!city_hit.empty()) {
|
||
city = city_hit;// 如果NVS中存在对应城市信息，更新当前城市信息
|
||
}
|
||
}
|
||
nvs_close(h);// 关闭NVS句柄
|
||
}
|
||
// 更新config参数
|
||
std::string params = std::string("{\\\"bot_id\\\":\\\"7585449675889608233\\\",\\\"stream\\\":true,\\\"location_info\\\":{\\\"city\\\":\\\"") + city + "\\\"}}";
|
||
std::string config = std::string("{\"Config\":{\"WebSearchAgentConfig\":{\"ParamsString\":\"") + params + "\"}}}";
|
||
volc_protocol->SetAgentConfig(config);// 设置AgentConfig: 这里的配置会在RTC入会时透传给云端 WebSearchAgentConfigvxiassfdfdfdevde
|
||
protocol_ = std::move(volc_protocol);
|
||
#elif CONFIG_CONNECTION_TYPE_WEBSOCKET
|
||
protocol_ = std::make_unique<WebsocketProtocol>();
|
||
#else
|
||
protocol_ = std::make_unique<MqttProtocol>();
|
||
#endif
|
||
protocol_->OnNetworkError([this](const std::string& message) {
|
||
// SetDeviceState(kDeviceStateIdle);
|
||
// Alert(Lang::Strings::ERROR, message.c_str(), "sad", Lang::Sounds::P3_EXCLAMATION);
|
||
|
||
ESP_LOGW(TAG, "网络错误发生：%s", message.c_str());// 网络错误发生：%s
|
||
// 检查是否是TLS连接重置错误
|
||
if (message.find("TLS") != std::string::npos || message.find("-76") != std::string::npos) {
|
||
ESP_LOGI(TAG, "检测到TLS连接重置错误，将在3秒后自动重试连接");// 检测到TLS连接重置错误，将在3秒后自动重试连接
|
||
SetDeviceState(kDeviceStateIdle);
|
||
|
||
// 3秒后自动重试连接
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(3000));
|
||
if (GetDeviceState() == kDeviceStateIdle) {
|
||
ESP_LOGI(TAG, "自动重试连接，TLS错误已解决");// 自动重试连接，TLS错误已解决
|
||
ToggleChatState();
|
||
}
|
||
});
|
||
} else {
|
||
// 其他网络错误正常处理
|
||
SetDeviceState(kDeviceStateIdle);
|
||
Alert(Lang::Strings::ERROR, message.c_str(), "sad", Lang::Sounds::P3_EXCLAMATION);
|
||
}
|
||
});
|
||
protocol_->OnIncomingAudio([this](std::vector<uint8_t>&& data) {
|
||
if (websocket_protocol_ && websocket_protocol_->IsAudioChannelOpened()) {
|
||
aborted_ = true;
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);// 🔒 保护音频队列操作
|
||
// 如果音频队列不为空
|
||
if (!audio_decode_queue_.empty()) {
|
||
ESP_LOGI(TAG, "清空音频队列，大小=%zu", audio_decode_queue_.size());
|
||
audio_decode_queue_.clear();// 清空音频队列
|
||
}
|
||
}
|
||
ResetDecoder();
|
||
ws_downlink_enabled_.store(false);
|
||
ws_playback_active_.store(false);
|
||
websocket_protocol_->CloseAudioChannel();// 关闭WebSocket通道
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(120));
|
||
aborted_ = false;
|
||
});
|
||
}
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
size_t len = data.size();
|
||
audio_decode_queue_.emplace_back(std::move(data));
|
||
static bool first_enqueue_logged = false;
|
||
if (!first_enqueue_logged && len > 0) {
|
||
ESP_LOGI(TAG, "收到下行音频首包入队: 字节=%zu", len);
|
||
first_enqueue_logged = true;
|
||
}
|
||
ESP_LOGD(TAG, "收到下行音频入队: 字节=%zu 队列大小=%zu", len, audio_decode_queue_.size());
|
||
});
|
||
protocol_->OnAudioChannelOpened([this, codec, &board]() {
|
||
ESP_LOGI(TAG, "🟢 音频通道已打开");
|
||
ESP_LOGI(TAG, "当前设备状态: %s", STATE_STRINGS[device_state_]);
|
||
|
||
// 🔧 关键修复：立即取消所有待执行的电源管理任务
|
||
static TaskHandle_t power_save_task = nullptr;
|
||
if (power_save_task != nullptr) {
|
||
vTaskDelete(power_save_task);
|
||
power_save_task = nullptr;
|
||
ESP_LOGI(TAG, "🔧 取消了待执行的电源管理任务");
|
||
}
|
||
|
||
// 唤醒PowerSaveTimer，从低功耗模式恢复到正常模式
|
||
board.WakeUp();
|
||
|
||
// 立即禁用电源管理，确保连接稳定
|
||
ESP_LOGI(TAG, "🔄 禁用电源低功耗管理模式");
|
||
board.SetPowerSaveMode(false);
|
||
|
||
// 关键修复：检查服务器采样率与设备输出采样率是否匹配
|
||
if (protocol_->server_sample_rate() != codec->output_sample_rate()) {
|
||
ESP_LOGW(TAG, "⚠️ 服务器采样率 %d 与设备输出采样率 %d 不匹配，重采样可能导致失真",
|
||
protocol_->server_sample_rate(), codec->output_sample_rate());
|
||
}
|
||
|
||
// 设置解码采样率和帧持续时间
|
||
SetDecodeSampleRate(protocol_->server_sample_rate(), protocol_->server_frame_duration());
|
||
|
||
// 关键修复：明确启用音频编解码器输出
|
||
ESP_LOGI(TAG, "🔊 启用音频编解码器输出");
|
||
codec->EnableOutput(true);// 启用音频编解码器输出
|
||
|
||
if (!player_pipeline_) {
|
||
player_pipeline_ = player_pipeline_open();
|
||
player_pipeline_run(player_pipeline_);
|
||
}
|
||
|
||
// 发送IoT状态信息
|
||
auto& thing_manager = iot::ThingManager::GetInstance();
|
||
protocol_->SendIotDescriptors(thing_manager.GetDescriptorsJson());
|
||
std::string states;
|
||
if (thing_manager.GetStatesJson(states, false)) {
|
||
protocol_->SendIotStates(states);
|
||
}
|
||
|
||
// if (websocket_protocol_ && !websocket_protocol_->IsAudioChannelOpened()) {
|
||
// ESP_LOGI(TAG, "WS辅助通道连接");
|
||
// websocket_protocol_->OpenAudioChannel();//
|
||
// }
|
||
|
||
// 🔧 修复：RTC连接后切换到Speaking状态以播放欢迎语音
|
||
ESP_LOGI(TAG, "🔄 音频通道打开，准备播放欢迎语音");
|
||
if (GetDeviceState() != kDeviceStateDialog) {
|
||
SetDeviceState(kDeviceStateSpeaking);
|
||
}
|
||
ESP_LOGI(TAG, "当前设备状态: %s", STATE_STRINGS[device_state_]);
|
||
ESP_LOGI(TAG, "🟢 音频通道初始化完成");
|
||
});
|
||
protocol_->OnAudioChannelClosed([this, &board]() {
|
||
ESP_LOGI(TAG, "🔴 音频通道关闭，开始清理任务");
|
||
|
||
// 🔧 关键修复：取消所有待执行的电源管理任务，防止时序冲突
|
||
static TaskHandle_t power_save_task = nullptr;
|
||
if (power_save_task != nullptr) {
|
||
vTaskDelete(power_save_task);
|
||
power_save_task = nullptr;
|
||
ESP_LOGI(TAG, "🔧 取消了之前的电源管理任务，防止时序冲突");
|
||
}
|
||
|
||
// 音频处理器已经在WebSocket断开时停止了
|
||
// 等待所有后台任务完成
|
||
background_task_->WaitForCompletion();
|
||
if (player_pipeline_) {
|
||
player_pipeline_close(player_pipeline_);
|
||
player_pipeline_ = nullptr;
|
||
}
|
||
ESP_LOGI(TAG, "🔴 后台任务完成");
|
||
|
||
// 🔧 方案2：先设置设备状态，再启用电源管理，避免时序问题
|
||
Schedule([this, &board]() {
|
||
ESP_LOGI(TAG, "🔄 设置设备为空闲状态");
|
||
auto display = Board::GetInstance().GetDisplay();
|
||
display->SetChatMessage("system", "");
|
||
SetDeviceState(kDeviceStateIdle);
|
||
|
||
// 状态设置完成后，再启用电源管理
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
ESP_LOGI(TAG, "🔄 设备已稳定在idle状态，启用电源低功耗管理");
|
||
try {
|
||
board.SetPowerSaveMode(true);
|
||
} catch (...) {
|
||
ESP_LOGE(TAG, "❌ 设置电源管理模式失败");
|
||
}
|
||
});
|
||
});
|
||
protocol_->OnIncomingJson([this, display](const cJSON* root) {
|
||
// Parse JSON data
|
||
auto type = cJSON_GetObjectItem(root, "type");
|
||
if (!(type && cJSON_IsString(type))) {
|
||
auto tool_calls = cJSON_GetObjectItem(root, "tool_calls");
|
||
if (tool_calls && cJSON_IsArray(tool_calls)) {
|
||
for (int i = 0; i < cJSON_GetArraySize(tool_calls); ++i) {
|
||
cJSON* call = cJSON_GetArrayItem(tool_calls, i);
|
||
cJSON* fn = cJSON_GetObjectItem(call, "function");
|
||
if (fn && cJSON_IsObject(fn)) {
|
||
cJSON* name = cJSON_GetObjectItem(fn, "name");
|
||
cJSON* args = cJSON_GetObjectItem(fn, "arguments");
|
||
cJSON* args_obj = nullptr;
|
||
const char* args_str = (args && cJSON_IsString(args) && args->valuestring) ? args->valuestring : "";
|
||
if (args && cJSON_IsString(args) && args->valuestring) {
|
||
args_obj = cJSON_Parse(args->valuestring);
|
||
}
|
||
if (name && cJSON_IsString(name) && name->valuestring) {
|
||
if (args_obj) {
|
||
char* printed = cJSON_PrintUnformatted(args_obj);
|
||
if (printed) {
|
||
ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, printed);
|
||
cJSON_free(printed);
|
||
} else {
|
||
ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, args_str);
|
||
}
|
||
if (strcmp(name->valuestring, "adjust_audio_val") == 0) {
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
int user = HARDWARE_TO_USER_VOLUME(codec->output_volume());
|
||
cJSON* v = cJSON_GetObjectItem(args_obj, "value");
|
||
if (!v) v = cJSON_GetObjectItem(args_obj, "action");
|
||
if (v) {
|
||
std::string msg;
|
||
if (cJSON_IsString(v) && v->valuestring) {
|
||
if (strcmp(v->valuestring, "up") == 0) {
|
||
user += 10;
|
||
msg = "音量已经调大了哦～";
|
||
} else if (strcmp(v->valuestring, "down") == 0) {
|
||
user -= 10;
|
||
msg = "音量已经调小了哦～";
|
||
} else {
|
||
// 处理字符串形式的数字
|
||
char* endptr;
|
||
long val = strtol(v->valuestring, &endptr, 10);
|
||
if (*endptr == '\0' && val >= 0 && val <= 100) {
|
||
user = (int)val;
|
||
msg = std::string("音量值已经调整为") + std::to_string(user) + "%";
|
||
}
|
||
}
|
||
} else if (cJSON_IsNumber(v)) {
|
||
user = (int)v->valuedouble;
|
||
msg = std::string("音量值已经调整为") + std::to_string(user) + "%";
|
||
}
|
||
if (user > 100) user = 100;
|
||
if (user < 0) user = 0;
|
||
int mapped = USER_TO_HARDWARE_VOLUME(user);
|
||
codec->SetOutputVolume(mapped);
|
||
ESP_LOGI(TAG, "设置音量: 用户=%d%% 硬件=%d%%", user, mapped);
|
||
if (!msg.empty()) {
|
||
cJSON* call_id_item = cJSON_GetObjectItem(call, "id");
|
||
const char* call_id = (call_id_item && cJSON_IsString(call_id_item) && call_id_item->valuestring) ? call_id_item->valuestring : "";
|
||
if (protocol_ && call_id && call_id[0] != '\0') {
|
||
protocol_->SendFunctionResult(call_id, msg);
|
||
} else if (protocol_) {
|
||
protocol_->SendTextMessage(msg);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// // 添加天气查询功能处理 get_weather_aihub
|
||
// else if (strcmp(name->valuestring, "get_weather_aihub") == 0) {
|
||
// ESP_LOGI(TAG, "[WeatherAPI] ===== 收到get_weather_aihub工具调用 =====");
|
||
//
|
||
// // 打印完整参数信息用于调试
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 参数对象检查:");
|
||
// if (args_obj && cJSON_IsObject(args_obj)) {
|
||
// cJSON* current = args_obj->child;
|
||
// while (current) {
|
||
// ESP_LOGI(TAG, "[WeatherAPI] %s: %s",
|
||
// current->string,
|
||
// cJSON_IsString(current) ? current->valuestring :
|
||
// (cJSON_IsNumber(current) ? "(number)" :
|
||
// (cJSON_IsBool(current) ? (current->valueint ? "true" : "false") :
|
||
// "(other)")));
|
||
// current = current->next;
|
||
// }
|
||
// } else {
|
||
// ESP_LOGI(TAG, "[WeatherAPI] args_obj为空或不是对象类型");
|
||
// }
|
||
//
|
||
// // 解析参数
|
||
// cJSON* location = cJSON_GetObjectItem(args_obj, "location");
|
||
// cJSON* lang = cJSON_GetObjectItem(args_obj, "lang");
|
||
//
|
||
// ESP_LOGI(TAG, "[WeatherAPI] location参数存在: %s, 类型: %s",
|
||
// location ? "是" : "否",
|
||
// location && cJSON_IsString(location) ? "字符串" :
|
||
// (location ? "非字符串" : "不适用"));
|
||
// ESP_LOGI(TAG, "[WeatherAPI] lang参数存在: %s, 类型: %s",
|
||
// lang ? "是" : "否",
|
||
// lang && cJSON_IsString(lang) ? "字符串" :
|
||
// (lang ? "非字符串" : "不适用"));
|
||
//
|
||
// // 设置默认值
|
||
// const char* location_str = (location && cJSON_IsString(location)) ? location->valuestring : "";
|
||
// const char* lang_str = (lang && cJSON_IsString(lang)) ? lang->valuestring : "zh_CN";
|
||
// std::string location_copy = std::string(location_str);
|
||
// std::string lang_copy = std::string(lang_str);
|
||
// if (location_copy.empty() || location_copy == "None") {
|
||
// wifi_config_t wc{};
|
||
// esp_wifi_get_config(WIFI_IF_STA, &wc);
|
||
// std::string ssid = std::string(reinterpret_cast<char*>(wc.sta.ssid));
|
||
// wifi_ap_record_t ap{};
|
||
// std::string bssid;
|
||
// if (esp_wifi_sta_get_ap_info(&ap) == ESP_OK) {
|
||
// char buf[18];
|
||
// snprintf(buf, sizeof(buf), "%02x:%02x:%02x:%02x:%02x:%02x",
|
||
// ap.bssid[0], ap.bssid[1], ap.bssid[2],
|
||
// ap.bssid[3], ap.bssid[4], ap.bssid[5]);
|
||
// bssid.assign(buf);
|
||
// }
|
||
// nvs_handle_t h;
|
||
// if (nvs_open("wifi_city_map", NVS_READONLY, &h) == ESP_OK) {
|
||
// auto try_get = [&](const std::string& key)->std::string{
|
||
// size_t len = 0;
|
||
// if (nvs_get_str(h, key.c_str(), NULL, &len) == ESP_OK && len > 0) {
|
||
// std::vector<char> buf(len);
|
||
// if (nvs_get_str(h, key.c_str(), buf.data(), &len) == ESP_OK) {
|
||
// return std::string(buf.data());
|
||
// }
|
||
// }
|
||
// return std::string();
|
||
// };
|
||
// if (!ssid.empty()) {
|
||
// std::string city;
|
||
// if (!bssid.empty()) {
|
||
// city = try_get(ssid + "|" + bssid);
|
||
// }
|
||
// if (city.empty()) {
|
||
// city = try_get(ssid);
|
||
// }
|
||
// if (!city.empty()) {
|
||
// location_copy = city;
|
||
// }
|
||
// }
|
||
// nvs_close(h);
|
||
// }
|
||
// }
|
||
//
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 提取的参数值: location='%s' (长度: %zu), lang='%s'",
|
||
// location_str, strlen(location_str), lang_str);
|
||
//
|
||
// // 获取call_id用于后续响应
|
||
// cJSON* call_id_item = cJSON_GetObjectItem(call, "id");
|
||
// const char* call_id = (call_id_item && cJSON_IsString(call_id_item) && call_id_item->valuestring) ? call_id_item->valuestring : "";
|
||
// std::string call_id_copy = call_id ? call_id : "";
|
||
//
|
||
// ESP_LOGI(TAG, "[WeatherAPI] call_id_item存在: %s", call_id_item ? "是" : "否");
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 获取的call_id: '%s'", call_id_copy.c_str());
|
||
//
|
||
// // 创建异步任务处理天气获取
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 准备创建异步任务处理天气API调用");
|
||
// Schedule([this, location_copy, lang_copy, call_id_copy]() {
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 异步任务开始执行");
|
||
// try {
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 准备调用全局函数GetWeatherInfo(location='%s', lang='%s')",
|
||
// location_copy.c_str(), lang_copy.c_str());
|
||
//
|
||
// // 调用天气API获取结果
|
||
// std::string weather_result = GetWeatherInfo(location_copy, lang_copy);
|
||
//
|
||
// ESP_LOGI(TAG, "[WeatherAPI] GetWeatherInfo调用完成，结果长度: %zu 字节", weather_result.length());
|
||
// ESP_LOGD(TAG, "[WeatherAPI] GetWeatherInfo返回结果前100字节: '%s'",
|
||
// weather_result.substr(0, std::min(size_t(100), weather_result.length())).c_str());
|
||
//
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 准备发送天气结果响应");
|
||
// if (!call_id_copy.empty()) {
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 使用call_id发送FunctionResult: '%s'", call_id_copy.c_str());
|
||
// } else {
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 无call_id，将发送TextMessage");
|
||
// }
|
||
//
|
||
// if (!call_id_copy.empty() && protocol_) {
|
||
// protocol_->SendFunctionResult(call_id_copy.c_str(), weather_result);
|
||
// ESP_LOGI(TAG, "[WeatherAPI] FunctionResult发送成功");
|
||
// } else if (protocol_) {
|
||
// protocol_->SendTextMessage(weather_result);// 发送天气结果
|
||
// ESP_LOGI(TAG, "[WeatherAPI] TextMessage发送成功");
|
||
// } else {
|
||
// ESP_LOGE(TAG, "[WeatherAPI] protocol_为空，无法发送响应");
|
||
// }
|
||
// } catch (const std::exception& e) {
|
||
// ESP_LOGE(TAG, "[WeatherAPI] 天气获取异常: %s", e.what());
|
||
// std::string error_msg = "获取天气信息失败，请稍后重试";
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 准备发送错误响应: '%s'", error_msg.c_str());
|
||
//
|
||
// if (!call_id_copy.empty() && protocol_) {
|
||
// protocol_->SendFunctionResult(call_id_copy.c_str(), error_msg);
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 错误FunctionResult发送成功");
|
||
// } else if (protocol_) {
|
||
// protocol_->SendTextMessage(error_msg);
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 错误TextMessage发送成功");
|
||
// } else {
|
||
// ESP_LOGE(TAG, "[WeatherAPI] protocol_为空，无法发送错误响应");
|
||
// }
|
||
// }
|
||
// ESP_LOGI(TAG, "[WeatherAPI] ===== get_weather_aihub异步任务处理完成 =====");
|
||
// });
|
||
//
|
||
// ESP_LOGI(TAG, "[WeatherAPI] 异步任务已调度，将在后台执行");
|
||
// ESP_LOGI(TAG, "[WeatherAPI] ===== get_weather_aihub工具调用响应已发送 =====");
|
||
// }
|
||
cJSON_Delete(args_obj);// 释放参数对象
|
||
} else {
|
||
ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, args_str);
|
||
if (strcmp(name->valuestring, "adjust_audio_val") == 0) {
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
int user = HARDWARE_TO_USER_VOLUME(codec->output_volume());
|
||
if (args && cJSON_IsString(args) && args->valuestring) {
|
||
cJSON* tmp = cJSON_Parse(args->valuestring);
|
||
if (tmp) {
|
||
cJSON* v = cJSON_GetObjectItem(tmp, "value");
|
||
if (!v) v = cJSON_GetObjectItem(tmp, "action");
|
||
if (v) {
|
||
std::string msg;
|
||
if (cJSON_IsString(v) && v->valuestring) {
|
||
if (strcmp(v->valuestring, "up") == 0) {
|
||
user += 10;
|
||
msg = "音量已经调大了哦～";
|
||
} else if (strcmp(v->valuestring, "down") == 0) {
|
||
user -= 10;
|
||
msg = "音量已经调小了哦～";
|
||
} else {
|
||
// 处理字符串形式的数字
|
||
char* endptr;
|
||
long val = strtol(v->valuestring, &endptr, 10);
|
||
if (*endptr == '\0' && val >= 0 && val <= 100) {
|
||
user = (int)val;
|
||
msg = std::string("音量值已经调整为") + std::to_string(user) + "%";
|
||
}
|
||
}
|
||
} else if (cJSON_IsNumber(v)) {
|
||
user = (int)v->valuedouble;
|
||
msg = std::string("音量值已经调整为") + std::to_string(user) + "%";
|
||
}
|
||
if (user > 100) user = 100;
|
||
if (user < 0) user = 0;
|
||
int mapped = USER_TO_HARDWARE_VOLUME(user);
|
||
codec->SetOutputVolume(mapped);
|
||
ESP_LOGI(TAG, "设置音量: 用户=%d%% 硬件=%d%%", user, mapped);
|
||
if (!msg.empty()) {
|
||
cJSON* call_id_item = cJSON_GetObjectItem(call, "id");
|
||
const char* call_id = (call_id_item && cJSON_IsString(call_id_item) && call_id_item->valuestring) ? call_id_item->valuestring : "";
|
||
if (protocol_ && call_id && call_id[0] != '\0') {
|
||
protocol_->SendFunctionResult(call_id, msg);
|
||
} else if (protocol_) {
|
||
protocol_->SendTextMessage(msg);
|
||
}
|
||
}
|
||
}
|
||
cJSON_Delete(tmp);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
return;
|
||
}
|
||
if (strcmp(type->valuestring, "tts") == 0) {
|
||
auto state = cJSON_GetObjectItem(root, "state");
|
||
if (strcmp(state->valuestring, "start") == 0) {
|
||
Schedule([this]() {
|
||
aborted_ = false;
|
||
if (device_state_ == kDeviceStateIdle || device_state_ == kDeviceStateListening) {
|
||
SetDeviceState(kDeviceStateSpeaking);
|
||
}
|
||
});
|
||
} else if (strcmp(state->valuestring, "stop") == 0) {
|
||
Schedule([this]() {
|
||
background_task_->WaitForCompletion();
|
||
if (device_state_ == kDeviceStateSpeaking) {
|
||
if (listening_mode_ == kListeningModeManualStop) {
|
||
SetDeviceState(kDeviceStateIdle);
|
||
} else {
|
||
SetDeviceState(kDeviceStateListening);
|
||
}
|
||
}
|
||
});
|
||
} else if (strcmp(state->valuestring, "sentence_start") == 0) {
|
||
auto text = cJSON_GetObjectItem(root, "text");
|
||
if (text != NULL) {
|
||
ESP_LOGI(TAG, "<< %s", text->valuestring);
|
||
Schedule([this, display, message = std::string(text->valuestring)]() {
|
||
display->SetChatMessage("assistant", message.c_str());
|
||
});
|
||
}
|
||
}
|
||
} else if (strcmp(type->valuestring, "stt") == 0) {
|
||
auto text = cJSON_GetObjectItem(root, "text");
|
||
if (text != NULL) {
|
||
ESP_LOGI(TAG, ">> %s", text->valuestring);
|
||
Schedule([this, display, message = std::string(text->valuestring)]() {
|
||
display->SetChatMessage("user", message.c_str());
|
||
});
|
||
}
|
||
} else if (strcmp(type->valuestring, "llm") == 0) {
|
||
auto emotion = cJSON_GetObjectItem(root, "emotion");
|
||
if (emotion != NULL) {
|
||
Schedule([this, display, emotion_str = std::string(emotion->valuestring)]() {
|
||
display->SetEmotion(emotion_str.c_str());
|
||
});
|
||
}
|
||
} else if (strcmp(type->valuestring, "iot") == 0) {
|
||
auto commands = cJSON_GetObjectItem(root, "commands");
|
||
if (commands != NULL) {
|
||
auto& thing_manager = iot::ThingManager::GetInstance();
|
||
for (int i = 0; i < cJSON_GetArraySize(commands); ++i) {
|
||
auto command = cJSON_GetArrayItem(commands, i);
|
||
thing_manager.Invoke(command);
|
||
}
|
||
}
|
||
} else if (strcmp(type->valuestring, "response.function_call_arguments.done") == 0) {
|
||
auto name = cJSON_GetObjectItem(root, "name");
|
||
auto arguments = cJSON_GetObjectItem(root, "arguments");
|
||
cJSON* args_obj = nullptr;
|
||
const char* args_str = (arguments && cJSON_IsString(arguments) && arguments->valuestring) ? arguments->valuestring : "";
|
||
if (arguments && cJSON_IsString(arguments) && arguments->valuestring) {
|
||
args_obj = cJSON_Parse(arguments->valuestring);
|
||
}
|
||
if (name && cJSON_IsString(name) && name->valuestring) {
|
||
if (args_obj) {
|
||
char* printed = cJSON_PrintUnformatted(args_obj);
|
||
if (printed) {
|
||
ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, printed);
|
||
cJSON_free(printed);
|
||
} else {
|
||
ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, args_str);
|
||
}
|
||
if (strcmp(name->valuestring, "adjust_audio_val") == 0) {
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
int user = HARDWARE_TO_USER_VOLUME(codec->output_volume());
|
||
cJSON* v = cJSON_GetObjectItem(args_obj, "value");// 获取value字段
|
||
if (!v) v = cJSON_GetObjectItem(args_obj, "action");// 如果value字段不存在，尝试action字段
|
||
if (v) {
|
||
std::string msg;
|
||
if (cJSON_IsString(v) && v->valuestring) {
|
||
if (strcmp(v->valuestring, "up") == 0) {
|
||
user += 10;
|
||
msg = "音量已经调大了哦～";
|
||
} else if (strcmp(v->valuestring, "down") == 0) {
|
||
user -= 10;
|
||
msg = "音量已经调小了哦～";
|
||
} else {
|
||
// 处理字符串形式的数字
|
||
char* endptr;
|
||
long val = strtol(v->valuestring, &endptr, 10);
|
||
if (*endptr == '\0' && val >= 0 && val <= 100) {
|
||
user = (int)val;
|
||
msg = std::string("音量值已经调整为") + std::to_string(user) + "%";
|
||
}
|
||
}
|
||
} else if (cJSON_IsNumber(v)) {
|
||
user = (int)v->valuedouble;
|
||
msg = std::string("音量值已经调整为") + std::to_string(user) + "%";
|
||
}
|
||
if (user > 100) user = 100;
|
||
if (user < 0) user = 0;
|
||
int mapped = USER_TO_HARDWARE_VOLUME(user);
|
||
codec->SetOutputVolume(mapped);
|
||
ESP_LOGI(TAG, "设置音量: 用户=%d%% 硬件=%d%%", user, mapped);
|
||
if (!msg.empty()) {
|
||
cJSON* call_id_item = cJSON_GetObjectItem(root, "call_id");
|
||
const char* call_id = (call_id_item && cJSON_IsString(call_id_item) && call_id_item->valuestring) ? call_id_item->valuestring : "";
|
||
if (protocol_ && call_id && call_id[0] != '\0') {
|
||
protocol_->SendFunctionResult(call_id, msg);
|
||
} else if (protocol_) {
|
||
protocol_->SendTextMessage(msg);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
cJSON_Delete(args_obj);
|
||
} else {
|
||
ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, args_str);
|
||
if (strcmp(name->valuestring, "adjust_audio_val") == 0) {
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
int user = HARDWARE_TO_USER_VOLUME(codec->output_volume());
|
||
if (arguments && cJSON_IsString(arguments) && arguments->valuestring) {
|
||
cJSON* tmp = cJSON_Parse(arguments->valuestring);
|
||
if (tmp) {
|
||
cJSON* v = cJSON_GetObjectItem(tmp, "value");
|
||
if (!v) v = cJSON_GetObjectItem(tmp, "action");
|
||
if (v) {
|
||
std::string msg;
|
||
if (cJSON_IsString(v) && v->valuestring) {
|
||
if (strcmp(v->valuestring, "up") == 0) {
|
||
user += 10;
|
||
msg = "音量已经调大了哦～";
|
||
} else if (strcmp(v->valuestring, "down") == 0) {
|
||
user -= 10;
|
||
msg = "音量已经调小了哦～";
|
||
} else {
|
||
// 处理字符串形式的数字
|
||
char* endptr;
|
||
long val = strtol(v->valuestring, &endptr, 10);
|
||
if (*endptr == '\0' && val >= 0 && val <= 100) {
|
||
user = (int)val;
|
||
msg = std::string("音量值已经调整为") + std::to_string(user) + "%";
|
||
}
|
||
}
|
||
} else if (cJSON_IsNumber(v)) {
|
||
user = (int)v->valuedouble;
|
||
msg = std::string("音量值已经调整为") + std::to_string(user) + "%";
|
||
}
|
||
if (user > 100) user = 100;
|
||
if (user < 0) user = 0;
|
||
int mapped = USER_TO_HARDWARE_VOLUME(user);
|
||
codec->SetOutputVolume(mapped);
|
||
ESP_LOGI(TAG, "设置音量: 用户=%d%% 硬件=%d%%", user, mapped);
|
||
if (!msg.empty()) {
|
||
cJSON* call_id_item = cJSON_GetObjectItem(root, "call_id");
|
||
const char* call_id = (call_id_item && cJSON_IsString(call_id_item) && call_id_item->valuestring) ? call_id_item->valuestring : "";
|
||
if (protocol_ && call_id && call_id[0] != '\0') {
|
||
protocol_->SendFunctionResult(call_id, msg);
|
||
} else if (protocol_) {
|
||
protocol_->SendTextMessage(msg);
|
||
}
|
||
}
|
||
}
|
||
cJSON_Delete(tmp);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// 新增代码(小程序控制 暂停/继续播放 音频)
|
||
// ====================================================================
|
||
}
|
||
else if (strcmp(type->valuestring, "music_control") == 0) {
|
||
auto action = cJSON_GetObjectItem(root, "action");
|
||
if (action && cJSON_IsString(action) && strcmp(action->valuestring, "pause") == 0) {
|
||
// 只有在speaking状态下才响应暂停指令
|
||
if (device_state_ == kDeviceStateSpeaking) {
|
||
ESP_LOGI(TAG, "🔇 从服务器接收到暂停播放指令 (speaking状态)");
|
||
Schedule([this]() {
|
||
PauseAudioPlayback();// 暂停播放
|
||
});
|
||
} else {
|
||
ESP_LOGI(TAG, "🔇 收到暂停指令但设备不在speaking状态，忽略指令 (当前状态: %s)", STATE_STRINGS[device_state_]);
|
||
}
|
||
} else if (action && cJSON_IsString(action) && strcmp(action->valuestring, "resume") == 0) {
|
||
// 只有在speaking状态下才响应恢复播放指令
|
||
if (device_state_ == kDeviceStateSpeaking) {
|
||
ESP_LOGI(TAG, "🔊 从服务器接收到继续播放指令 (speaking状态)");
|
||
Schedule([this]() {
|
||
ResumeAudioPlayback();// 恢复播放
|
||
});
|
||
} else {
|
||
ESP_LOGI(TAG, "🔊 收到恢复播放指令但设备不在speaking状态，忽略指令 (当前状态: %s)", STATE_STRINGS[device_state_]);
|
||
}
|
||
} else if (action && cJSON_IsString(action) && strcmp(action->valuestring, "play") == 0) {
|
||
// 处理新故事推送 - 确保在音频暂停状态和播放状态下都能正常播放
|
||
ESP_LOGI(TAG, "🎵 从服务器接收到新故事推送指令 (action: play)");
|
||
Schedule([this]() {
|
||
// 参考 AbortSpeakingAndReturnToListening 第1583-1651行的逻辑
|
||
// 检查并处理音频暂停状态，确保新故事能正常播放
|
||
if (audio_paused_) {
|
||
ESP_LOGI(TAG, "🔵 检测到音频暂停状态，为新故事推送清除暂停状态");
|
||
audio_paused_ = false;
|
||
ESP_LOGI(TAG, "✅ 音频暂停状态已清除");
|
||
|
||
// 清空音频播放队列，避免播放暂停时残留的音频
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.clear();
|
||
lock.unlock();
|
||
ESP_LOGI(TAG, "🧹 已清空音频播放队列，避免播放残留音频");
|
||
|
||
// 重新启用音频编解码器输出
|
||
auto& board = Board::GetInstance();
|
||
auto codec = board.GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(true);
|
||
ESP_LOGI(TAG, "🔧 为新故事推送重新启用音频编解码器输出");
|
||
}
|
||
}
|
||
|
||
// 如果当前在播放状态，也需要清空队列确保新故事能正常播放
|
||
if (device_state_ == kDeviceStateSpeaking) {
|
||
ESP_LOGI(TAG, "🔵 当前在播放状态，为新故事推送清空音频队列");
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.clear();
|
||
lock.unlock();
|
||
ESP_LOGI(TAG, "🧹 已清空音频播放队列，准备播放新故事");
|
||
|
||
// 确保音频编解码器输出已启用
|
||
auto& board = Board::GetInstance();
|
||
auto codec = board.GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(true);
|
||
ESP_LOGI(TAG, "🔧 确保音频编解码器输出已启用");
|
||
}
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🎵 新故事推送处理完成，音频系统已准备就绪");
|
||
});
|
||
}
|
||
}
|
||
// ====================================================================
|
||
});
|
||
protocol_->Start();
|
||
|
||
// Check for new firmware version or get the MQTT broker address
|
||
ota_.SetCheckVersionUrl(CONFIG_OTA_VERSION_URL);
|
||
ota_.SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str());
|
||
ota_.SetHeader("Client-Id", board.GetUuid());
|
||
ota_.SetHeader("Accept-Language", Lang::CODE);
|
||
auto app_desc = esp_app_get_description();
|
||
ota_.SetHeader("User-Agent", std::string(BOARD_NAME "/") + app_desc->version);
|
||
|
||
// 禁用自动OTA - 注释掉下面的任务创建OTA自动升级
|
||
xTaskCreate([](void* arg) {
|
||
Application* app = (Application*)arg;
|
||
app->CheckNewVersion();
|
||
vTaskDelete(NULL);
|
||
}, "check_new_version", 4096 * 2, this, 2, nullptr);
|
||
|
||
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.Initialize(codec, realtime_chat_enabled_);
|
||
|
||
// 🎯 根据语音打断功能启用状态配置VAD参数
|
||
EchoAwareVadParams enhanced_params;
|
||
if (realtime_chat_enabled_) {
|
||
// 语音打断功能启用：配置增强的回声感知参数 - 基于小智AI官方优化方案
|
||
// 🎯 平衡配置 - 防误触发同时保证音频流畅
|
||
enhanced_params.snr_threshold = 60.0f; // 平衡基础阈值：足够严格但不过度
|
||
enhanced_params.min_silence_ms = 2000; // 平衡静音要求：2秒
|
||
enhanced_params.interrupt_cooldown_ms = 10000; // 平衡冷却时间：10秒
|
||
enhanced_params.adaptive_threshold = true; // 启用自适应阈值
|
||
|
||
// 🔊 平衡噪声抑制参数 - 优化性能与效果
|
||
enhanced_params.adaptive_noise_suppression = true; // 启用自适应噪声抑制
|
||
enhanced_params.noise_suppression_base = 5.0f; // 平衡基础抑制强度
|
||
enhanced_params.volume_sensitivity = 3.0f; // 平衡音量敏感度：适度的音量影响
|
||
enhanced_params.echo_detection_threshold = 0.15f; // 平衡回声检测阈值
|
||
enhanced_params.distance_estimation_factor = 3.0f; // 平衡距离估算因子
|
||
ESP_LOGI(TAG, "🎯 Adaptive noise suppression enabled for realtime chat - smart volume/distance adjustment");
|
||
} else {
|
||
// 🔧 语音打断功能禁用：关闭复杂VAD，只使用简单VAD
|
||
enhanced_params.adaptive_threshold = false; // 禁用自适应阈值
|
||
enhanced_params.adaptive_noise_suppression = false; // 禁用自适应噪声抑制
|
||
ESP_LOGI(TAG, "🔧 Using simple VAD for basic voice detection - complex echo-aware VAD disabled");
|
||
}
|
||
audio_processor_.SetEchoAwareParams(enhanced_params);// 🔊 设置回声感知参数
|
||
|
||
// 🔊 注册音频处理输出回调 - 处理回声感知后的PCM数据
|
||
audio_processor_.OnOutput([this](std::vector<int16_t>&& data) {
|
||
background_task_->Schedule([this, data = std::move(data)]() mutable {
|
||
static uint64_t last_us = 0;
|
||
static size_t frames = 0;
|
||
std::vector<int16_t> resampled(uplink_resampler_.GetOutputSamples(data.size()));
|
||
if (!resampled.empty()) {
|
||
uplink_resampler_.Process(data.data(), data.size(), resampled.data());
|
||
}
|
||
std::vector<uint8_t> bytes(resampled.size() * sizeof(int16_t));
|
||
for (size_t i = 0; i < resampled.size(); ++i) {
|
||
int16_t s = resampled[i];
|
||
bytes[i * 2] = (uint8_t)(s & 0xFF);
|
||
bytes[i * 2 + 1] = (uint8_t)((s >> 8) & 0xFF);
|
||
}
|
||
frames += 1;
|
||
uint64_t now_us = esp_timer_get_time();
|
||
if (last_us == 0) last_us = now_us;
|
||
if (now_us - last_us >= 2000000) {
|
||
ESP_LOGI(TAG, "AFE输出统计: 帧=%zu 样本=%zu ", frames, data.size());
|
||
frames = 0;
|
||
last_us = now_us;
|
||
}
|
||
Schedule([this, bytes = std::move(bytes)]() {
|
||
if (protocol_ && protocol_->IsAudioChannelOpened() && (device_state_ == kDeviceStateListening || device_state_ == kDeviceStateDialog || (listening_mode_ == kListeningModeRealtime && device_state_ == kDeviceStateSpeaking))) {
|
||
protocol_->SendPcm(bytes);
|
||
} else {
|
||
ESP_LOGD(TAG, "通道未打开或不在dialog/listening状态时跳过发送上行");
|
||
}
|
||
});
|
||
});
|
||
});
|
||
// 🎯 根据语音打断功能启用状态选择VAD类型
|
||
if (realtime_chat_enabled_) {
|
||
// 语音打断功能启用：使用复杂的回声感知VAD
|
||
audio_processor_.OnVadStateChange([this](bool speaking) {
|
||
ESP_LOGI(TAG, "Complex VAD state change: speaking=%s, device_state=%d, listening_mode=%d",
|
||
speaking ? "true" : "false", (int)device_state_, (int)listening_mode_);
|
||
|
||
if (device_state_ == kDeviceStateListening) {
|
||
Schedule([this, speaking]() {
|
||
if (speaking) {
|
||
voice_detected_ = true;
|
||
} else {
|
||
voice_detected_ = false;
|
||
}
|
||
auto led = Board::GetInstance().GetLed();
|
||
led->OnStateChanged();
|
||
});
|
||
}
|
||
});
|
||
}
|
||
|
||
// 🔧 简单VAD：用于普通业务（触摸忽略、LED状态等）
|
||
audio_processor_.OnSimpleVadStateChange([this](bool speaking) {
|
||
ESP_LOGI(TAG, "Simple VAD state change: speaking=%s, device_state=%d",
|
||
speaking ? "true" : "false", (int)device_state_);
|
||
|
||
if (device_state_ == kDeviceStateListening) {
|
||
Schedule([this, speaking]() {
|
||
if (speaking) {
|
||
voice_detected_ = true;
|
||
} else {
|
||
voice_detected_ = false;
|
||
}
|
||
auto led = Board::GetInstance().GetLed();
|
||
led->OnStateChanged();
|
||
});
|
||
}
|
||
|
||
// 🔊 语音打断逻辑：只在简单VAD中处理，因为复杂VAD可能过于严格
|
||
if (device_state_ == kDeviceStateSpeaking && listening_mode_ == kListeningModeRealtime) {
|
||
Schedule([this, speaking]() {
|
||
static auto speech_start_time = std::chrono::steady_clock::now();
|
||
static bool speech_confirmation_pending = false;
|
||
auto now = std::chrono::steady_clock::now();
|
||
|
||
if (speaking) {
|
||
// 小智AI方案：检测到人声开始，启动确认流程
|
||
speech_start_time = now;
|
||
speech_confirmation_pending = true;
|
||
ESP_LOGD(TAG, "Human voice detected during playback, starting interrupt evaluation");
|
||
} else if (speech_confirmation_pending) {
|
||
// 小智AI方案：人声结束，评估是否触发打断
|
||
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now - speech_start_time);
|
||
|
||
// 🎯 平衡自适应打断策略：防误触发同时保证响应性
|
||
// 基础持续时间：3秒，平衡根据干扰情况调整
|
||
int required_duration = 3000; // 基础要求3秒
|
||
|
||
// 🔊 根据当前音量动态调整持续时间要求（平衡策略）
|
||
if (current_speaker_volume_ > 0.4f) {
|
||
required_duration = 5000; // 高音量：5秒
|
||
} else if (current_speaker_volume_ > 0.1f) {
|
||
required_duration = 4000; // 中音量：4秒
|
||
}
|
||
// 低音量或静音：保持3秒
|
||
|
||
if (duration.count() >= required_duration) {
|
||
static auto last_interrupt_time = std::chrono::steady_clock::now();
|
||
auto interrupt_duration = std::chrono::duration_cast<std::chrono::milliseconds>(now - last_interrupt_time);
|
||
|
||
// 🎯 平衡自适应多重保护机制：防误触发同时保证性能
|
||
bool volume_protection = (current_speaker_volume_ > 0.01f); // 平衡音量保护：1%阈值
|
||
bool cooldown_protection = (interrupt_duration.count() <= 10000); // 平衡冷却：10秒
|
||
static int false_positive_count = 0; // 误触发计数器
|
||
|
||
// 🎯 平衡连续误触发保护：适度学习机制
|
||
if (interrupt_duration.count() <= 20000) { // 20秒内的误触发相关
|
||
false_positive_count++;
|
||
} else if (interrupt_duration.count() > 60000) { // 60秒后开始衰减
|
||
false_positive_count = std::max(false_positive_count - 1, 0); // 适度衰减
|
||
}
|
||
|
||
bool consecutive_protection = (false_positive_count >= 2); // 2次误触发后保护
|
||
|
||
if (!volume_protection && !cooldown_protection && !consecutive_protection) {
|
||
// 小智AI核心逻辑：StopPlayback -> SetDeviceState(Listening)
|
||
ESP_LOGI(TAG, "🎯 Adaptive voice interrupt triggered (duration: %.0fms/%dms, vol: %.3f) - stopping playback",
|
||
(float)duration.count(), required_duration, current_speaker_volume_);
|
||
AbortSpeaking(kAbortReasonVoiceInterrupt);
|
||
SetDeviceState(kDeviceStateListening);
|
||
last_interrupt_time = now;
|
||
} else {
|
||
ESP_LOGI(TAG, "🎯 Adaptive interrupt suppressed - vol_protection: %s (%.3f), cooldown: %.0fms, consecutive: %d",
|
||
volume_protection ? "true" : "false", current_speaker_volume_,
|
||
10000.0f - (float)interrupt_duration.count(), false_positive_count);
|
||
}
|
||
} else {
|
||
ESP_LOGI(TAG, "🎯 Voice too brief (%.0fms), likely echo or noise - adaptive threshold: %dms",
|
||
(float)duration.count(), required_duration);
|
||
}
|
||
speech_confirmation_pending = false;
|
||
}
|
||
});
|
||
}
|
||
});
|
||
#endif
|
||
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
if (!wake_word_detect_.Initialize(codec)) {
|
||
ESP_LOGE(TAG, "Failed to initialize wake word detection");
|
||
return;
|
||
}
|
||
wake_word_detect_.OnWakeWordDetected([this](const std::string& wake_word) {
|
||
Schedule([this, &wake_word]() {
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
SetDeviceState(kDeviceStateConnecting);
|
||
wake_word_detect_.EncodeWakeWordData();
|
||
|
||
// 将OpenAudioChannel调用移到后台任务执行，避免main任务栈溢出
|
||
background_task_->Schedule([this, wake_word]() {
|
||
// 打开音频通道并发送唤醒词数据到服务器
|
||
if (!protocol_->OpenAudioChannel()) {
|
||
Schedule([this]() {
|
||
wake_word_detect_.Start();
|
||
});
|
||
return;
|
||
}
|
||
// 编码并发送唤醒词音频数据
|
||
std::vector<uint8_t> opus;
|
||
// Encode and send the wake word data to the server
|
||
while (wake_word_detect_.GetWakeWordOpus(opus)) {
|
||
protocol_->SendAudio(opus);
|
||
}
|
||
// Set the chat state to wake word detected
|
||
protocol_->SendWakeWordDetected(wake_word);
|
||
ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str());
|
||
|
||
// SetListeningMode需要在main task中执行，因为它涉及UI更新等
|
||
Schedule([this]() {
|
||
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeAutoStop);
|
||
});
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
AbortSpeaking(kAbortReasonWakeWordDetected);
|
||
} else if (device_state_ == kDeviceStateActivating) {
|
||
SetDeviceState(kDeviceStateIdle);
|
||
}
|
||
});
|
||
});
|
||
wake_word_detect_.Start();
|
||
#endif
|
||
|
||
SetDeviceState(kDeviceStateIdle);
|
||
// 每次设备开机后idle状态下测试 自动检测并设置当前位置打印
|
||
//此逻辑为冗余操作，当前NVS中没有城市信息时会自动调用 位置查询API
|
||
// Schedule([]() {
|
||
// AutoDetectAndSetLocation();
|
||
// });
|
||
esp_timer_start_periodic(clock_timer_handle_, 1000000);
|
||
|
||
#if 0
|
||
while (true) {
|
||
SystemInfo::PrintRealTimeStats(pdMS_TO_TICKS(1000));
|
||
vTaskDelay(pdMS_TO_TICKS(10000));
|
||
}
|
||
#endif
|
||
}
|
||
// 时钟定时器回调函数
|
||
void Application::OnClockTimer() {
|
||
clock_ticks_++;
|
||
|
||
// 每10秒打印一次调试信息
|
||
if (clock_ticks_ % 10 == 0) {
|
||
int free_sram = heap_caps_get_free_size(MALLOC_CAP_INTERNAL);
|
||
int min_free_sram = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL);
|
||
// ESP_LOGI(TAG, "Free internal: %u minimal internal: %u", free_sram, min_free_sram);// 打印内部内存空闲大小和最小空闲大小
|
||
// // 打印Wi-Fi的Mac地址
|
||
// ESP_LOGI(MAC_TAG, "Wi-Fi MAC Address: %s", SystemInfo::GetMacAddress().c_str());// 生产测试打印
|
||
|
||
//ESP_LOGI(TAG, "此设备角色为: %s",CONFIG_DEVICE_ROLE);
|
||
// ESP_LOGI(TAG, "此设备角色为: KAKA 1028 升级成功!");
|
||
|
||
// 如果我们已经同步了服务器时间，如果设备处于空闲状态，请将状态设置为时钟“HH:MM”
|
||
if (ota_.HasServerTime()) {
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
Schedule([this]() {
|
||
// Set status to clock "HH:MM"
|
||
time_t now = time(NULL);// 获取当前时间
|
||
char time_str[64];// 时间字符串缓冲区
|
||
strftime(time_str, sizeof(time_str), "%H:%M ", localtime(&now));// 格式化时间字符串
|
||
Board::GetInstance().GetDisplay()->SetStatus(time_str);// 设置显示状态
|
||
});
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 添加任务到主循环
|
||
void Application::Schedule(std::function<void()> callback) {
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);// 加锁保护任务队列
|
||
main_tasks_.push_back(std::move(callback));// 添加任务到队列
|
||
}
|
||
xEventGroupSetBits(event_group_, SCHEDULE_EVENT);// 通知主循环有任务需要执行
|
||
}
|
||
|
||
// 主循环控制聊天状态和Websocket连接
|
||
// 如果其他任务需要访问Websocket或聊天状态，
|
||
// 它们应该使用Schedule调用此函数
|
||
void Application::MainLoop() {
|
||
while (true) {
|
||
auto bits = xEventGroupWaitBits(event_group_, SCHEDULE_EVENT, pdTRUE, pdFALSE, portMAX_DELAY);// 等待任务事件触发
|
||
// 检查是否有任务需要执行
|
||
if (bits & SCHEDULE_EVENT) {
|
||
std::unique_lock<std::mutex> lock(mutex_);// 加锁保护任务队列
|
||
std::list<std::function<void()>> tasks = std::move(main_tasks_);// 移动任务队列到本地
|
||
lock.unlock();// 解锁，允许其他任务添加到队列
|
||
for (auto& task : tasks) {
|
||
task();// 执行任务
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 音频环路用于输入和输出音频数据
|
||
void Application::AudioLoop() {
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
while (true) {
|
||
OnAudioInput();
|
||
if (codec->output_enabled()) {
|
||
OnAudioOutput();
|
||
}
|
||
}
|
||
}
|
||
|
||
// 启动对话看门狗
|
||
void Application::StartDialogWatchdog() {
|
||
if (dialog_watchdog_task_handle_ != nullptr) {
|
||
return;// 如果看门狗任务已存在，直接返回
|
||
}
|
||
dialog_watchdog_running_ = true;// 设置看门狗运行标志为true
|
||
dialog_watchdog_last_logged_ = -1;// 重置上次记录的日志时间为-1
|
||
xTaskCreatePinnedToCore([](void* arg) {
|
||
Application* app = (Application*)arg;// 获取应用实例指针
|
||
ESP_LOGI(TAG, "Dialog watchdog started, initial device state: %d", app->GetDeviceState());
|
||
while (app->dialog_watchdog_running_) {
|
||
vTaskDelay(pdMS_TO_TICKS(2000));// 减少延时到2秒，更及时地检测和更新倒计时
|
||
|
||
// 检查设备状态
|
||
DeviceState current_state = app->GetDeviceState();
|
||
if (current_state != kDeviceStateDialog) {
|
||
ESP_LOGD(TAG, "Dialog watchdog skipping check, not in dialog state (current: %d)", current_state);
|
||
continue;
|
||
}
|
||
|
||
auto now = std::chrono::steady_clock::now();// 获取当前时间点
|
||
auto elapsed = std::chrono::duration_cast<std::chrono::seconds>(now - app->last_audible_output_time_).count();// 计算自上次有音频输出以来的秒数
|
||
|
||
// 确保elapsed为非负数
|
||
if (elapsed < 0) {
|
||
ESP_LOGW(TAG, "Dialog watchdog: invalid elapsed time: %lld", elapsed);
|
||
continue;
|
||
}
|
||
|
||
int remaining = DIALOG_IDLE_COUNTDOWN_SECONDS - (int)elapsed;// 计算对话空闲倒计时剩余秒数
|
||
|
||
// 调试日志
|
||
ESP_LOGD(TAG, "Dialog watchdog: elapsed=%d, remaining=%d", (int)elapsed, remaining);
|
||
|
||
// 如果剩余秒数小于等于0，说明对话空闲倒计时已到，需要重启设备
|
||
if (remaining <= 0) {
|
||
ESP_LOGI(TAG, "Dialog watchdog idle reached, elapsed=%d, marking and rebooting", (int)elapsed);
|
||
Settings sys("system", true);
|
||
ESP_LOGI(TAG, "Dialog watchdog: preparing NVS writes (system)");
|
||
sys.SetInt("reboot_dlg_idle", 1);
|
||
sys.SetInt("reboot_origin", 1);
|
||
ESP_LOGI(TAG, "Dialog watchdog: committing NVS (system)");
|
||
sys.Commit();
|
||
Settings sysr("system");
|
||
int32_t verify = sysr.GetInt("reboot_dlg_idle", 0);
|
||
int32_t origin_read = sysr.GetInt("reboot_origin", 0);
|
||
if (verify != 1) {
|
||
ESP_LOGW(TAG, "Dialog watchdog: NVS verify failed, cause=%d, origin=%d", (int)verify, (int)origin_read);
|
||
ESP_LOGW(TAG, "建议: 检查NVS空间是否不足、确认nvs_flash_init成功、避免并发写入(system)");
|
||
}
|
||
ESP_LOGI(TAG, "Dialog watchdog (task) set reboot_cause=1, verify=%d, restart in 2000ms", (int)verify);
|
||
vTaskDelay(pdMS_TO_TICKS(2000));
|
||
esp_restart();// 重启设备
|
||
app->dialog_watchdog_running_ = false;// 设置看门狗运行标志为false
|
||
} else {
|
||
// 简化条件判断，移除冗余检查
|
||
// 优化桶计算逻辑，使用1秒一个桶，更精确地显示倒计时
|
||
int bucket = remaining; // 使用剩余秒数作为桶标识，实现每秒更新
|
||
if (bucket != app->dialog_watchdog_last_logged_ && remaining <= DIALOG_IDLE_COUNTDOWN_SECONDS) {
|
||
ESP_LOGI(TAG, "dialog对话空闲倒计时剩余: %d 秒", remaining);// 打印剩余秒数
|
||
app->dialog_watchdog_last_logged_ = bucket;// 更新上次记录的日志时间为当前桶
|
||
}
|
||
}
|
||
}
|
||
app->dialog_watchdog_task_handle_ = nullptr;
|
||
ESP_LOGI(TAG, "Dialog watchdog stopped");
|
||
vTaskDelete(NULL);
|
||
}, "dialog_watchdog", 4096, this, 5, &dialog_watchdog_task_handle_, 0);
|
||
}
|
||
|
||
// 停止对话看门狗
|
||
void Application::StopDialogWatchdog() {
|
||
dialog_watchdog_running_ = false;
|
||
|
||
if (dialog_watchdog_task_handle_ != nullptr) {
|
||
vTaskDelete(dialog_watchdog_task_handle_);
|
||
dialog_watchdog_task_handle_ = nullptr;
|
||
ESP_LOGI(TAG, "Dialog watchdog stopped");
|
||
}
|
||
|
||
dialog_watchdog_last_logged_ = -1;
|
||
}
|
||
|
||
// 音频输出函数
|
||
void Application::OnAudioOutput() {
|
||
auto now = std::chrono::steady_clock::now();// 获取当前时间
|
||
auto codec = Board::GetInstance().GetAudioCodec();// 获取音频编解码器
|
||
const int max_silence_seconds = 10;// 最大静音秒数
|
||
std::unique_lock<std::mutex> lock(mutex_);// 加锁保护音频队列
|
||
|
||
// 调试日志：检查设备状态和音频队列
|
||
ESP_LOGD(TAG, "🔊 OnAudioOutput called, device_state: %d, audio_queue_size: %zu, codec_output_enabled: %d",
|
||
device_state_, audio_decode_queue_.size(), codec->output_enabled());
|
||
|
||
// 新增代码(小程序控制 暂停/继续播放 音频)
|
||
// =========================================================
|
||
// 🔧 暂停状态下停止从队列取数据，但保留队列状态
|
||
if (audio_paused_) {
|
||
// 暂停时重置音量状态，避免误判
|
||
if (current_speaker_volume_ > 0.0f) {
|
||
current_speaker_volume_ = 0.0f;// 暂停时重置音量状态
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
}
|
||
return;
|
||
}
|
||
// =========================================================
|
||
|
||
if (audio_decode_queue_.empty()) {
|
||
// 重要：没有音频数据时立即重置音量状态
|
||
if (current_speaker_volume_ > 0.0f) {
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
}
|
||
ESP_LOGD(TAG, "🔊 音频队列为空");
|
||
|
||
// Disable the output if there is no audio data for a long time
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
auto duration = std::chrono::duration_cast<std::chrono::seconds>(now - last_output_time_).count();
|
||
if (duration > max_silence_seconds) {
|
||
codec->EnableOutput(false);
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (device_state_ == kDeviceStateListening && listening_mode_ != kListeningModeRealtime) {
|
||
audio_decode_queue_.clear();
|
||
current_speaker_volume_ = 0.0f;// 清空音频队列后重置音量状态
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
return;
|
||
}
|
||
|
||
auto opus = std::move(audio_decode_queue_.front());
|
||
audio_decode_queue_.pop_front();
|
||
lock.unlock();
|
||
|
||
background_task_->Schedule([this, codec, opus = std::move(opus)]() mutable {
|
||
if (aborted_) {
|
||
return;
|
||
}
|
||
|
||
std::vector<int16_t> pcm;
|
||
bool decoded = false;
|
||
bool treat_as_pcm = (protocol_ && protocol_->downlink_is_pcm() && !ws_playback_active_.load());
|
||
if (!treat_as_pcm) {
|
||
decoded = opus_decoder_->Decode(std::move(opus), pcm);
|
||
}
|
||
if (!decoded) {
|
||
if (treat_as_pcm && !opus.empty() && (opus.size() % 2 == 0)) {
|
||
pcm.resize(opus.size() / 2);
|
||
memcpy(pcm.data(), opus.data(), opus.size());
|
||
int srv = protocol_ ? protocol_->server_sample_rate() : 16000;
|
||
if (!player_pipeline_) {
|
||
if (srv != codec->output_sample_rate()) {
|
||
output_resampler_.Configure(srv, codec->output_sample_rate());
|
||
int target_size = output_resampler_.GetOutputSamples(pcm.size());
|
||
std::vector<int16_t> resampled(target_size);
|
||
output_resampler_.Process(pcm.data(), pcm.size(), resampled.data());
|
||
pcm = std::move(resampled);
|
||
}
|
||
}
|
||
} else {
|
||
return;
|
||
}
|
||
}
|
||
// Resample if the sample rate is different
|
||
if (!treat_as_pcm && decoded && !player_pipeline_ && opus_decoder_->sample_rate() != codec->output_sample_rate()) {
|
||
int target_size = output_resampler_.GetOutputSamples(pcm.size());
|
||
std::vector<int16_t> resampled(target_size);
|
||
output_resampler_.Process(pcm.data(), pcm.size(), resampled.data());
|
||
pcm = std::move(resampled);
|
||
}
|
||
// 计算并同步音频输出音量到音频处理器，用于动态VAD阈值调整
|
||
if (!pcm.empty()) {
|
||
// 计算音频块的RMS音量 (0.0 - 1.0)
|
||
float sum_squares = 0.0f;
|
||
for (const auto& sample : pcm) {
|
||
float normalized = (float)sample / 32768.0f;
|
||
sum_squares += normalized * normalized;
|
||
}
|
||
float rms_volume = std::sqrt(sum_squares / pcm.size());
|
||
|
||
// 当有可听见的音频输出时，更新最后声音输出时间戳
|
||
const float audible_volume_threshold = 0.01f; // 设置一个合理的音量阈值
|
||
if (rms_volume >= audible_volume_threshold) {
|
||
this->last_audible_output_time_ = std::chrono::steady_clock::now();
|
||
ESP_LOGD(TAG, "🔊 更新last_audible_output_time_，当前音量: %.4f", rms_volume);
|
||
}
|
||
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
// 同步音量到音频处理器，用于动态阈值调整
|
||
current_speaker_volume_ = rms_volume; // 保存当前音量供打断逻辑使用
|
||
audio_processor_.SetSpeakerVolume(rms_volume);
|
||
#endif
|
||
}
|
||
|
||
int src_rate = decoded ? opus_decoder_->sample_rate() : (protocol_ ? protocol_->server_sample_rate() : 16000);
|
||
static bool first_play_logged = false;
|
||
if (player_pipeline_) {
|
||
player_pipeline_set_src_rate(player_pipeline_, src_rate);
|
||
int bytes = (int)(pcm.size() * sizeof(int16_t));
|
||
ESP_LOGD(TAG, "写入播放管道: 采样率=%d 字节=%d", src_rate, bytes);
|
||
player_pipeline_write(player_pipeline_, (char*)pcm.data(), bytes);
|
||
if (bytes > 0) {
|
||
this->last_audible_output_time_ = std::chrono::steady_clock::now();
|
||
}
|
||
if (!first_play_logged && bytes > 0) {
|
||
ESP_LOGI(TAG, "开始播放下行音频: 字节=%d 采样率=%d", bytes, src_rate);
|
||
first_play_logged = true;
|
||
}
|
||
} else {
|
||
ESP_LOGD(TAG, "直接输出PCM到编解码器: 样本=%zu", pcm.size());
|
||
codec->OutputData(pcm);// 直接输出PCM数据
|
||
if (!pcm.empty()) {
|
||
this->last_audible_output_time_ = std::chrono::steady_clock::now();
|
||
}
|
||
if (!first_play_logged && !pcm.empty()) {
|
||
ESP_LOGI(TAG, "开始播放下行音频: 样本=%zu 采样率=%d", pcm.size(), src_rate);
|
||
first_play_logged = true;
|
||
}
|
||
|
||
// 解决本地资源声音播放尖锐问题方案1
|
||
// // 如果是单声道，转换为立体声
|
||
// if (codec->output_channels() == 2) {// 单声道转换为立体声
|
||
// std::vector<int16_t> stereo(pcm.size() * 2);// 立体声PCM数据
|
||
// for (size_t i = 0, j = 0; i < pcm.size(); ++i) {
|
||
// stereo[j++] = pcm[i];// 左声道
|
||
// stereo[j++] = pcm[i];// 右声道
|
||
// }
|
||
// codec->OutputData(stereo);// 输出立体声PCM数据
|
||
// } else {
|
||
// codec->OutputData(pcm);// 输出单声道PCM数据
|
||
// }
|
||
|
||
// 解决本地资源声音播放尖锐问题方案2
|
||
// player_pipeline_ = player_pipeline_open();// 打开音频播放管道
|
||
// player_pipeline_run(player_pipeline_);// 启动音频播放管道
|
||
// player_pipeline_set_src_rate(player_pipeline_, src_rate);// 设置播放管道源采样率
|
||
// player_pipeline_write(player_pipeline_, (char*)pcm.data(), (int)(pcm.size() * sizeof(int16_t)));// 写入PCM数据到播放管道
|
||
|
||
}
|
||
last_output_time_ = std::chrono::steady_clock::now();// 更新最后输出时间
|
||
});
|
||
}
|
||
|
||
void Application::OnAudioInput() {
|
||
std::vector<int16_t> data;
|
||
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
if (wake_word_detect_.IsRunning()) {
|
||
ReadAudio(data, 16000, wake_word_detect_.GetFeedSize());
|
||
wake_word_detect_.Feed(data);// 将音频数据喂给唤醒词检测器
|
||
return;
|
||
}
|
||
#endif
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (audio_processor_.IsRunning()) {
|
||
ReadAudio(data, 16000, audio_processor_.GetFeedSize());
|
||
if (!data.empty()) {
|
||
int n = (int)data.size();
|
||
int64_t sum = 0;
|
||
int peak = 0;
|
||
for (int i = 0; i < n; ++i) {
|
||
int v = data[i];
|
||
int a = v < 0 ? -v : v;
|
||
sum += a;
|
||
if (a > peak) peak = a;
|
||
}
|
||
(void)sum;
|
||
// if (avg > 150 || peak > 800) {
|
||
// ESP_LOGI(TAG, "🎙️ 输入幅度: 均值=%d 峰值=%d 样本=%d", avg, peak, n);
|
||
// }
|
||
}
|
||
audio_processor_.Feed(data);
|
||
return;
|
||
}
|
||
#else
|
||
if (device_state_ == kDeviceStateListening || device_state_ == kDeviceStateDialog) {
|
||
if (send_g711a_uplink_) {
|
||
ReadAudio(data, 16000, 20 * 16000 / 1000);
|
||
if (!data.empty()) {
|
||
std::vector<int16_t> resampled(uplink_resampler_.GetOutputSamples((int)data.size()));
|
||
if (!resampled.empty()) {
|
||
uplink_resampler_.Process(data.data(), (int)data.size(), resampled.data());
|
||
}
|
||
int out_samples = (int)resampled.size();
|
||
std::vector<uint8_t> bytes(out_samples);
|
||
for (int i = 0; i < out_samples; ++i) {
|
||
int16_t s = resampled[i];
|
||
int sign = (s >> 8) & 0x80;
|
||
if (sign) s = -s;
|
||
if (s > 32635) s = 32635;
|
||
int exp = 7;
|
||
for (int mask = 0x4000; exp > 0 && (s & mask) == 0; mask >>= 1) exp--;
|
||
int mant = (s >> ((exp == 0) ? 4 : (exp + 3))) & 0x0F;
|
||
uint8_t a = (uint8_t)(sign | (exp << 4) | mant);
|
||
bytes[i] = (uint8_t)(a ^ 0xD5);
|
||
}
|
||
Schedule([this, bytes = std::move(bytes)]() {
|
||
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
||
protocol_->SendG711A(bytes);// 发送G711A音频数据
|
||
}
|
||
});
|
||
}
|
||
} else if (send_pcm_uplink_) {
|
||
ReadAudio(data, 16000, 20 * 16000 / 1000);
|
||
if (!data.empty()) {
|
||
int out_samples = (int)data.size() / 2;
|
||
std::vector<int16_t> down(out_samples);
|
||
for (int i = 0, j = 0; i < out_samples; ++i, j += 2) {
|
||
down[i] = data[j];
|
||
}
|
||
std::vector<int16_t> resampled(uplink_resampler_.GetOutputSamples((int)down.size()));
|
||
if (!resampled.empty()) {
|
||
uplink_resampler_.Process(down.data(), (int)down.size(), resampled.data());
|
||
}
|
||
std::vector<uint8_t> bytes(resampled.size() * sizeof(int16_t));
|
||
for (size_t i = 0; i < resampled.size(); ++i) {
|
||
int16_t s = resampled[i];
|
||
bytes[i * 2] = (uint8_t)(s & 0xFF);
|
||
bytes[i * 2 + 1] = (uint8_t)((s >> 8) & 0xFF);
|
||
}
|
||
Schedule([this, bytes = std::move(bytes)]() {
|
||
if (protocol_ && protocol_->IsAudioChannelOpened() && (device_state_ == kDeviceStateListening || device_state_ == kDeviceStateDialog || (listening_mode_ == kListeningModeRealtime && device_state_ == kDeviceStateSpeaking))) {
|
||
protocol_->SendPcm(bytes);
|
||
}
|
||
});
|
||
}
|
||
} else {
|
||
ReadAudio(data, 16000, 30 * 16000 / 1000);
|
||
background_task_->Schedule([this, data = std::move(data)]() mutable {
|
||
opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
|
||
Schedule([this, opus = std::move(opus)]() {
|
||
if (protocol_) {
|
||
protocol_->SendAudio(opus);
|
||
}
|
||
});
|
||
});
|
||
});
|
||
}
|
||
return;
|
||
}
|
||
#endif
|
||
vTaskDelay(pdMS_TO_TICKS(30));
|
||
}
|
||
|
||
void Application::ReadAudio(std::vector<int16_t>& data, int sample_rate, int samples) {
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
// 当音频处理器运行且存在参考通道时，保持原有双通道读取以支持AEC
|
||
if (audio_processor_.IsRunning() && codec->input_channels() == 2) {
|
||
if (codec->input_sample_rate() != sample_rate) {
|
||
data.resize(samples * codec->input_sample_rate() / sample_rate);
|
||
if (!codec->InputData(data)) {
|
||
return;
|
||
}
|
||
auto mic_channel = std::vector<int16_t>(data.size() / 2);
|
||
auto reference_channel = std::vector<int16_t>(data.size() / 2);
|
||
for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) {
|
||
mic_channel[i] = data[j];
|
||
reference_channel[i] = data[j + 1];
|
||
}
|
||
auto resampled_mic = std::vector<int16_t>(input_resampler_.GetOutputSamples(mic_channel.size()));
|
||
auto resampled_reference = std::vector<int16_t>(reference_resampler_.GetOutputSamples(reference_channel.size()));
|
||
input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data());
|
||
reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data());
|
||
data.resize(resampled_mic.size() + resampled_reference.size());
|
||
for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) {
|
||
data[j] = resampled_mic[i];
|
||
data[j + 1] = resampled_reference[i];
|
||
}
|
||
return;
|
||
} else {
|
||
data.resize(samples * codec->input_channels());
|
||
if (!codec->InputData(data)) {
|
||
return;
|
||
}
|
||
static bool first_equal_sr_dual_read_logged = false;
|
||
if (!first_equal_sr_dual_read_logged) {
|
||
ESP_LOGI(TAG, "AFE输入首包: 双通道等采样率 目标样本=%d 通道=%d 实际向量=%zu", samples, codec->input_channels(), data.size());
|
||
first_equal_sr_dual_read_logged = true;
|
||
}
|
||
return;
|
||
}
|
||
}
|
||
#endif
|
||
|
||
// 默认优先使用recorder管道读取（目标采样率16000），无参考通道需求
|
||
if (recorder_pipeline_ && sample_rate == 16000) {
|
||
int need_bytes = samples * (int)sizeof(int16_t);
|
||
int default_bytes = recorder_pipeline_get_default_read_size(recorder_pipeline_);
|
||
std::vector<int16_t> out;
|
||
out.reserve(samples);// 预分配内存空间,避免后续动态扩容
|
||
std::vector<char> buf(default_bytes);// 内存音频缓冲区,用于存储从录音管道读取的音频数据
|
||
while ((int)out.size() < samples) {
|
||
int to_read = std::min(default_bytes, (need_bytes - (int)out.size() * (int)sizeof(int16_t)));// 计算本次读取的字节数,不超过默认读取大小和剩余需要读取的字节数
|
||
if (to_read <= 0) break;// 读取到的数据大小小于等于0,跳出循环
|
||
int got = recorder_pipeline_read(recorder_pipeline_, buf.data(), to_read);// 从录音管道读取音频数据,并赋值给内存音频缓冲区
|
||
if (got <= 0) {
|
||
ESP_LOGW(TAG, "🎙️ 录音管道读取失败，未收到输入数据");
|
||
break;
|
||
}
|
||
int got_samples = got / (int)sizeof(int16_t);// 计算本次读取的样本数,即读取到的字节数除以每个样本的字节数
|
||
int16_t* p = (int16_t*)buf.data();// 将内存音频缓冲区转换为int16_t指针,方便按样本读取
|
||
for (int i = 0; i < got_samples && (int)out.size() < samples; ++i) {
|
||
out.push_back(p[i]);// 将读取到的样本添加到输出向量中,直到达到预期样本数或读取完所有数据
|
||
}
|
||
}
|
||
if (!out.empty()) {
|
||
data.assign(out.begin(), out.end());// 将输出向量中的数据赋值给输出参数data
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 回退到直接从codec读取的实现
|
||
if (codec->input_sample_rate() != sample_rate) {
|
||
data.resize(samples * codec->input_sample_rate() / sample_rate);
|
||
if (!codec->InputData(data)) {
|
||
ESP_LOGW(TAG, "🎙️ 麦克风采样失败（重采样路径），未收到输入数据");
|
||
return;
|
||
}
|
||
if (codec->input_channels() == 2) {
|
||
// 双通道约定：当前缓冲按 [主麦,参考] 排列；必须与 ALGORITHM_INPUT_FORMAT 的 M/R 顺序
|
||
// 以及 CHANNEL_FORMAT 的物理通道对应一致，否则可能导致 AEC 失效或增益反向
|
||
auto mic_channel = std::vector<int16_t>(data.size() / 2);
|
||
auto reference_channel = std::vector<int16_t>(data.size() / 2);
|
||
for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) {
|
||
mic_channel[i] = data[j];
|
||
reference_channel[i] = data[j + 1];
|
||
}
|
||
auto resampled_mic = std::vector<int16_t>(input_resampler_.GetOutputSamples(mic_channel.size()));
|
||
auto resampled_reference = std::vector<int16_t>(reference_resampler_.GetOutputSamples(reference_channel.size()));
|
||
input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data());
|
||
reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data());
|
||
data.resize(resampled_mic.size() + resampled_reference.size());
|
||
for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) {
|
||
data[j] = resampled_mic[i];
|
||
data[j + 1] = resampled_reference[i];
|
||
}
|
||
} else {
|
||
auto resampled = std::vector<int16_t>(input_resampler_.GetOutputSamples(data.size()));
|
||
input_resampler_.Process(data.data(), data.size(), resampled.data());
|
||
data = std::move(resampled);
|
||
}
|
||
} else {
|
||
data.resize(samples);
|
||
if (!codec->InputData(data)) {
|
||
ESP_LOGW(TAG, "🎙️ 麦克风采样失败（直读路径），未收到输入数据");
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
// 打断语音播报(终止播放)
|
||
void Application::AbortSpeaking(AbortReason reason) {
|
||
// 🔧 防止重复中止操作，避免竞态条件
|
||
bool expected = false;
|
||
if (!is_aborting_.compare_exchange_strong(expected, true)) {
|
||
ESP_LOGD(TAG, "AbortSpeaking already in progress, ignoring duplicate call");
|
||
return;
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🔴 Abort speaking - immediate stop");
|
||
aborted_ = true;
|
||
|
||
// 🔧 更新安全操作时间戳
|
||
last_safe_operation_.store(std::chrono::steady_clock::now());
|
||
|
||
// 🔧 修复：立即清空音频队列，确保音频播放立即停止
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
if (!audio_decode_queue_.empty()) {
|
||
ESP_LOGI(TAG, "🔴 Clearing %zu audio frames from queue", audio_decode_queue_.size());
|
||
audio_decode_queue_.clear();
|
||
|
||
// 重置音量状态
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
}
|
||
}
|
||
|
||
// ⚠️ 移除WaitForCompletion避免死锁，让后台任务通过aborted_标志自然结束
|
||
ESP_LOGI(TAG, "🔴 Audio queue cleared, background tasks will stop on next iteration");
|
||
|
||
// 🔧 修复：始终尝试发送中止消息以打断RTC下行（不受IsSafeToOperate限制）
|
||
if (protocol_) {
|
||
try {
|
||
// 🔧 额外的连接状态验证
|
||
if (protocol_->IsAudioChannelOpened()) {
|
||
protocol_->SendAbortSpeaking(reason);
|
||
ESP_LOGI(TAG, "AbortSpeaking message sent successfully");
|
||
// 更新安全操作时间戳
|
||
last_safe_operation_.store(std::chrono::steady_clock::now());
|
||
} else {
|
||
ESP_LOGW(TAG, "Audio channel not properly opened, skipping AbortSpeaking");
|
||
}
|
||
} catch (const std::exception& e) {
|
||
ESP_LOGW(TAG, "Failed to send AbortSpeaking message: %s", e.what());
|
||
}
|
||
} else {
|
||
ESP_LOGD(TAG, "Skipping AbortSpeaking message - protocol_ is null");
|
||
}
|
||
|
||
// 🔧 确保中止窗口后恢复播放流程，避免长时间阻塞导致WS音频无法播放
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(120));
|
||
aborted_ = false;
|
||
ESP_LOGI(TAG, "🔵 Abort window ended, resume playback tasks");
|
||
});
|
||
|
||
// 🔧 重置中止标志，允许后续操作
|
||
is_aborting_.store(false);
|
||
}
|
||
|
||
// 发送讲故事请求 webscoket协议
|
||
void Application::SendStoryRequest() {
|
||
if (!websocket_protocol_) {
|
||
InitializeWebsocketProtocol();// 初始化WebSocket协议
|
||
if (!websocket_protocol_) {
|
||
ESP_LOGW(TAG, "WebSocket协议初始化失败");
|
||
return;
|
||
}
|
||
}
|
||
Schedule([this]() {
|
||
ws_downlink_enabled_.store(true);
|
||
// 确保音频通道已打开
|
||
if (!websocket_protocol_->IsAudioChannelOpened()) {
|
||
websocket_protocol_->OpenAudioChannel();// 打开音频通道
|
||
}
|
||
websocket_protocol_->SendStoryRequest();// 发送故事请求
|
||
ESP_LOGI(TAG, "通过WebSocket发送的故事请求！");
|
||
});
|
||
}
|
||
|
||
// 设置监听模式
|
||
void Application::SetListeningMode(ListeningMode mode) {
|
||
ESP_LOGI(TAG, "Setting listening mode to %d", (int)mode);// 打印设置监听模式日志
|
||
listening_mode_ = mode;
|
||
SetDeviceState(kDeviceStateListening);
|
||
}
|
||
|
||
// 设置设备状态
|
||
void Application::SetDeviceState(DeviceState state) {
|
||
if (device_state_ == state) {
|
||
return;
|
||
}
|
||
|
||
clock_ticks_ = 0;
|
||
auto previous_state = device_state_;// 记录上一个设备状态
|
||
device_state_ = state;// 设置设备状态
|
||
if (state == kDeviceStateDialog) {
|
||
StartDialogWatchdog();
|
||
} else if (previous_state == kDeviceStateDialog) {
|
||
StopDialogWatchdog();
|
||
}
|
||
ESP_LOGI(TAG, "打印设置设备状态日志: %s", STATE_STRINGS[device_state_]);// 打印设置设备状态日志
|
||
// The state is changed, wait for all background tasks to finish
|
||
background_task_->WaitForCompletion();
|
||
|
||
auto& board = Board::GetInstance();
|
||
auto display = board.GetDisplay();
|
||
auto led = board.GetLed();
|
||
led->OnStateChanged();
|
||
|
||
// 检查是否正在进行BluFi配网，配网时禁止播放待命音效(新增代码)
|
||
// =================================================================
|
||
bool is_blufi_provisioning = false;
|
||
if (Board::GetInstance().GetBoardType() == "wifi") {
|
||
auto& wifi_board = static_cast<WifiBoard&>(Board::GetInstance());
|
||
is_blufi_provisioning = wifi_board.IsBluFiProvisioningActive();
|
||
}
|
||
// =================================================================
|
||
|
||
switch (state) {
|
||
case kDeviceStateUnknown:
|
||
case kDeviceStateIdle:
|
||
display->SetStatus(Lang::Strings::STANDBY);
|
||
display->SetEmotion("neutral");
|
||
|
||
|
||
// // 只有从非待命状态进入待命状态时才播放待命音效，避免重复播放(原来的代码)
|
||
// if (previous_state != kDeviceStateIdle &&
|
||
// previous_state != kDeviceStateUnknown &&
|
||
// previous_state != kDeviceStateWifiConfiguring) {
|
||
// ESP_LOGI(TAG, "Entering idle state, playing standby sound");
|
||
// PlaySound(Lang::Sounds::P3_DAIMING);
|
||
// }
|
||
// 开机后 进入待命状态 播报 卡卡正在待命(配网模式下不播报“卡卡正在待命”)-新增代码
|
||
//=====================================================================================
|
||
if (previous_state != kDeviceStateIdle && previous_state != kDeviceStateUnknown &&
|
||
previous_state != kDeviceStateWifiConfiguring && !is_blufi_provisioning && !IsLowBatteryTransition()) {
|
||
ESP_LOGI(TAG, "Entering idle state, playing standby sound");
|
||
// PlaySound(Lang::Sounds::P3_DAIMING); 原有 待命 播报
|
||
if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){
|
||
PlaySound(Lang::Sounds::P3_KAKA_DAIMING);
|
||
}
|
||
else if(strcmp(CONFIG_DEVICE_ROLE, "RTC_Test") == 0){
|
||
PlaySound(Lang::Sounds::P3_LALA_DAIMING);
|
||
}
|
||
}
|
||
//=====================================================================================
|
||
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.Stop();
|
||
#endif
|
||
#if 1
|
||
if (recorder_pipeline_) {
|
||
recorder_pipeline_close(recorder_pipeline_);
|
||
recorder_pipeline_ = nullptr;
|
||
}
|
||
#endif
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Start();
|
||
#endif
|
||
// 设备开机后首次进入idle状态时，自动检测NVS中的位置并调用API后设置当前位置
|
||
if (!first_idle_location_checked_) {
|
||
first_idle_location_checked_ = true;// 首次查询城市天气
|
||
Schedule([]() {
|
||
AutoDetectAndSetLocation();// 自动检测并设置当前位置
|
||
});
|
||
}
|
||
break;
|
||
case kDeviceStateConnecting:
|
||
display->SetStatus(Lang::Strings::CONNECTING);
|
||
display->SetEmotion("neutral");
|
||
display->SetChatMessage("system", "");
|
||
break;
|
||
case kDeviceStateListening:
|
||
display->SetStatus(Lang::Strings::LISTENING);
|
||
display->SetEmotion("neutral");
|
||
|
||
// 关键修复：只有在非音效播放状态下才重置音量，避免中断正在播放的音效
|
||
// 检查是否有音频正在播放，如果有则延迟重置音量
|
||
if (IsAudioQueueEmpty()) {
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
} else {
|
||
// 如果有音频正在播放，延迟重置音量
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(500)); // 等待音效播放完成
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
});
|
||
}
|
||
|
||
// Update the IoT states before sending the start listening command
|
||
UpdateIotStates();
|
||
|
||
// Make sure the audio processor is running
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (!audio_processor_.IsRunning()) {
|
||
#else
|
||
if (true) {
|
||
#endif
|
||
// 🔧 关键修复：检查协议连接状态，防止发送到无效连接
|
||
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
||
// Send the start listening command
|
||
protocol_->SendStartListening(listening_mode_);
|
||
if (listening_mode_ == kListeningModeAutoStop && previous_state == kDeviceStateSpeaking) {
|
||
// FIXME: Wait for the speaker to empty the buffer
|
||
vTaskDelay(pdMS_TO_TICKS(120));
|
||
}
|
||
opus_encoder_->ResetState();
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Stop();
|
||
#endif
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.Start();
|
||
#endif
|
||
if (!recorder_pipeline_) {
|
||
recorder_pipeline_ = recorder_pipeline_open();
|
||
recorder_pipeline_run(recorder_pipeline_);
|
||
}
|
||
} else {
|
||
ESP_LOGW(TAG, "Audio channel not available, skipping SendStartListening");// 音频通道未打开，跳过发送开始聆听命令
|
||
// 保持在聆听状态，不自动回退到idle状态
|
||
ESP_LOGI(TAG, "🔵 Staying in listening state despite audio channel unavailable");
|
||
}
|
||
}
|
||
break;
|
||
case kDeviceStateSpeaking:
|
||
display->SetStatus(Lang::Strings::SPEAKING);
|
||
|
||
if (listening_mode_ != kListeningModeRealtime) {
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.Stop();
|
||
#endif
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Start();
|
||
#endif
|
||
} else {
|
||
// 在实时模式下，保持audio_processor运行以检测语音打断
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (!audio_processor_.IsRunning()) {
|
||
audio_processor_.Start();
|
||
}
|
||
#endif
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Stop();
|
||
#endif
|
||
}
|
||
ResetDecoder();
|
||
break;
|
||
case kDeviceStateDialog:
|
||
display->SetStatus(Lang::Strings::SPEAKING);
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (!audio_processor_.IsRunning()) {
|
||
audio_processor_.Start();
|
||
}
|
||
#endif
|
||
#if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD
|
||
wake_word_detect_.Stop();
|
||
#endif
|
||
{
|
||
auto codec2 = Board::GetInstance().GetAudioCodec();// 获取音频编解码器
|
||
codec2->EnableOutput(true);// 启用音频输出
|
||
last_audible_output_time_ = std::chrono::steady_clock::now();// 更新最后有声音输出的时间
|
||
}
|
||
if (!recorder_pipeline_) {
|
||
recorder_pipeline_ = recorder_pipeline_open();
|
||
recorder_pipeline_run(recorder_pipeline_);
|
||
}
|
||
break;
|
||
default:
|
||
// Do nothing
|
||
break;
|
||
}
|
||
}
|
||
|
||
void Application::ResetDecoder() {
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
opus_decoder_->ResetState();
|
||
audio_decode_queue_.clear();
|
||
last_output_time_ = std::chrono::steady_clock::now();
|
||
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
codec->EnableOutput(true);
|
||
}
|
||
|
||
void Application::SetDecodeSampleRate(int sample_rate, int frame_duration) {
|
||
if (opus_decoder_->sample_rate() == sample_rate && opus_decoder_->duration_ms() == frame_duration) {
|
||
return;
|
||
}
|
||
|
||
opus_decoder_.reset();
|
||
opus_decoder_ = std::make_unique<OpusDecoderWrapper>(sample_rate, 1, frame_duration);
|
||
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
if (opus_decoder_->sample_rate() != codec->output_sample_rate()) {
|
||
ESP_LOGI(TAG, "Resampling audio from %d to %d", opus_decoder_->sample_rate(), codec->output_sample_rate());
|
||
output_resampler_.Configure(opus_decoder_->sample_rate(), codec->output_sample_rate());
|
||
}
|
||
}
|
||
|
||
void Application::UpdateIotStates() {
|
||
auto& thing_manager = iot::ThingManager::GetInstance();
|
||
std::string states;
|
||
if (thing_manager.GetStatesJson(states, true)) {
|
||
protocol_->SendIotStates(states);
|
||
}
|
||
}
|
||
|
||
void Application::Reboot() {
|
||
ESP_LOGI(TAG, "Rebooting...");
|
||
esp_restart();
|
||
}
|
||
|
||
// 唤醒词触发函数
|
||
void Application::WakeWordInvoke(const std::string& wake_word) {
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
ToggleChatState();
|
||
Schedule([this, wake_word]() {
|
||
if (protocol_) {
|
||
protocol_->SendWakeWordDetected(wake_word);
|
||
}
|
||
});
|
||
} else if (device_state_ == kDeviceStateSpeaking) {
|
||
//AbortSpeakingAndReturnToListening();// 使用唤醒词打断时立即切换到聆听状态
|
||
Schedule([this]() {
|
||
AbortSpeaking(kAbortReasonNone);
|
||
});
|
||
} else if (device_state_ == kDeviceStateListening) {
|
||
Schedule([this]() {
|
||
if (protocol_) {
|
||
protocol_->CloseAudioChannel();
|
||
}
|
||
});
|
||
}
|
||
}
|
||
|
||
bool Application::CanEnterSleepMode() {
|
||
if (device_state_ != kDeviceStateIdle) {
|
||
return false;
|
||
}
|
||
|
||
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
||
return false;
|
||
}
|
||
|
||
// Now it is safe to enter sleep mode
|
||
return true;
|
||
}
|
||
void Application::WaitForAudioPlayback() {
|
||
// 等待 audio_decode_queue_ 清空且音频输出完成
|
||
auto codec = Board::GetInstance().GetAudioCodec();
|
||
int timeout_count = 0;
|
||
const int max_timeout = 150; // 3秒超时 (150 * 20ms = 3000ms)
|
||
|
||
while (timeout_count < max_timeout) {
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
if (audio_decode_queue_.empty()) {
|
||
// 检查音频输出是否已关闭或静音
|
||
if (!codec->output_enabled() || device_state_ != kDeviceStateSpeaking) {
|
||
ESP_LOGI(TAG, "Audio playback completed");
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
vTaskDelay(pdMS_TO_TICKS(20));
|
||
timeout_count++;
|
||
}
|
||
|
||
if (timeout_count >= max_timeout) {
|
||
ESP_LOGW(TAG, "WaitForAudioPlayback timeout after 3 seconds");
|
||
}
|
||
}
|
||
|
||
bool Application::IsAudioQueueEmpty() {
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
return audio_decode_queue_.empty();
|
||
}
|
||
|
||
void Application::ClearAudioQueue() {
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.clear();
|
||
audio_paused_ = false; // 清除暂停状态
|
||
// ESP_LOGI(TAG, "🧹 音频播放队列已清空，暂停状态已清除");
|
||
ESP_LOGI(TAG, "🎵 测试模式：音频开始播放，等待播放完成"); // 生产测试打印
|
||
|
||
|
||
// 重新启用音频编解码器输出，确保后续音频能正常播放
|
||
auto& board = Board::GetInstance();
|
||
auto codec = board.GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(true);
|
||
// ESP_LOGI(TAG, "🔧 音频编解码器输出已重新启用");
|
||
ESP_LOGI(TAG, "✅ 测试模式：音频播放完成"); // 生产测试打印
|
||
}
|
||
}
|
||
|
||
// 🔧 检查当前是否可以安全执行操作
|
||
bool Application::IsSafeToOperate() {
|
||
// 检查是否正在执行中止操作
|
||
if (is_aborting_.load()) {
|
||
return false;
|
||
}
|
||
|
||
// 检查最近是否有操作过于频繁
|
||
auto now = std::chrono::steady_clock::now();
|
||
auto last_op = last_safe_operation_.load();
|
||
auto time_diff = std::chrono::duration_cast<std::chrono::milliseconds>(now - last_op);
|
||
|
||
// 如果距离上次操作少于50ms，认为可能存在竞态风险
|
||
if (time_diff.count() < 50) {
|
||
ESP_LOGD(TAG, "Operation too frequent, waiting for safety");
|
||
return false;
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
void Application::StopAudioProcessor() {
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.Stop();
|
||
#endif
|
||
}
|
||
|
||
// 🔴 专门处理从说话状态到空闲状态的切换
|
||
void Application::AbortSpeakingAndReturnToIdle() {
|
||
ESP_LOGI(TAG, "🔴 AbortSpeakingAndReturnToIdle: Starting transition from speaking to idle state");
|
||
ESP_LOGI(TAG, "📊 当前设备状态: %s", STATE_STRINGS[device_state_]);
|
||
ESP_LOGI(TAG, "🎯 目标状态: idle (空闲状态)");
|
||
|
||
// 检查当前状态是否为说话状态
|
||
if (device_state_ != kDeviceStateSpeaking) {
|
||
ESP_LOGW(TAG, "🔴 AbortSpeakingAndReturnToIdle: Device not in speaking state, current state: %s", STATE_STRINGS[device_state_]);
|
||
return;
|
||
}
|
||
|
||
ESP_LOGI(TAG, "✅ 状态检查通过，当前处于说话状态");
|
||
|
||
// 检查操作安全性
|
||
if (!IsSafeToOperate()) {
|
||
ESP_LOGW(TAG, "🔴 AbortSpeakingAndReturnToIdle: Operation not safe, scheduling retry");
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
AbortSpeakingAndReturnToIdle();
|
||
});
|
||
return;
|
||
}
|
||
|
||
// 更新安全操作时间戳
|
||
last_safe_operation_.store(std::chrono::steady_clock::now());
|
||
ESP_LOGI(TAG, "⏰ 安全操作时间戳已更新");
|
||
|
||
// 立即停止音频处理
|
||
ESP_LOGI(TAG, "🔇 开始停止音频处理");
|
||
{
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
if (!audio_decode_queue_.empty()) {
|
||
ESP_LOGI(TAG, "🗑️ 清空音频队列，当前队列大小: %zu", audio_decode_queue_.size());
|
||
audio_decode_queue_.clear();
|
||
current_speaker_volume_ = 0.0f;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
#endif
|
||
ESP_LOGI(TAG, "✅ 音频队列已清空，音量已重置为0");
|
||
} else {
|
||
ESP_LOGI(TAG, "ℹ️ 音频队列已为空，无需清空");
|
||
}
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🔴 AbortSpeakingAndReturnToIdle: Sending abort message to server");
|
||
|
||
// 发送中止消息给服务器
|
||
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
||
ESP_LOGI(TAG, "📡 WebSocket连接正常，发送中止消息");
|
||
try {
|
||
protocol_->SendAbortSpeaking(kAbortReasonNone);
|
||
ESP_LOGI(TAG, "✅ 中止消息发送成功");
|
||
} catch (const std::exception& e) {
|
||
ESP_LOGW(TAG, "❌ 发送中止消息失败: %s", e.what());
|
||
}
|
||
|
||
// 延迟100ms后主动关闭连接，确保服务器有时间处理中止消息
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
ESP_LOGI(TAG, "⏳ 延迟100ms后开始主动关闭WebSocket连接");
|
||
ESP_LOGI(TAG, "🔌 执行主动断开WebSocket连接");
|
||
if (protocol_) {
|
||
protocol_->CloseAudioChannel();
|
||
ESP_LOGI(TAG, "✅ CloseAudioChannel调用完成");
|
||
} else {
|
||
ESP_LOGW(TAG, "⚠️ protocol_为空，无法关闭音频通道");
|
||
}
|
||
});
|
||
} else {
|
||
ESP_LOGW(TAG, "⚠️ WebSocket连接不可用，强制关闭连接");
|
||
if (protocol_) {
|
||
ESP_LOGI(TAG, "🔌 强制执行WebSocket断开");
|
||
protocol_->CloseAudioChannel();
|
||
ESP_LOGI(TAG, "✅ 强制断开完成");
|
||
} else {
|
||
ESP_LOGW(TAG, "❌ protocol_为空，无法执行断开操作");
|
||
}
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🎯 主动断开流程已启动，等待OnAudioChannelClosed回调触发状态转换");
|
||
ESP_LOGI(TAG, "📋 预期流程: WebSocket断开 → 回调触发 → 转换到idle状态 → 播放待机音");
|
||
}
|
||
|
||
// 🔵 专门处理从说话状态到聆听状态的切换
|
||
void Application::AbortSpeakingAndReturnToListening() {
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Starting transition from speaking to listening state (断开连接方案)");
|
||
|
||
// 检查当前状态是否为说话状态或可切换状态
|
||
// =========================================================================================
|
||
if (device_state_ != kDeviceStateSpeaking && device_state_ != kDeviceStateListening && device_state_ != kDeviceStateIdle) {
|
||
ESP_LOGW(TAG, "🔵 AbortSpeakingAndReturnToListening: Device not in valid state for transition, current state: %s", STATE_STRINGS[device_state_]);
|
||
return;
|
||
}
|
||
// 如果已经在listening状态，直接返回避免重复切换
|
||
if (device_state_ == kDeviceStateListening) {
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Already in listening state, skipping transition");
|
||
return;
|
||
}
|
||
// 🔧 检查并处理音频播放状态(BOOT按键优化方案)
|
||
if (!audio_paused_ && device_state_ == kDeviceStateSpeaking) {
|
||
ESP_LOGI(TAG, "🔵 检测到播放状态，一次按键完成暂停和状态切换");
|
||
|
||
// 第一步：禁用音频输出(立即停止播放)
|
||
auto& board = Board::GetInstance();// 获取音频编解码器
|
||
auto codec = board.GetAudioCodec();// 获取音频编解码器
|
||
if (codec) {
|
||
codec->EnableOutput(false);// 暂停时禁用音频编解码器输出
|
||
ESP_LOGI(TAG, "🔧 暂停时禁用音频编解码器输出");
|
||
}
|
||
// 第二步：切换到暂停状态
|
||
audio_paused_ = true;
|
||
ESP_LOGI(TAG, "✅ 已切换到暂停状态");
|
||
// 第三步：立即执行状态切换逻辑（不返回，继续执行下面的代码）
|
||
ESP_LOGI(TAG, "🔵 继续执行状态切换到聆听状态");
|
||
}
|
||
|
||
// 🔧 检查并处理音频暂停状态(BOOT按键优化方案)
|
||
if (audio_paused_) {
|
||
ESP_LOGI(TAG, "🔵 检测到音频暂停状态，应用BOOT按键优化方案");
|
||
audio_paused_ = false;
|
||
ESP_LOGI(TAG, "✅ 音频暂停状态已清除");
|
||
|
||
// 🔧 关键优化：清空音频播放队列，避免播放暂停时残留的音频
|
||
std::unique_lock<std::mutex> lock(mutex_);
|
||
audio_decode_queue_.clear();
|
||
lock.unlock();
|
||
ESP_LOGI(TAG, "🧹 已清空音频播放队列，避免播放残留音频");
|
||
|
||
// BOOT按键切换时的优化方案：确保音频系统能正常响应状态切换
|
||
auto& board = Board::GetInstance();
|
||
auto codec = board.GetAudioCodec();
|
||
if (codec) {
|
||
codec->EnableOutput(true);
|
||
ESP_LOGI(TAG, "🔧 为状态切换重新启用音频编解码器输出");// 重新启用输出,后续可以播放
|
||
}
|
||
// 🔧 关键修复：强制停止音频处理器，确保后续状态切换时能重新启动
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (audio_processor_.IsRunning()) {
|
||
ESP_LOGI(TAG, "🔧 强制停止音频处理器以确保状态切换成功");
|
||
audio_processor_.Stop();
|
||
}
|
||
#endif
|
||
|
||
// 🔧 音频暂停状态下直接切换，避免复杂的异步处理
|
||
ESP_LOGI(TAG, "🔵 音频暂停状态下直接执行状态切换");
|
||
|
||
// 播放提示音
|
||
if (codec && codec->output_enabled()) {
|
||
ESP_LOGI(TAG, "播放提示音：卡卡在呢");
|
||
// PlaySound(Lang::Sounds::P3_KAKAZAINNE); 原有蜡笔小新 音色播报
|
||
if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){
|
||
PlaySound(Lang::Sounds::P3_KAKA_ZAINNE);
|
||
}
|
||
else if(strcmp(CONFIG_DEVICE_ROLE, "RTC_Test") == 0){
|
||
PlaySound(Lang::Sounds::P3_LALA_ZAINNE);
|
||
}
|
||
|
||
|
||
// 简化等待逻辑
|
||
vTaskDelay(pdMS_TO_TICKS(620)); // 等待音效播放完成
|
||
ESP_LOGI(TAG, "音频播放完成");
|
||
}
|
||
|
||
// 直接切换到聆听状态
|
||
SetDeviceState(kDeviceStateListening);
|
||
ESP_LOGI(TAG, "🔵 音频暂停状态下状态切换完成");
|
||
return;
|
||
}
|
||
// =========================================================================================
|
||
|
||
// 检查操作安全性
|
||
if (!IsSafeToOperate()) {
|
||
ESP_LOGW(TAG, "🔵 AbortSpeakingAndReturnToListening: Operation not safe, scheduling retry");
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
AbortSpeakingAndReturnToListening();
|
||
});
|
||
return;
|
||
}
|
||
|
||
// 更新安全操作时间戳
|
||
last_safe_operation_.store(std::chrono::steady_clock::now());
|
||
|
||
// 立即停止音频处理器和清空音频队列
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
if (audio_processor_.IsRunning()) {
|
||
ESP_LOGI(TAG, "🔵 停止音频处理器");
|
||
audio_processor_.Stop();
|
||
}
|
||
|
||
// 清空音频队列并重置音量
|
||
if (!IsAudioQueueEmpty()) {
|
||
ESP_LOGI(TAG, "🔵 清空音频队列并重置音量");
|
||
while (!IsAudioQueueEmpty()) {
|
||
vTaskDelay(pdMS_TO_TICKS(10));
|
||
}
|
||
current_speaker_volume_ = 0.0f;
|
||
audio_processor_.SetSpeakerVolume(0.0f);
|
||
ESP_LOGI(TAG, "✅ 音频队列已清空，音量已重置为0");
|
||
} else {
|
||
ESP_LOGI(TAG, "ℹ️ 音频队列已为空，无需清空");
|
||
}
|
||
#endif
|
||
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Sending abort message to server");
|
||
|
||
// 发送中止消息给服务器
|
||
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
||
ESP_LOGI(TAG, "📡 WebSocket连接正常，发送中止消息");
|
||
try {
|
||
protocol_->SendAbortSpeaking(kAbortReasonVoiceInterrupt);
|
||
ESP_LOGI(TAG, "✅ 中止消息发送成功");
|
||
} catch (const std::exception& e) {
|
||
ESP_LOGW(TAG, "❌ 发送中止消息失败: %s", e.what());
|
||
}
|
||
|
||
// 延迟100ms后播放音效并直接切换到聆听状态，不关闭WebSocket连接
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
ESP_LOGI(TAG, "⏳ 延迟100ms后播放音效并切换到聆听状态");
|
||
|
||
// 先播放"卡卡在呢"音效
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Playing KAKAZAINNE sound");
|
||
|
||
// 🔧 修复：STATE:，确保硬件状态正确
|
||
auto& board = Board::GetInstance();
|
||
auto audio_codec = board.GetAudioCodec();
|
||
ESP_LOGI(TAG, "强制重新初始化音频输出");
|
||
audio_codec->EnableOutput(false); // 先关闭音频输出
|
||
vTaskDelay(pdMS_TO_TICKS(50)); // 短暂延迟让硬件复位
|
||
audio_codec->EnableOutput(true); // 再开启，强制硬件重新初始化
|
||
|
||
// 🔧 检查音频资源是否可用
|
||
if (audio_codec->output_enabled()) {
|
||
ESP_LOGI(TAG, "播放提示音：卡卡在呢");
|
||
ResetDecoder(); // 🔧 关键修复：重置解码器状态，清除残留
|
||
|
||
// 获取当前系统音量并临时设置以确保音效能播放
|
||
float system_volume = audio_codec ? (audio_codec->output_volume() / 100.0f) : 0.7f; // 默认70%
|
||
current_speaker_volume_ = system_volume;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(system_volume);
|
||
ESP_LOGI(TAG, "✅ 音量设置成功: %.2f", system_volume);
|
||
#endif
|
||
|
||
// PlaySound(Lang::Sounds::P3_KAKAZAINNE); 原有蜡笔小新 音色播报
|
||
if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){
|
||
PlaySound(Lang::Sounds::P3_KAKA_ZAINNE);
|
||
}
|
||
else if(strcmp(CONFIG_DEVICE_ROLE, "RTC_Test") == 0){
|
||
PlaySound(Lang::Sounds::P3_LALA_ZAINNE);
|
||
}
|
||
|
||
// 🔧 修复：使用改进的等待逻辑，确保音频真正播放完成
|
||
ESP_LOGI(TAG, "等待音频播放完成...");
|
||
vTaskDelay(pdMS_TO_TICKS(100)); // 给音频足够的时间开始播放
|
||
|
||
// 等待音频队列清空 + 额外缓冲时间确保I2S硬件完成输出
|
||
int timeout_count = 0;
|
||
const int max_timeout = 150; // 3秒超时
|
||
|
||
while (timeout_count < max_timeout) {
|
||
if (IsAudioQueueEmpty()) {
|
||
// 队列清空后，再等待500ms确保I2S硬件完成输出
|
||
ESP_LOGI(TAG, "音频队列已清空，等待硬件输出完成...");
|
||
vTaskDelay(pdMS_TO_TICKS(500));
|
||
ESP_LOGI(TAG, "音频播放完成");
|
||
break;
|
||
}
|
||
vTaskDelay(pdMS_TO_TICKS(20));
|
||
timeout_count++;
|
||
}
|
||
|
||
if (timeout_count >= max_timeout) {
|
||
ESP_LOGW(TAG, "等待音频播放超时，继续状态切换");
|
||
}
|
||
} else {
|
||
ESP_LOGW(TAG, "音频输出无法启用，跳过提示音播放");
|
||
}
|
||
|
||
// 直接切换到聆听状态，音频播放已在上面完成
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Switching to listening state (保持WebSocket连接)");
|
||
SetDeviceState(kDeviceStateListening);
|
||
});
|
||
} else {
|
||
ESP_LOGW(TAG, "⚠️ WebSocket连接不可用，直接切换状态");
|
||
|
||
// 直接播放音效并切换状态
|
||
Schedule([this]() {
|
||
vTaskDelay(pdMS_TO_TICKS(100));
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Playing KAKAZAINNE sound");
|
||
|
||
// 🔧 修复：强制重新初始化音频输出，确保硬件状态正确
|
||
auto& board = Board::GetInstance();
|
||
auto audio_codec = board.GetAudioCodec();
|
||
ESP_LOGI(TAG, "强制重新初始化音频输出");
|
||
audio_codec->EnableOutput(false); // 先关闭音频输出
|
||
vTaskDelay(pdMS_TO_TICKS(50)); // 短暂延迟让硬件复位
|
||
audio_codec->EnableOutput(true); // 再开启，强制硬件重新初始化
|
||
|
||
// 🔧 检查音频资源是否可用
|
||
if (audio_codec->output_enabled()) {
|
||
ESP_LOGI(TAG, "播放提示音：卡卡在呢");
|
||
ResetDecoder(); // 🔧 关键修复：重置解码器状态，清除残留
|
||
|
||
// 获取当前系统音量并临时设置以确保音效能播放
|
||
float system_volume = audio_codec ? (audio_codec->output_volume() / 100.0f) : 0.7f; // 默认70%
|
||
current_speaker_volume_ = system_volume;
|
||
#if CONFIG_USE_AUDIO_PROCESSOR
|
||
audio_processor_.SetSpeakerVolume(system_volume);
|
||
ESP_LOGI(TAG, "✅ 音量设置成功: %.2f", system_volume);
|
||
#endif
|
||
|
||
// PlaySound(Lang::Sounds::P3_KAKAZAINNE); 原有蜡笔小新 音色播报
|
||
if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){
|
||
PlaySound(Lang::Sounds::P3_KAKA_ZAINNE);
|
||
}
|
||
else if(strcmp(CONFIG_DEVICE_ROLE, "RTC_Test") == 0){
|
||
PlaySound(Lang::Sounds::P3_LALA_ZAINNE);
|
||
}
|
||
|
||
// 🔧 修复：使用改进的等待逻辑，确保音频真正播放完成
|
||
ESP_LOGI(TAG, "等待音频播放完成...");
|
||
vTaskDelay(pdMS_TO_TICKS(100)); // 给音频足够的时间开始播放
|
||
|
||
// 等待音频队列清空 + 额外缓冲时间确保I2S硬件完成输出
|
||
int timeout_count = 0;
|
||
const int max_timeout = 150; // 3秒超时
|
||
|
||
while (timeout_count < max_timeout) {
|
||
if (IsAudioQueueEmpty()) {
|
||
// 队列清空后，再等待500ms确保I2S硬件完成输出
|
||
ESP_LOGI(TAG, "音频队列已清空，等待硬件输出完成...");
|
||
vTaskDelay(pdMS_TO_TICKS(500));
|
||
ESP_LOGI(TAG, "音频播放完成");
|
||
break;
|
||
}
|
||
vTaskDelay(pdMS_TO_TICKS(20));
|
||
timeout_count++;
|
||
}
|
||
|
||
if (timeout_count >= max_timeout) {
|
||
ESP_LOGW(TAG, "等待音频播放超时，继续状态切换");
|
||
}
|
||
} else {
|
||
ESP_LOGW(TAG, "音频输出无法启用，跳过提示音播放");
|
||
}
|
||
|
||
// 直接切换到聆听状态，音频播放已在上面完成
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Switching to listening state");
|
||
SetDeviceState(kDeviceStateListening);
|
||
});
|
||
}
|
||
|
||
ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Transition initiated - keeping WebSocket connection and switching to listening");
|
||
}
|
||
|
||
// 姿态传感器接口实现
|
||
bool Application::IsImuSensorAvailable() {
|
||
auto& board = Board::GetInstance();
|
||
if (board.GetBoardType() == "movecall-moji-esp32s3") {
|
||
auto& moji_board = static_cast<MovecallMojiESP32S3&>(board);
|
||
return moji_board.IsImuInitialized();
|
||
}
|
||
return false;
|
||
}
|
||
|
||
bool Application::GetImuData(float* acc_x, float* acc_y, float* acc_z,
|
||
float* gyro_x, float* gyro_y, float* gyro_z,
|
||
float* temperature) {
|
||
auto& board = Board::GetInstance();
|
||
if (board.GetBoardType() == "movecall-moji-esp32s3") {
|
||
auto& moji_board = static_cast<MovecallMojiESP32S3&>(board);
|
||
qmi8658a_data_t imu_data;
|
||
if (moji_board.GetImuData(&imu_data)) {
|
||
if (acc_x) *acc_x = imu_data.acc_x;
|
||
if (acc_y) *acc_y = imu_data.acc_y;
|
||
if (acc_z) *acc_z = imu_data.acc_z;
|
||
if (gyro_x) *gyro_x = imu_data.gyro_x;
|
||
if (gyro_y) *gyro_y = imu_data.gyro_y;
|
||
if (gyro_z) *gyro_z = imu_data.gyro_z;
|
||
if (temperature) *temperature = imu_data.temperature;
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
void Application::ClearDialogIdleSkipSession() {
|
||
// 清除内存中的跳过标志
|
||
skip_dialog_idle_session_ = false;
|
||
|
||
// 清除NVS中的标志
|
||
Settings sys("system", true);
|
||
sys.SetInt("reboot_dlg_idle", 0);
|
||
sys.SetInt("reboot_origin", 0);
|
||
sys.Commit();
|
||
ESP_LOGI(TAG, "跳过对话待机会话标志已清除");
|
||
}
|
||
|
||
void Application::SetDialogUploadEnabled(bool enabled) {
|
||
dialog_upload_enabled_ = enabled;
|
||
ESP_LOGI(TAG, "对话上传状态已设置为: %s", enabled ? "启用" : "禁用");
|
||
}
|
||
|
||
void Application::OnMotionDetected() {
|
||
ESP_LOGI(TAG, "Motion detected by IMU sensor");
|
||
|
||
// 如果设备处于空闲状态，可以触发一些动作
|
||
if (device_state_ == kDeviceStateIdle) {
|
||
// 例如：显示运动检测提示
|
||
auto& board = Board::GetInstance();
|
||
auto display = board.GetDisplay();
|
||
display->SetChatMessage("system", "检测到运动");
|
||
|
||
// 可以在这里添加更多的运动检测处理逻辑
|
||
// 比如：唤醒设备、记录运动数据等
|
||
}
|
||
}
|
||
|
||
void Application::SetLowBatteryTransition(bool value) {
|
||
is_low_battery_transition_.store(value);// 设置低电量过渡状态
|
||
}
|
||
|
||
bool Application::IsLowBatteryTransition() const {
|
||
return is_low_battery_transition_.load();// 获取低电量过渡状态
|
||
}
|
||
|
||
// 🌐 初始化WebSocket协议（RTC连接成功后调用）
|
||
void Application::InitializeWebsocketProtocol() {
|
||
ESP_LOGI(TAG, "🌐 开始初始化WebSocket协议...");
|
||
|
||
// 检查是否已经初始化过
|
||
if (websocket_protocol_) {
|
||
ESP_LOGW(TAG, "⚠️ WebSocket协议已经初始化，跳过重复初始化");
|
||
return;
|
||
}
|
||
|
||
// 创建WebsocketProtocol实例
|
||
ESP_LOGI(TAG, "🔧 创建WebsocketProtocol实例");
|
||
websocket_protocol_ = std::make_unique<WebsocketProtocol>();
|
||
websocket_protocol_->SetPrimary(false);
|
||
websocket_protocol_->OnIncomingAudio([this](std::vector<uint8_t>&& data) {
|
||
if (!ws_downlink_enabled_.load()) {
|
||
return;
|
||
}
|
||
ws_playback_active_.store(true);
|
||
std::lock_guard<std::mutex> lock(mutex_);
|
||
size_t len = data.size();
|
||
audio_decode_queue_.emplace_back(std::move(data));
|
||
ESP_LOGD(TAG, "WS辅助音频入队: 字节=%zu 队列大小=%zu", len, audio_decode_queue_.size());
|
||
});
|
||
websocket_protocol_->OnIncomingJson([this](const cJSON* root) {
|
||
auto type = cJSON_GetObjectItem(root, "type");
|
||
if (type && cJSON_IsString(type) && type->valuestring) {
|
||
ESP_LOGD(TAG, "WS辅助JSON消息: %s", type->valuestring);
|
||
if (strcmp(type->valuestring, "hello") == 0) {
|
||
auto audio_params = cJSON_GetObjectItem(root, "audio_params");
|
||
if (audio_params && cJSON_IsObject(audio_params)) {
|
||
auto sr = cJSON_GetObjectItem(audio_params, "sample_rate");
|
||
auto fd = cJSON_GetObjectItem(audio_params, "frame_duration");
|
||
int sample_rate = (sr && cJSON_IsNumber(sr)) ? sr->valueint : 16000;
|
||
int frame_duration = (fd && cJSON_IsNumber(fd)) ? fd->valueint : 60;
|
||
SetDecodeSampleRate(sample_rate, frame_duration);
|
||
} else {
|
||
SetDecodeSampleRate(16000, 60);
|
||
}
|
||
}
|
||
}
|
||
});
|
||
|
||
// 启动WebSocket协议
|
||
ESP_LOGI(TAG, "🚀 启动WebSocket协议");
|
||
websocket_protocol_->Start();// 启动WebSocket协议
|
||
|
||
ESP_LOGI(TAG, "✅ WebSocket协议初始化完成");
|
||
}
|
||
|
||
// void Application::SendTextViaWebsocket(const std::string& text) {
|
||
// Schedule([this, text]() {
|
||
// if (websocket_protocol_ && websocket_protocol_->IsAudioChannelOpened()) {
|
||
// websocket_protocol_->SendTextMessage(text);
|
||
// ESP_LOGI(TAG, "WS辅助文本发送：%s", text.c_str());
|
||
// } else {
|
||
// ESP_LOGW(TAG, "WS辅助未连接，丢弃文本：%s", text.c_str());
|
||
// }
|
||
// });
|
||
// }
|