#include "application.h" #include "board.h" #include "wifi_board.h" #include "display.h" #include "system_info.h" #include "ml307_ssl_transport.h" #include "audio_codec.h" #include "settings.h" #include "mqtt_protocol.h" #include "websocket_protocol.h" #include "volc_rtc_protocol.h" #include "font_awesome_symbols.h" #include "iot/thing_manager.h" #include "assets/lang_config.h" #include "volume_config.h" #include "boards/common/qmi8658a.h" // 添加qmi8658a_data_t类型的头文件 #include "boards/movecall-moji-esp32s3/movecall_moji_esp32s3.h" // 添加MovecallMojiESP32S3类的头文件 #include "weather_api.h" #include #include #include #include #include #include #include #include #include #include #define TAG "Application" #define MAC_TAG "WiFiMAC" #define DIALOG_IDLE_COUNTDOWN_SECONDS 40 // 定义设备状态字符串 static const char* const STATE_STRINGS[] = { "unknown", "starting", "configuring", "idle", "connecting", "listening", "speaking", "dialog", "upgrading", "activating", "fatal_error" }; Application::Application() { event_group_ = xEventGroupCreate(); background_task_ = new BackgroundTask(4096 * 8); last_audible_output_time_ = std::chrono::steady_clock::now(); // 初始化最后一次有声音输出的时间点 skip_dialog_idle_session_ = false; // 初始化跳过对话待机会话标志为false dialog_watchdog_running_ = false; // 初始化对话看门狗运行标志 dialog_watchdog_last_logged_ = -1; // 初始化对话看门狗日志记录 dialog_watchdog_task_handle_ = nullptr; // 初始化对话看门狗任务句柄 clock_ticks_ = 0; // 初始化时钟计数 main_loop_task_handle_ = nullptr; // 初始化主循环任务句柄 check_new_version_task_handle_ = nullptr; // 初始化版本检查任务句柄 audio_loop_task_handle_ = nullptr; // 初始化音频循环任务句柄 esp_timer_create_args_t clock_timer_args = { .callback = [](void* arg) { Application* app = (Application*)arg; app->OnClockTimer(); }, .arg = this, .dispatch_method = ESP_TIMER_TASK, .name = "clock_timer", .skip_unhandled_events = true }; esp_timer_create(&clock_timer_args, &clock_timer_handle_); } Application::~Application() { // 停止并清理对话看门狗 StopDialogWatchdog(); if (clock_timer_handle_ != nullptr) { esp_timer_stop(clock_timer_handle_); esp_timer_delete(clock_timer_handle_); } if (background_task_ != nullptr) { delete background_task_; } if (recorder_pipeline_) { recorder_pipeline_close(recorder_pipeline_); recorder_pipeline_ = nullptr; } if (player_pipeline_) { player_pipeline_close(player_pipeline_); player_pipeline_ = nullptr; } vEventGroupDelete(event_group_); } void Application::CheckNewVersion() { // ESP_LOGI(TAG, "OTA版本检查已临时禁用"); // return; auto& board = Board::GetInstance(); auto display = board.GetDisplay(); // Check if there is a new firmware version available ota_.SetPostData(board.GetJson());// 发送当前设备的JSON数据到OTA服务器,用于检查是否有新的固件版本 包办板载信息 BOARD_TYPE const int MAX_RETRY = 10; int retry_count = 0; while (true) { if (!ota_.CheckVersion()) { retry_count++; if (retry_count >= MAX_RETRY) { ESP_LOGE(TAG, "Too many retries, exit version check"); return; } ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", 60, retry_count, MAX_RETRY); vTaskDelay(pdMS_TO_TICKS(60000)); continue; } retry_count = 0; if (ota_.HasNewVersion()) { Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "happy", Lang::Sounds::P3_UPGRADE); // Wait for the chat state to be idle do { vTaskDelay(pdMS_TO_TICKS(3000)); } while (GetDeviceState() != kDeviceStateIdle); // Use main task to do the upgrade, not cancelable Schedule([this, display]() { SetDeviceState(kDeviceStateUpgrading); display->SetIcon(FONT_AWESOME_DOWNLOAD); std::string message = std::string(Lang::Strings::NEW_VERSION) + ota_.GetFirmwareVersion(); display->SetChatMessage("system", message.c_str()); auto& board = Board::GetInstance(); board.SetPowerSaveMode(false);// 关闭低功耗模式 #if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD wake_word_detect_.Stop(); #endif // 预先关闭音频输出,避免升级过程有音频操作 auto codec = board.GetAudioCodec(); codec->EnableInput(false); codec->EnableOutput(false); { std::lock_guard lock(mutex_); audio_decode_queue_.clear(); } background_task_->WaitForCompletion(); delete background_task_; background_task_ = nullptr; vTaskDelay(pdMS_TO_TICKS(1000)); ota_.StartUpgrade([display](int progress, size_t speed) { char buffer[64]; snprintf(buffer, sizeof(buffer), "%d%% %zuKB/s", progress, speed / 1024); display->SetChatMessage("system", buffer); }); // If upgrade success, the device will reboot and never reach here display->SetStatus(Lang::Strings::UPGRADE_FAILED); ESP_LOGI(TAG, "Firmware upgrade failed..."); vTaskDelay(pdMS_TO_TICKS(3000)); Reboot(); }); return; } // No new version, mark the current version as valid ota_.MarkCurrentVersionValid(); std::string message = std::string(Lang::Strings::VERSION) + ota_.GetCurrentVersion(); display->ShowNotification(message.c_str()); // 检查是否有设备激活码 // if (ota_.HasActivationCode()) { // // Activation code is valid // SetDeviceState(kDeviceStateActivating);//设置设备状态为激活中 // // ShowActivationCode();//显示设备激活码 // // Check again in 60 seconds or until the device is idle // for (int i = 0; i < 60; ++i) { // if (device_state_ == kDeviceStateIdle) { // break; // } // vTaskDelay(pdMS_TO_TICKS(1000)); // } // continue; // } SetDeviceState(kDeviceStateIdle); display->SetChatMessage("system", ""); ResetDecoder(); PlaySound(Lang::Sounds::P3_SUCCESS); // Exit the loop if upgrade or idle break; } } // 取消设备激活码播报,当前设备绑定使用Wi-Fi的Mac地址进行绑定 // void Application::ShowActivationCode() { // auto& message = ota_.GetActivationMessage(); // auto& code = ota_.GetActivationCode(); // struct digit_sound { // char digit; // const std::string_view& sound; // }; // static const std::array digit_sounds{{ // digit_sound{'0', Lang::Sounds::P3_0}, // digit_sound{'1', Lang::Sounds::P3_1}, // digit_sound{'2', Lang::Sounds::P3_2}, // digit_sound{'3', Lang::Sounds::P3_3}, // digit_sound{'4', Lang::Sounds::P3_4}, // digit_sound{'5', Lang::Sounds::P3_5}, // digit_sound{'6', Lang::Sounds::P3_6}, // digit_sound{'7', Lang::Sounds::P3_7}, // digit_sound{'8', Lang::Sounds::P3_8}, // digit_sound{'9', Lang::Sounds::P3_9} // }}; // // This sentence uses 9KB of SRAM, so we need to wait for it to finish // Alert(Lang::Strings::ACTIVATION, message.c_str(), "happy", Lang::Sounds::P3_ACTIVATION); // vTaskDelay(pdMS_TO_TICKS(1000)); // background_task_->WaitForCompletion(); // for (const auto& digit : code) { // auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(), // [digit](const digit_sound& ds) { return ds.digit == digit; }); // if (it != digit_sounds.end()) { // PlaySound(it->sound); // } // } // } // 新增代码(小程序控制 暂停播放 音频) // ========================================================= void Application::PauseAudioPlayback() { std::unique_lock lock(mutex_); if (!audio_paused_) { audio_paused_ = true;// 暂停播放(更新标志位) ESP_LOGI(TAG, "🔇 从服务器接收到暂停播放指令"); // 恢复原始处理方式:立即停止音频输出 auto codec = Board::GetInstance().GetAudioCodec(); if (codec) { codec->EnableOutput(false);// 暂停时立即停止音频输出 ESP_LOGI(TAG, "⏸️ 音频编解码器输出已禁用,实现立即暂停"); } ESP_LOGI(TAG, "⏸️ 音频播放已暂停"); } } // 新增代码(小程序控制 继续播放 音频) void Application::ResumeAudioPlayback() { std::unique_lock lock(mutex_); if (audio_paused_) { audio_paused_ = false;// 恢复播放(更新标志位) ESP_LOGI(TAG, "� 从服务器接收到继续播放指令"); // 恢复原始处理方式:重新启用音频输出 auto codec = Board::GetInstance().GetAudioCodec(); if (codec) { codec->EnableOutput(true);// 恢复时重新启用音频输出 ESP_LOGI(TAG, "▶️ 音频编解码器输出已启用"); } ESP_LOGI(TAG, "▶️ 音频播放已恢复"); } } // ========================================================= void Application::Alert(const char* status, const char* message, const char* emotion, const std::string_view& sound) { ESP_LOGW(TAG, "Alert %s: %s [%s]", status, message, emotion); auto display = Board::GetInstance().GetDisplay(); display->SetStatus(status); display->SetEmotion(emotion); display->SetChatMessage("system", message); if (!sound.empty()) { ResetDecoder(); PlaySound(sound); } } void Application::DismissAlert() { if (device_state_ == kDeviceStateIdle) { auto display = Board::GetInstance().GetDisplay(); display->SetStatus(Lang::Strings::STANDBY); display->SetEmotion("neutral"); display->SetChatMessage("system", ""); } } // 播放音频文件的函数,用于播放存储在内存中的音频数据 void Application::PlaySound(const std::string_view& sound) { // The assets are encoded at 16000Hz, 60ms frame duration SetDecodeSampleRate(16000, 60); const char* data = sound.data(); size_t size = sound.size(); for (const char* p = data; p < data + size; ) { auto p3 = (BinaryProtocol3*)p; p += sizeof(BinaryProtocol3); auto payload_size = ntohs(p3->payload_size); std::vector opus; opus.resize(payload_size); memcpy(opus.data(), p3->payload, payload_size); p += payload_size; std::lock_guard lock(mutex_); audio_decode_queue_.emplace_back(std::move(opus)); } } // 切换聊天状态的函数,用于在不同的设备状态之间进行切换 void Application::ToggleChatState() { // 如果当前设备状态是激活中,则将状态设置为空闲并返回 if (device_state_ == kDeviceStateActivating) { SetDeviceState(kDeviceStateIdle); // 设置设备状态为空闲 return; // 直接返回,不执行后续逻辑 } // 检查协议对象是否已初始化 if (!protocol_) { ESP_LOGE(TAG, "协议未初始化"); // 记录错误日志:协议未初始化 return; // 协议未初始化则直接返回 } // 如果当前设备状态是idle空闲,则尝试进入对话模式 if (device_state_ == kDeviceStateIdle) { Schedule([this]() { SetDeviceState(kDeviceStateConnecting); ESP_LOGI(TAG, "正在尝试打开音频通道"); Board::GetInstance().SetPowerSaveMode(false);// 关闭低功耗模式 if (!protocol_->OpenAudioChannel()) { auto ac = Board::GetInstance().GetAudioCodec(); ESP_LOGW(TAG, "打开音频通道失败,将在2秒后重试"); if (ac) { ESP_LOGW(TAG, "Diag: codec out_channels=%d in_channels=%d out_sr=%d in_sr=%d", ac->output_channels(), ac->input_channels(), ac->output_sample_rate(), ac->input_sample_rate()); } SetDeviceState(kDeviceStateIdle); Schedule([this]() { vTaskDelay(pdMS_TO_TICKS(2000)); ESP_LOGI(TAG, "正在重试音频通道连接"); ToggleChatState();// 打开音频通道 }); return; } listening_mode_ = kListeningModeRealtime;// 设置监听模式为实时监听 SetDeviceState(kDeviceStateDialog);// 设置设备状态为对话模式 protocol_->SendStartListening(listening_mode_);// 发送开始监听消息 auto codec = Board::GetInstance().GetAudioCodec();// 获取音频编解码器 if (codec) { codec->EnableOutput(true);// 启用音频输出 } ESP_LOGI(TAG, "进入对话框状态:启用全双工"); }); } else if (device_state_ == kDeviceStateDialog) { Schedule([this]() { // protocol_->CloseAudioChannel();// 关闭音频通道 // ESP_LOGI(TAG, "关闭音频通道并切换到空闲状态");// 关闭音频通道并切换到空闲状态 protocol_->SendStartListening(listening_mode_);// 发送开始监听消息 }); } else if (device_state_ == kDeviceStateSpeaking) { Schedule([this]() { AbortSpeaking(kAbortReasonNone); protocol_->CloseAudioChannel(); ESP_LOGI(TAG, "关闭音频通道并切换到空闲状态");// 关闭音频通道并切换到空闲状态 }); } else if (device_state_ == kDeviceStateListening || (listening_mode_ == kListeningModeRealtime && device_state_ == kDeviceStateSpeaking)) { Schedule([this]() { protocol_->CloseAudioChannel(); ESP_LOGI(TAG, "关闭音频通道并切换到空闲状态");// 关闭音频通道并切换到空闲状态 }); } } void Application::ToggleListeningState() { if (device_state_ == kDeviceStateActivating) { SetDeviceState(kDeviceStateIdle); return; } if (!protocol_) { ESP_LOGE(TAG, "协议未初始化!");// 记录错误日志:协议未初始化 return; } // 简单的状态切换:idle <-> listening if (device_state_ == kDeviceStateIdle) { // 从待命状态进入聆听状态 Schedule([this]() { SetDeviceState(kDeviceStateConnecting); if (!protocol_->OpenAudioChannel()) { return; } SetListeningMode(kListeningModeManualStop); ESP_LOGI(TAG, "中断按钮:进入聆听状态");// 中断按钮:进入聆听状态 }); } else if (device_state_ == kDeviceStateListening) { // 从聆听状态返回待命状态 Schedule([this]() { protocol_->CloseAudioChannel(); ESP_LOGI(TAG, "中断按钮:返回待命状态");// 中断按钮:返回待命状态 }); } else if (device_state_ == kDeviceStateSpeaking) { // 如果正在说话,中止说话并返回待命状态 Schedule([this]() { AbortSpeaking(kAbortReasonNone); if (protocol_) { protocol_->CloseAudioChannel(); } SetDeviceState(kDeviceStateIdle); ESP_LOGI(TAG, "中断按钮:停止说话,关闭音频通道并返回待命状态");// 中断按钮:停止说话,关闭音频通道并返回待命状态 }); } else if (device_state_ == kDeviceStateConnecting) { // 如果正在连接,直接返回待命状态 Schedule([this]() { SetDeviceState(kDeviceStateIdle); ESP_LOGI(TAG, "中断按钮:取消连接并返回待命状态");// 中断按钮:取消连接并返回待命状态 }); } } void Application::StartListening() { if (device_state_ == kDeviceStateActivating) { SetDeviceState(kDeviceStateIdle); return; } if (!protocol_) { ESP_LOGE(TAG, "协议未初始化!");// 记录错误日志:协议未初始化 return; } if (device_state_ == kDeviceStateIdle) { Schedule([this]() { if (!protocol_->IsAudioChannelOpened()) { SetDeviceState(kDeviceStateConnecting);// 切换到连接状态 if (!protocol_->OpenAudioChannel()) { return; } } SetListeningMode(kListeningModeManualStop);// 设置监听模式为手动停止 }); } else if (device_state_ == kDeviceStateSpeaking) { Schedule([this]() { AbortSpeaking(kAbortReasonNone);// 中止说话 SetListeningMode(kListeningModeManualStop);// 设置监听模式为手动停止 }); } } void Application::StopListening() { Schedule([this]() { if (device_state_ == kDeviceStateListening) { protocol_->SendStopListening(); SetDeviceState(kDeviceStateIdle); } }); } // 🔊 发送文本消息到RTC(传入大模型上下文信息) void Application::SendTextMessage(const std::string& text) { if (!protocol_) { ESP_LOGE(TAG, "协议未初始化!");// 记录错误日志:协议未初始化 return; } if (device_state_ == kDeviceStateIdle) { Schedule([this, text]() { SetDeviceState(kDeviceStateConnecting);// 切换到连接状态 if (!protocol_->OpenAudioChannel()) { return; } SetDeviceState(kDeviceStateDialog); protocol_->SendStartListening(listening_mode_); // 发送文本消息 protocol_->SendTextMessage(text); ESP_LOGI(TAG, "发送文本消息:%s", text.c_str());// 发送文本消息:%s // 立即启动监听模式以接收语音回复 ESP_LOGI(TAG, "realtime_chat_enabled_=%s", realtime_chat_enabled_ ? "true" : "false"); SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeManualStop); }); } else if (device_state_ == kDeviceStateDialog) { Schedule([this, text]() { // if (!protocol_->IsAudioChannelOpened()) {// 如果音频通道未打开 // if (!protocol_->OpenAudioChannel()) {// 尝试打开音频通道 // return; // } // } if (!dialog_upload_enabled_) { SetDialogUploadEnabled(true);// 启用对话上传 protocol_->SendStartListening(listening_mode_);// 发送开始监听消息 } protocol_->SendTextMessage(text);// 发送文本消息 ESP_LOGI(TAG, "发送文本消息:%s", text.c_str());// 发送文本消息:%s }); } else if (device_state_ == kDeviceStateSpeaking) { Schedule([this, text]() { AbortSpeaking(kAbortReasonNone); protocol_->SendTextMessage(text); ESP_LOGI(TAG, "发送文本消息:%s", text.c_str());// 发送文本消息:%s // 启动监听模式以接收语音回复 ESP_LOGI(TAG, "realtime_chat_enabled_=%s", realtime_chat_enabled_ ? "true" : "false"); SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeManualStop); }); } else if (device_state_ == kDeviceStateListening) { Schedule([this, text]() { protocol_->SendTextMessage(text); ESP_LOGI(TAG, "发送文本消息:%s", text.c_str());// 发送文本消息:%s }); } } void Application::Start() { auto& board = Board::GetInstance(); SetDeviceState(kDeviceStateStarting); // 读取NVS中的重启标志 Settings sys("system", true); int32_t reboot_dlg_idle = sys.GetInt("reboot_dlg_idle", 0); int32_t reboot_origin = sys.GetInt("reboot_origin", 0); // 检查是否是因为对话空闲倒计时而重启的 if (reboot_dlg_idle == 1 && reboot_origin == 1) { ESP_LOGI(TAG, "检测到对话空闲倒计时重启标志,将跳过开机播报和网络连接播报"); skip_dialog_idle_session_ = true; Settings sysclr("system", true); sysclr.SetInt("reboot_dlg_idle", 0); sysclr.SetInt("reboot_origin", 0); sysclr.Commit(); } else { ESP_LOGI(TAG, "正常启动流程,将执行开机播报和网络连接播报"); skip_dialog_idle_session_ = false; } /* Setup the display */ auto display = board.GetDisplay(); /* Setup the audio codec */ auto codec = board.GetAudioCodec(); opus_decoder_ = std::make_unique(codec->output_sample_rate(), 1, OPUS_FRAME_DURATION_MS); opus_encoder_ = std::make_unique(16000, 1, OPUS_FRAME_DURATION_MS); if (realtime_chat_enabled_) { ESP_LOGI(TAG, "实时聊天已启用,将opus编码器复杂度设置为0");// 实时聊天已启用,将opus编码器复杂度设置为0 opus_encoder_->SetComplexity(0); } else if (board.GetBoardType() == "ml307") { ESP_LOGI(TAG, "检测到ML307板卡,将opus编码器复杂度设置为5");// 检测到ML307板卡,将opus编码器复杂度设置为5 opus_encoder_->SetComplexity(5); } else { ESP_LOGI(TAG, "检测到WiFi板卡,将opus编码器复杂度设置为3");// 检测到WiFi板卡,将opus编码器复杂度设置为3 opus_encoder_->SetComplexity(3); } if (codec->input_sample_rate() != 16000) { input_resampler_.Configure(codec->input_sample_rate(), 16000); reference_resampler_.Configure(codec->input_sample_rate(), 16000); } uplink_resampler_.Configure(16000, 8000); codec->Start(); { int battery_level = 0; bool charging = false; bool discharging = false; if (board.GetBatteryLevel(battery_level, charging, discharging)) { // 如果电池电量低于25%,则将输出音量设置为0(静音) if (battery_level <= 25) { codec->SetOutputVolumeRuntime(0); } else { Settings s("audio", false); int vol = s.GetInt("output_volume", AudioCodec::default_output_volume()); if (vol <= 0) { vol = AudioCodec::default_output_volume(); } codec->SetOutputVolumeRuntime(vol);// 设置运行时输出音量 } } } // // 在启动阶段创建并运行播放管道以统一输出(开机启动播放管道) // if (!player_pipeline_) { // player_pipeline_ = player_pipeline_open(); // player_pipeline_run(player_pipeline_); // } xTaskCreatePinnedToCore([](void* arg) { Application* app = (Application*)arg; app->AudioLoop(); vTaskDelete(NULL); }, "audio_loop", 4096 * 3, this, 8, &audio_loop_task_handle_, realtime_chat_enabled_ ? 1 : 0); /* Start the main loop */ xTaskCreatePinnedToCore([](void* arg) { Application* app = (Application*)arg; app->MainLoop(); vTaskDelete(NULL); }, "main_loop", 4096 * 3, this, 4, &main_loop_task_handle_, 0); // 根据标志决定是否播放开机播报语音 if (!skip_dialog_idle_session_) { ESP_LOGI(TAG, "设备启动完成,播放开机播报语音");// 设备启动完成,播放开机播报语音 //PlaySound(Lang::Sounds::P3_KAIJIBOBAO); 原有蜡笔小新音色 if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){ PlaySound(Lang::Sounds::P3_KAKA_KAIJIBOBAO); } else if(strcmp(CONFIG_DEVICE_ROLE, "RTC_Test") == 0){ PlaySound(Lang::Sounds::P3_LALA_KAIJIBOBAO); } } else { ESP_LOGI(TAG, "跳过开机播报语音"); } /* Wait for the network to be ready */ board.StartNetwork(); // Initialize the protocol display->SetStatus(Lang::Strings::LOADING_PROTOCOL); #if CONFIG_CONNECTION_TYPE_VOLC_RTC auto volc_protocol = std::make_unique();// 初始化VolcRtc协议 // 设置AgentConfig: 这里的配置会在RTC入会时透传给云端 // WelcomeMessage: 设置开场白 // std::string agent_config = "{\"agent_config\":{\"WelcomeMessage\":\"我是推销员雷军,有什么产品可以帮您介绍的嘛\"}}"; // 已请求成功,配置生效 // std::string config = "{\"Config\":{\"WebSearchAgentConfig\":{\"ComfortWords\":\"啦啦正在上网查询,等一下哦~\"}}}"; // 已请求成功,配置生效 // std::string config = "{\"Config\":{\"WebSearchAgentConfig\":{\"ParamsString\":\"{\\\"bot_id\\\":\\\"7585449675889608233\\\",\\\"stream\\\":true,\\\"location_info\\\":{\\\"city\\\":\\\"上海\\\"}}\"}}}";// 已经请求成功,无报错,配置不生效 std::string city = g_weather_api.GetDefaultLocation();// 获取当前默认城市信息 wifi_config_t wc{};// 获取当前WiFi配置 esp_wifi_get_config(WIFI_IF_STA, &wc);// 获取当前WiFi配置 std::string ssid = std::string(reinterpret_cast(wc.sta.ssid));// 获取当前WiFi SSID wifi_ap_record_t ap{};// 获取当前AP信息 std::string bssid;// 获取当前AP BSSID if (esp_wifi_sta_get_ap_info(&ap) == ESP_OK) { char buf[18]; snprintf(buf, sizeof(buf), "%02x:%02x:%02x:%02x:%02x:%02x",ap.bssid[0], ap.bssid[1], ap.bssid[2],ap.bssid[3], ap.bssid[4], ap.bssid[5]); bssid.assign(buf); } nvs_handle_t h; // 从NVS中读取当前WiFi SSID和BSSID对应的城市信息 if (nvs_open("wifi_city_map", NVS_READONLY, &h) == ESP_OK) { auto try_get = [&](const std::string& key)->std::string{ size_t len = 0; if (nvs_get_str(h, key.c_str(), NULL, &len) == ESP_OK && len > 0) { std::vector buf(len); if (nvs_get_str(h, key.c_str(), buf.data(), &len) == ESP_OK) { return std::string(buf.data()); } } return std::string();// 如果NVS中没有对应城市信息,返回空字符串 }; // 从NVS中读取当前WiFi SSID和BSSID对应的城市信息 if (!ssid.empty()) { std::string city_hit;// 从NVS中读取当前WiFi SSID和BSSID对应的城市信息 if (!bssid.empty()) { city_hit = try_get(ssid + "|" + bssid);// 从NVS中读取当前WiFi SSID和BSSID对应的城市信息 } if (city_hit.empty()) { city_hit = try_get(ssid);// 从NVS中读取当前WiFi SSID对应的城市信息 } if (!city_hit.empty()) { city = city_hit;// 如果NVS中存在对应城市信息,更新当前城市信息 } } nvs_close(h);// 关闭NVS句柄 } // 更新config参数 std::string params = std::string("{\\\"bot_id\\\":\\\"7585449675889608233\\\",\\\"stream\\\":true,\\\"location_info\\\":{\\\"city\\\":\\\"") + city + "\\\"}}"; std::string config = std::string("{\"Config\":{\"WebSearchAgentConfig\":{\"ParamsString\":\"") + params + "\"}}}"; volc_protocol->SetAgentConfig(config);// 设置AgentConfig: 这里的配置会在RTC入会时透传给云端 WebSearchAgentConfigvxiassfdfdfdevde protocol_ = std::move(volc_protocol); #elif CONFIG_CONNECTION_TYPE_WEBSOCKET protocol_ = std::make_unique(); #else protocol_ = std::make_unique(); #endif protocol_->OnNetworkError([this](const std::string& message) { // SetDeviceState(kDeviceStateIdle); // Alert(Lang::Strings::ERROR, message.c_str(), "sad", Lang::Sounds::P3_EXCLAMATION); ESP_LOGW(TAG, "网络错误发生:%s", message.c_str());// 网络错误发生:%s // 检查是否是TLS连接重置错误 if (message.find("TLS") != std::string::npos || message.find("-76") != std::string::npos) { ESP_LOGI(TAG, "检测到TLS连接重置错误,将在3秒后自动重试连接");// 检测到TLS连接重置错误,将在3秒后自动重试连接 SetDeviceState(kDeviceStateIdle); // 3秒后自动重试连接 Schedule([this]() { vTaskDelay(pdMS_TO_TICKS(3000)); if (GetDeviceState() == kDeviceStateIdle) { ESP_LOGI(TAG, "自动重试连接,TLS错误已解决");// 自动重试连接,TLS错误已解决 ToggleChatState(); } }); } else { // 其他网络错误正常处理 SetDeviceState(kDeviceStateIdle); Alert(Lang::Strings::ERROR, message.c_str(), "sad", Lang::Sounds::P3_EXCLAMATION); } }); protocol_->OnIncomingAudio([this](std::vector&& data) { if (websocket_protocol_ && websocket_protocol_->IsAudioChannelOpened()) { aborted_ = true; { std::lock_guard lock(mutex_);// 🔒 保护音频队列操作 // 如果音频队列不为空 if (!audio_decode_queue_.empty()) { ESP_LOGI(TAG, "清空音频队列,大小=%zu", audio_decode_queue_.size()); audio_decode_queue_.clear();// 清空音频队列 } } ResetDecoder(); ws_downlink_enabled_.store(false); ws_playback_active_.store(false); websocket_protocol_->CloseAudioChannel();// 关闭WebSocket通道 Schedule([this]() { vTaskDelay(pdMS_TO_TICKS(120)); aborted_ = false; }); } std::lock_guard lock(mutex_); size_t len = data.size(); audio_decode_queue_.emplace_back(std::move(data)); static bool first_enqueue_logged = false; if (!first_enqueue_logged && len > 0) { ESP_LOGI(TAG, "收到下行音频首包入队: 字节=%zu", len); first_enqueue_logged = true; } ESP_LOGD(TAG, "收到下行音频入队: 字节=%zu 队列大小=%zu", len, audio_decode_queue_.size()); }); protocol_->OnAudioChannelOpened([this, codec, &board]() { ESP_LOGI(TAG, "🟢 音频通道已打开"); ESP_LOGI(TAG, "当前设备状态: %s", STATE_STRINGS[device_state_]); // 🔧 关键修复:立即取消所有待执行的电源管理任务 static TaskHandle_t power_save_task = nullptr; if (power_save_task != nullptr) { vTaskDelete(power_save_task); power_save_task = nullptr; ESP_LOGI(TAG, "🔧 取消了待执行的电源管理任务"); } // 唤醒PowerSaveTimer,从低功耗模式恢复到正常模式 board.WakeUp(); // 立即禁用电源管理,确保连接稳定 ESP_LOGI(TAG, "🔄 禁用电源低功耗管理模式"); board.SetPowerSaveMode(false); // 关键修复:检查服务器采样率与设备输出采样率是否匹配 if (protocol_->server_sample_rate() != codec->output_sample_rate()) { ESP_LOGW(TAG, "⚠️ 服务器采样率 %d 与设备输出采样率 %d 不匹配,重采样可能导致失真", protocol_->server_sample_rate(), codec->output_sample_rate()); } // 设置解码采样率和帧持续时间 SetDecodeSampleRate(protocol_->server_sample_rate(), protocol_->server_frame_duration()); // 关键修复:明确启用音频编解码器输出 ESP_LOGI(TAG, "🔊 启用音频编解码器输出"); codec->EnableOutput(true);// 启用音频编解码器输出 if (!player_pipeline_) { player_pipeline_ = player_pipeline_open(); player_pipeline_run(player_pipeline_); } // 发送IoT状态信息 auto& thing_manager = iot::ThingManager::GetInstance(); protocol_->SendIotDescriptors(thing_manager.GetDescriptorsJson()); std::string states; if (thing_manager.GetStatesJson(states, false)) { protocol_->SendIotStates(states); } // if (websocket_protocol_ && !websocket_protocol_->IsAudioChannelOpened()) { // ESP_LOGI(TAG, "WS辅助通道连接"); // websocket_protocol_->OpenAudioChannel();// // } // 🔧 修复:RTC连接后切换到Speaking状态以播放欢迎语音 ESP_LOGI(TAG, "🔄 音频通道打开,准备播放欢迎语音"); if (GetDeviceState() != kDeviceStateDialog) { SetDeviceState(kDeviceStateSpeaking); } ESP_LOGI(TAG, "当前设备状态: %s", STATE_STRINGS[device_state_]); ESP_LOGI(TAG, "🟢 音频通道初始化完成"); }); protocol_->OnAudioChannelClosed([this, &board]() { ESP_LOGI(TAG, "🔴 音频通道关闭,开始清理任务"); // 🔧 关键修复:取消所有待执行的电源管理任务,防止时序冲突 static TaskHandle_t power_save_task = nullptr; if (power_save_task != nullptr) { vTaskDelete(power_save_task); power_save_task = nullptr; ESP_LOGI(TAG, "🔧 取消了之前的电源管理任务,防止时序冲突"); } // 音频处理器已经在WebSocket断开时停止了 // 等待所有后台任务完成 background_task_->WaitForCompletion(); if (player_pipeline_) { player_pipeline_close(player_pipeline_); player_pipeline_ = nullptr; } ESP_LOGI(TAG, "🔴 后台任务完成"); // 🔧 方案2:先设置设备状态,再启用电源管理,避免时序问题 Schedule([this, &board]() { ESP_LOGI(TAG, "🔄 设置设备为空闲状态"); auto display = Board::GetInstance().GetDisplay(); display->SetChatMessage("system", ""); SetDeviceState(kDeviceStateIdle); // 状态设置完成后,再启用电源管理 vTaskDelay(pdMS_TO_TICKS(100)); ESP_LOGI(TAG, "🔄 设备已稳定在idle状态,启用电源低功耗管理"); try { board.SetPowerSaveMode(true); } catch (...) { ESP_LOGE(TAG, "❌ 设置电源管理模式失败"); } }); }); protocol_->OnIncomingJson([this, display](const cJSON* root) { // Parse JSON data auto type = cJSON_GetObjectItem(root, "type"); if (!(type && cJSON_IsString(type))) { auto tool_calls = cJSON_GetObjectItem(root, "tool_calls"); if (tool_calls && cJSON_IsArray(tool_calls)) { for (int i = 0; i < cJSON_GetArraySize(tool_calls); ++i) { cJSON* call = cJSON_GetArrayItem(tool_calls, i); cJSON* fn = cJSON_GetObjectItem(call, "function"); if (fn && cJSON_IsObject(fn)) { cJSON* name = cJSON_GetObjectItem(fn, "name"); cJSON* args = cJSON_GetObjectItem(fn, "arguments"); cJSON* args_obj = nullptr; const char* args_str = (args && cJSON_IsString(args) && args->valuestring) ? args->valuestring : ""; if (args && cJSON_IsString(args) && args->valuestring) { args_obj = cJSON_Parse(args->valuestring); } if (name && cJSON_IsString(name) && name->valuestring) { if (args_obj) { char* printed = cJSON_PrintUnformatted(args_obj); if (printed) { ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, printed); cJSON_free(printed); } else { ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, args_str); } if (strcmp(name->valuestring, "adjust_audio_val") == 0) { auto codec = Board::GetInstance().GetAudioCodec(); int user = HARDWARE_TO_USER_VOLUME(codec->output_volume()); cJSON* v = cJSON_GetObjectItem(args_obj, "value"); if (!v) v = cJSON_GetObjectItem(args_obj, "action"); if (v) { std::string msg; if (cJSON_IsString(v) && v->valuestring) { if (strcmp(v->valuestring, "up") == 0) { user += 10; msg = "音量已经调大了哦~"; } else if (strcmp(v->valuestring, "down") == 0) { user -= 10; msg = "音量已经调小了哦~"; } else { // 处理字符串形式的数字 char* endptr; long val = strtol(v->valuestring, &endptr, 10); if (*endptr == '\0' && val >= 0 && val <= 100) { user = (int)val; msg = std::string("音量值已经调整为") + std::to_string(user) + "%"; } } } else if (cJSON_IsNumber(v)) { user = (int)v->valuedouble; msg = std::string("音量值已经调整为") + std::to_string(user) + "%"; } if (user > 100) user = 100; if (user < 0) user = 0; int mapped = USER_TO_HARDWARE_VOLUME(user); codec->SetOutputVolume(mapped); ESP_LOGI(TAG, "设置音量: 用户=%d%% 硬件=%d%%", user, mapped); if (!msg.empty()) { cJSON* call_id_item = cJSON_GetObjectItem(call, "id"); const char* call_id = (call_id_item && cJSON_IsString(call_id_item) && call_id_item->valuestring) ? call_id_item->valuestring : ""; if (protocol_ && call_id && call_id[0] != '\0') { protocol_->SendFunctionResult(call_id, msg); } else if (protocol_) { protocol_->SendTextMessage(msg); } } } } // // 添加天气查询功能处理 get_weather_aihub // else if (strcmp(name->valuestring, "get_weather_aihub") == 0) { // ESP_LOGI(TAG, "[WeatherAPI] ===== 收到get_weather_aihub工具调用 ====="); // // // 打印完整参数信息用于调试 // ESP_LOGI(TAG, "[WeatherAPI] 参数对象检查:"); // if (args_obj && cJSON_IsObject(args_obj)) { // cJSON* current = args_obj->child; // while (current) { // ESP_LOGI(TAG, "[WeatherAPI] %s: %s", // current->string, // cJSON_IsString(current) ? current->valuestring : // (cJSON_IsNumber(current) ? "(number)" : // (cJSON_IsBool(current) ? (current->valueint ? "true" : "false") : // "(other)"))); // current = current->next; // } // } else { // ESP_LOGI(TAG, "[WeatherAPI] args_obj为空或不是对象类型"); // } // // // 解析参数 // cJSON* location = cJSON_GetObjectItem(args_obj, "location"); // cJSON* lang = cJSON_GetObjectItem(args_obj, "lang"); // // ESP_LOGI(TAG, "[WeatherAPI] location参数存在: %s, 类型: %s", // location ? "是" : "否", // location && cJSON_IsString(location) ? "字符串" : // (location ? "非字符串" : "不适用")); // ESP_LOGI(TAG, "[WeatherAPI] lang参数存在: %s, 类型: %s", // lang ? "是" : "否", // lang && cJSON_IsString(lang) ? "字符串" : // (lang ? "非字符串" : "不适用")); // // // 设置默认值 // const char* location_str = (location && cJSON_IsString(location)) ? location->valuestring : ""; // const char* lang_str = (lang && cJSON_IsString(lang)) ? lang->valuestring : "zh_CN"; // std::string location_copy = std::string(location_str); // std::string lang_copy = std::string(lang_str); // if (location_copy.empty() || location_copy == "None") { // wifi_config_t wc{}; // esp_wifi_get_config(WIFI_IF_STA, &wc); // std::string ssid = std::string(reinterpret_cast(wc.sta.ssid)); // wifi_ap_record_t ap{}; // std::string bssid; // if (esp_wifi_sta_get_ap_info(&ap) == ESP_OK) { // char buf[18]; // snprintf(buf, sizeof(buf), "%02x:%02x:%02x:%02x:%02x:%02x", // ap.bssid[0], ap.bssid[1], ap.bssid[2], // ap.bssid[3], ap.bssid[4], ap.bssid[5]); // bssid.assign(buf); // } // nvs_handle_t h; // if (nvs_open("wifi_city_map", NVS_READONLY, &h) == ESP_OK) { // auto try_get = [&](const std::string& key)->std::string{ // size_t len = 0; // if (nvs_get_str(h, key.c_str(), NULL, &len) == ESP_OK && len > 0) { // std::vector buf(len); // if (nvs_get_str(h, key.c_str(), buf.data(), &len) == ESP_OK) { // return std::string(buf.data()); // } // } // return std::string(); // }; // if (!ssid.empty()) { // std::string city; // if (!bssid.empty()) { // city = try_get(ssid + "|" + bssid); // } // if (city.empty()) { // city = try_get(ssid); // } // if (!city.empty()) { // location_copy = city; // } // } // nvs_close(h); // } // } // // ESP_LOGI(TAG, "[WeatherAPI] 提取的参数值: location='%s' (长度: %zu), lang='%s'", // location_str, strlen(location_str), lang_str); // // // 获取call_id用于后续响应 // cJSON* call_id_item = cJSON_GetObjectItem(call, "id"); // const char* call_id = (call_id_item && cJSON_IsString(call_id_item) && call_id_item->valuestring) ? call_id_item->valuestring : ""; // std::string call_id_copy = call_id ? call_id : ""; // // ESP_LOGI(TAG, "[WeatherAPI] call_id_item存在: %s", call_id_item ? "是" : "否"); // ESP_LOGI(TAG, "[WeatherAPI] 获取的call_id: '%s'", call_id_copy.c_str()); // // // 创建异步任务处理天气获取 // ESP_LOGI(TAG, "[WeatherAPI] 准备创建异步任务处理天气API调用"); // Schedule([this, location_copy, lang_copy, call_id_copy]() { // ESP_LOGI(TAG, "[WeatherAPI] 异步任务开始执行"); // try { // ESP_LOGI(TAG, "[WeatherAPI] 准备调用全局函数GetWeatherInfo(location='%s', lang='%s')", // location_copy.c_str(), lang_copy.c_str()); // // // 调用天气API获取结果 // std::string weather_result = GetWeatherInfo(location_copy, lang_copy); // // ESP_LOGI(TAG, "[WeatherAPI] GetWeatherInfo调用完成,结果长度: %zu 字节", weather_result.length()); // ESP_LOGD(TAG, "[WeatherAPI] GetWeatherInfo返回结果前100字节: '%s'", // weather_result.substr(0, std::min(size_t(100), weather_result.length())).c_str()); // // ESP_LOGI(TAG, "[WeatherAPI] 准备发送天气结果响应"); // if (!call_id_copy.empty()) { // ESP_LOGI(TAG, "[WeatherAPI] 使用call_id发送FunctionResult: '%s'", call_id_copy.c_str()); // } else { // ESP_LOGI(TAG, "[WeatherAPI] 无call_id,将发送TextMessage"); // } // // if (!call_id_copy.empty() && protocol_) { // protocol_->SendFunctionResult(call_id_copy.c_str(), weather_result); // ESP_LOGI(TAG, "[WeatherAPI] FunctionResult发送成功"); // } else if (protocol_) { // protocol_->SendTextMessage(weather_result);// 发送天气结果 // ESP_LOGI(TAG, "[WeatherAPI] TextMessage发送成功"); // } else { // ESP_LOGE(TAG, "[WeatherAPI] protocol_为空,无法发送响应"); // } // } catch (const std::exception& e) { // ESP_LOGE(TAG, "[WeatherAPI] 天气获取异常: %s", e.what()); // std::string error_msg = "获取天气信息失败,请稍后重试"; // ESP_LOGI(TAG, "[WeatherAPI] 准备发送错误响应: '%s'", error_msg.c_str()); // // if (!call_id_copy.empty() && protocol_) { // protocol_->SendFunctionResult(call_id_copy.c_str(), error_msg); // ESP_LOGI(TAG, "[WeatherAPI] 错误FunctionResult发送成功"); // } else if (protocol_) { // protocol_->SendTextMessage(error_msg); // ESP_LOGI(TAG, "[WeatherAPI] 错误TextMessage发送成功"); // } else { // ESP_LOGE(TAG, "[WeatherAPI] protocol_为空,无法发送错误响应"); // } // } // ESP_LOGI(TAG, "[WeatherAPI] ===== get_weather_aihub异步任务处理完成 ====="); // }); // // ESP_LOGI(TAG, "[WeatherAPI] 异步任务已调度,将在后台执行"); // ESP_LOGI(TAG, "[WeatherAPI] ===== get_weather_aihub工具调用响应已发送 ====="); // } cJSON_Delete(args_obj);// 释放参数对象 } else { ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, args_str); if (strcmp(name->valuestring, "adjust_audio_val") == 0) { auto codec = Board::GetInstance().GetAudioCodec(); int user = HARDWARE_TO_USER_VOLUME(codec->output_volume()); if (args && cJSON_IsString(args) && args->valuestring) { cJSON* tmp = cJSON_Parse(args->valuestring); if (tmp) { cJSON* v = cJSON_GetObjectItem(tmp, "value"); if (!v) v = cJSON_GetObjectItem(tmp, "action"); if (v) { std::string msg; if (cJSON_IsString(v) && v->valuestring) { if (strcmp(v->valuestring, "up") == 0) { user += 10; msg = "音量已经调大了哦~"; } else if (strcmp(v->valuestring, "down") == 0) { user -= 10; msg = "音量已经调小了哦~"; } else { // 处理字符串形式的数字 char* endptr; long val = strtol(v->valuestring, &endptr, 10); if (*endptr == '\0' && val >= 0 && val <= 100) { user = (int)val; msg = std::string("音量值已经调整为") + std::to_string(user) + "%"; } } } else if (cJSON_IsNumber(v)) { user = (int)v->valuedouble; msg = std::string("音量值已经调整为") + std::to_string(user) + "%"; } if (user > 100) user = 100; if (user < 0) user = 0; int mapped = USER_TO_HARDWARE_VOLUME(user); codec->SetOutputVolume(mapped); ESP_LOGI(TAG, "设置音量: 用户=%d%% 硬件=%d%%", user, mapped); if (!msg.empty()) { cJSON* call_id_item = cJSON_GetObjectItem(call, "id"); const char* call_id = (call_id_item && cJSON_IsString(call_id_item) && call_id_item->valuestring) ? call_id_item->valuestring : ""; if (protocol_ && call_id && call_id[0] != '\0') { protocol_->SendFunctionResult(call_id, msg); } else if (protocol_) { protocol_->SendTextMessage(msg); } } } cJSON_Delete(tmp); } } } } } } } return; } return; } if (strcmp(type->valuestring, "tts") == 0) { auto state = cJSON_GetObjectItem(root, "state"); if (strcmp(state->valuestring, "start") == 0) { Schedule([this]() { aborted_ = false; if (device_state_ == kDeviceStateIdle || device_state_ == kDeviceStateListening) { SetDeviceState(kDeviceStateSpeaking); } }); } else if (strcmp(state->valuestring, "stop") == 0) { Schedule([this]() { background_task_->WaitForCompletion(); if (device_state_ == kDeviceStateSpeaking) { if (listening_mode_ == kListeningModeManualStop) { SetDeviceState(kDeviceStateIdle); } else { SetDeviceState(kDeviceStateListening); } } }); } else if (strcmp(state->valuestring, "sentence_start") == 0) { auto text = cJSON_GetObjectItem(root, "text"); if (text != NULL) { ESP_LOGI(TAG, "<< %s", text->valuestring); Schedule([this, display, message = std::string(text->valuestring)]() { display->SetChatMessage("assistant", message.c_str()); }); } } } else if (strcmp(type->valuestring, "stt") == 0) { auto text = cJSON_GetObjectItem(root, "text"); if (text != NULL) { ESP_LOGI(TAG, ">> %s", text->valuestring); Schedule([this, display, message = std::string(text->valuestring)]() { display->SetChatMessage("user", message.c_str()); }); } } else if (strcmp(type->valuestring, "llm") == 0) { auto emotion = cJSON_GetObjectItem(root, "emotion"); if (emotion != NULL) { Schedule([this, display, emotion_str = std::string(emotion->valuestring)]() { display->SetEmotion(emotion_str.c_str()); }); } } else if (strcmp(type->valuestring, "iot") == 0) { auto commands = cJSON_GetObjectItem(root, "commands"); if (commands != NULL) { auto& thing_manager = iot::ThingManager::GetInstance(); for (int i = 0; i < cJSON_GetArraySize(commands); ++i) { auto command = cJSON_GetArrayItem(commands, i); thing_manager.Invoke(command); } } } else if (strcmp(type->valuestring, "response.function_call_arguments.done") == 0) { auto name = cJSON_GetObjectItem(root, "name"); auto arguments = cJSON_GetObjectItem(root, "arguments"); cJSON* args_obj = nullptr; const char* args_str = (arguments && cJSON_IsString(arguments) && arguments->valuestring) ? arguments->valuestring : ""; if (arguments && cJSON_IsString(arguments) && arguments->valuestring) { args_obj = cJSON_Parse(arguments->valuestring); } if (name && cJSON_IsString(name) && name->valuestring) { if (args_obj) { char* printed = cJSON_PrintUnformatted(args_obj); if (printed) { ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, printed); cJSON_free(printed); } else { ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, args_str); } if (strcmp(name->valuestring, "adjust_audio_val") == 0) { auto codec = Board::GetInstance().GetAudioCodec(); int user = HARDWARE_TO_USER_VOLUME(codec->output_volume()); cJSON* v = cJSON_GetObjectItem(args_obj, "value");// 获取value字段 if (!v) v = cJSON_GetObjectItem(args_obj, "action");// 如果value字段不存在,尝试action字段 if (v) { std::string msg; if (cJSON_IsString(v) && v->valuestring) { if (strcmp(v->valuestring, "up") == 0) { user += 10; msg = "音量已经调大了哦~"; } else if (strcmp(v->valuestring, "down") == 0) { user -= 10; msg = "音量已经调小了哦~"; } else { // 处理字符串形式的数字 char* endptr; long val = strtol(v->valuestring, &endptr, 10); if (*endptr == '\0' && val >= 0 && val <= 100) { user = (int)val; msg = std::string("音量值已经调整为") + std::to_string(user) + "%"; } } } else if (cJSON_IsNumber(v)) { user = (int)v->valuedouble; msg = std::string("音量值已经调整为") + std::to_string(user) + "%"; } if (user > 100) user = 100; if (user < 0) user = 0; int mapped = USER_TO_HARDWARE_VOLUME(user); codec->SetOutputVolume(mapped); ESP_LOGI(TAG, "设置音量: 用户=%d%% 硬件=%d%%", user, mapped); if (!msg.empty()) { cJSON* call_id_item = cJSON_GetObjectItem(root, "call_id"); const char* call_id = (call_id_item && cJSON_IsString(call_id_item) && call_id_item->valuestring) ? call_id_item->valuestring : ""; if (protocol_ && call_id && call_id[0] != '\0') { protocol_->SendFunctionResult(call_id, msg); } else if (protocol_) { protocol_->SendTextMessage(msg); } } } } cJSON_Delete(args_obj); } else { ESP_LOGI(TAG, "工具调用: name=%s arguments=%s", name->valuestring, args_str); if (strcmp(name->valuestring, "adjust_audio_val") == 0) { auto codec = Board::GetInstance().GetAudioCodec(); int user = HARDWARE_TO_USER_VOLUME(codec->output_volume()); if (arguments && cJSON_IsString(arguments) && arguments->valuestring) { cJSON* tmp = cJSON_Parse(arguments->valuestring); if (tmp) { cJSON* v = cJSON_GetObjectItem(tmp, "value"); if (!v) v = cJSON_GetObjectItem(tmp, "action"); if (v) { std::string msg; if (cJSON_IsString(v) && v->valuestring) { if (strcmp(v->valuestring, "up") == 0) { user += 10; msg = "音量已经调大了哦~"; } else if (strcmp(v->valuestring, "down") == 0) { user -= 10; msg = "音量已经调小了哦~"; } else { // 处理字符串形式的数字 char* endptr; long val = strtol(v->valuestring, &endptr, 10); if (*endptr == '\0' && val >= 0 && val <= 100) { user = (int)val; msg = std::string("音量值已经调整为") + std::to_string(user) + "%"; } } } else if (cJSON_IsNumber(v)) { user = (int)v->valuedouble; msg = std::string("音量值已经调整为") + std::to_string(user) + "%"; } if (user > 100) user = 100; if (user < 0) user = 0; int mapped = USER_TO_HARDWARE_VOLUME(user); codec->SetOutputVolume(mapped); ESP_LOGI(TAG, "设置音量: 用户=%d%% 硬件=%d%%", user, mapped); if (!msg.empty()) { cJSON* call_id_item = cJSON_GetObjectItem(root, "call_id"); const char* call_id = (call_id_item && cJSON_IsString(call_id_item) && call_id_item->valuestring) ? call_id_item->valuestring : ""; if (protocol_ && call_id && call_id[0] != '\0') { protocol_->SendFunctionResult(call_id, msg); } else if (protocol_) { protocol_->SendTextMessage(msg); } } } cJSON_Delete(tmp); } } } } } // 新增代码(小程序控制 暂停/继续播放 音频) // ==================================================================== } else if (strcmp(type->valuestring, "music_control") == 0) { auto action = cJSON_GetObjectItem(root, "action"); if (action && cJSON_IsString(action) && strcmp(action->valuestring, "pause") == 0) { // 只有在speaking状态下才响应暂停指令 if (device_state_ == kDeviceStateSpeaking) { ESP_LOGI(TAG, "🔇 从服务器接收到暂停播放指令 (speaking状态)"); Schedule([this]() { PauseAudioPlayback();// 暂停播放 }); } else { ESP_LOGI(TAG, "🔇 收到暂停指令但设备不在speaking状态,忽略指令 (当前状态: %s)", STATE_STRINGS[device_state_]); } } else if (action && cJSON_IsString(action) && strcmp(action->valuestring, "resume") == 0) { // 只有在speaking状态下才响应恢复播放指令 if (device_state_ == kDeviceStateSpeaking) { ESP_LOGI(TAG, "🔊 从服务器接收到继续播放指令 (speaking状态)"); Schedule([this]() { ResumeAudioPlayback();// 恢复播放 }); } else { ESP_LOGI(TAG, "🔊 收到恢复播放指令但设备不在speaking状态,忽略指令 (当前状态: %s)", STATE_STRINGS[device_state_]); } } else if (action && cJSON_IsString(action) && strcmp(action->valuestring, "play") == 0) { // 处理新故事推送 - 确保在音频暂停状态和播放状态下都能正常播放 ESP_LOGI(TAG, "🎵 从服务器接收到新故事推送指令 (action: play)"); Schedule([this]() { // 参考 AbortSpeakingAndReturnToListening 第1583-1651行的逻辑 // 检查并处理音频暂停状态,确保新故事能正常播放 if (audio_paused_) { ESP_LOGI(TAG, "🔵 检测到音频暂停状态,为新故事推送清除暂停状态"); audio_paused_ = false; ESP_LOGI(TAG, "✅ 音频暂停状态已清除"); // 清空音频播放队列,避免播放暂停时残留的音频 std::unique_lock lock(mutex_); audio_decode_queue_.clear(); lock.unlock(); ESP_LOGI(TAG, "🧹 已清空音频播放队列,避免播放残留音频"); // 重新启用音频编解码器输出 auto& board = Board::GetInstance(); auto codec = board.GetAudioCodec(); if (codec) { codec->EnableOutput(true); ESP_LOGI(TAG, "🔧 为新故事推送重新启用音频编解码器输出"); } } // 如果当前在播放状态,也需要清空队列确保新故事能正常播放 if (device_state_ == kDeviceStateSpeaking) { ESP_LOGI(TAG, "🔵 当前在播放状态,为新故事推送清空音频队列"); std::unique_lock lock(mutex_); audio_decode_queue_.clear(); lock.unlock(); ESP_LOGI(TAG, "🧹 已清空音频播放队列,准备播放新故事"); // 确保音频编解码器输出已启用 auto& board = Board::GetInstance(); auto codec = board.GetAudioCodec(); if (codec) { codec->EnableOutput(true); ESP_LOGI(TAG, "🔧 确保音频编解码器输出已启用"); } } ESP_LOGI(TAG, "🎵 新故事推送处理完成,音频系统已准备就绪"); }); } } // ==================================================================== }); protocol_->Start(); // Check for new firmware version or get the MQTT broker address ota_.SetCheckVersionUrl(CONFIG_OTA_VERSION_URL); ota_.SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str()); ota_.SetHeader("Client-Id", board.GetUuid()); ota_.SetHeader("Accept-Language", Lang::CODE); auto app_desc = esp_app_get_description(); ota_.SetHeader("User-Agent", std::string(BOARD_NAME "/") + app_desc->version); // 禁用自动OTA - 注释掉下面的任务创建OTA自动升级 xTaskCreate([](void* arg) { Application* app = (Application*)arg; app->CheckNewVersion(); vTaskDelete(NULL); }, "check_new_version", 4096 * 2, this, 2, nullptr); #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.Initialize(codec, realtime_chat_enabled_); // 🎯 根据语音打断功能启用状态配置VAD参数 EchoAwareVadParams enhanced_params; if (realtime_chat_enabled_) { // 语音打断功能启用:配置增强的回声感知参数 - 基于小智AI官方优化方案 // 🎯 平衡配置 - 防误触发同时保证音频流畅 enhanced_params.snr_threshold = 60.0f; // 平衡基础阈值:足够严格但不过度 enhanced_params.min_silence_ms = 2000; // 平衡静音要求:2秒 enhanced_params.interrupt_cooldown_ms = 10000; // 平衡冷却时间:10秒 enhanced_params.adaptive_threshold = true; // 启用自适应阈值 // 🔊 平衡噪声抑制参数 - 优化性能与效果 enhanced_params.adaptive_noise_suppression = true; // 启用自适应噪声抑制 enhanced_params.noise_suppression_base = 5.0f; // 平衡基础抑制强度 enhanced_params.volume_sensitivity = 3.0f; // 平衡音量敏感度:适度的音量影响 enhanced_params.echo_detection_threshold = 0.15f; // 平衡回声检测阈值 enhanced_params.distance_estimation_factor = 3.0f; // 平衡距离估算因子 ESP_LOGI(TAG, "🎯 Adaptive noise suppression enabled for realtime chat - smart volume/distance adjustment"); } else { // 🔧 语音打断功能禁用:关闭复杂VAD,只使用简单VAD enhanced_params.adaptive_threshold = false; // 禁用自适应阈值 enhanced_params.adaptive_noise_suppression = false; // 禁用自适应噪声抑制 ESP_LOGI(TAG, "🔧 Using simple VAD for basic voice detection - complex echo-aware VAD disabled"); } audio_processor_.SetEchoAwareParams(enhanced_params);// 🔊 设置回声感知参数 // 🔊 注册音频处理输出回调 - 处理回声感知后的PCM数据 audio_processor_.OnOutput([this](std::vector&& data) { background_task_->Schedule([this, data = std::move(data)]() mutable { static uint64_t last_us = 0; static size_t frames = 0; std::vector resampled(uplink_resampler_.GetOutputSamples(data.size())); if (!resampled.empty()) { uplink_resampler_.Process(data.data(), data.size(), resampled.data()); } std::vector bytes(resampled.size() * sizeof(int16_t)); for (size_t i = 0; i < resampled.size(); ++i) { int16_t s = resampled[i]; bytes[i * 2] = (uint8_t)(s & 0xFF); bytes[i * 2 + 1] = (uint8_t)((s >> 8) & 0xFF); } frames += 1; uint64_t now_us = esp_timer_get_time(); if (last_us == 0) last_us = now_us; if (now_us - last_us >= 2000000) { ESP_LOGI(TAG, "AFE输出统计: 帧=%zu 样本=%zu ", frames, data.size()); frames = 0; last_us = now_us; } Schedule([this, bytes = std::move(bytes)]() { if (protocol_ && protocol_->IsAudioChannelOpened() && (device_state_ == kDeviceStateListening || device_state_ == kDeviceStateDialog || (listening_mode_ == kListeningModeRealtime && device_state_ == kDeviceStateSpeaking))) { protocol_->SendPcm(bytes); } else { ESP_LOGD(TAG, "通道未打开或不在dialog/listening状态时跳过发送上行"); } }); }); }); // 🎯 根据语音打断功能启用状态选择VAD类型 if (realtime_chat_enabled_) { // 语音打断功能启用:使用复杂的回声感知VAD audio_processor_.OnVadStateChange([this](bool speaking) { ESP_LOGI(TAG, "Complex VAD state change: speaking=%s, device_state=%d, listening_mode=%d", speaking ? "true" : "false", (int)device_state_, (int)listening_mode_); if (device_state_ == kDeviceStateListening) { Schedule([this, speaking]() { if (speaking) { voice_detected_ = true; } else { voice_detected_ = false; } auto led = Board::GetInstance().GetLed(); led->OnStateChanged(); }); } }); } // 🔧 简单VAD:用于普通业务(触摸忽略、LED状态等) audio_processor_.OnSimpleVadStateChange([this](bool speaking) { ESP_LOGI(TAG, "Simple VAD state change: speaking=%s, device_state=%d", speaking ? "true" : "false", (int)device_state_); if (device_state_ == kDeviceStateListening) { Schedule([this, speaking]() { if (speaking) { voice_detected_ = true; } else { voice_detected_ = false; } auto led = Board::GetInstance().GetLed(); led->OnStateChanged(); }); } // 🔊 语音打断逻辑:只在简单VAD中处理,因为复杂VAD可能过于严格 if (device_state_ == kDeviceStateSpeaking && listening_mode_ == kListeningModeRealtime) { Schedule([this, speaking]() { static auto speech_start_time = std::chrono::steady_clock::now(); static bool speech_confirmation_pending = false; auto now = std::chrono::steady_clock::now(); if (speaking) { // 小智AI方案:检测到人声开始,启动确认流程 speech_start_time = now; speech_confirmation_pending = true; ESP_LOGD(TAG, "Human voice detected during playback, starting interrupt evaluation"); } else if (speech_confirmation_pending) { // 小智AI方案:人声结束,评估是否触发打断 auto duration = std::chrono::duration_cast(now - speech_start_time); // 🎯 平衡自适应打断策略:防误触发同时保证响应性 // 基础持续时间:3秒,平衡根据干扰情况调整 int required_duration = 3000; // 基础要求3秒 // 🔊 根据当前音量动态调整持续时间要求(平衡策略) if (current_speaker_volume_ > 0.4f) { required_duration = 5000; // 高音量:5秒 } else if (current_speaker_volume_ > 0.1f) { required_duration = 4000; // 中音量:4秒 } // 低音量或静音:保持3秒 if (duration.count() >= required_duration) { static auto last_interrupt_time = std::chrono::steady_clock::now(); auto interrupt_duration = std::chrono::duration_cast(now - last_interrupt_time); // 🎯 平衡自适应多重保护机制:防误触发同时保证性能 bool volume_protection = (current_speaker_volume_ > 0.01f); // 平衡音量保护:1%阈值 bool cooldown_protection = (interrupt_duration.count() <= 10000); // 平衡冷却:10秒 static int false_positive_count = 0; // 误触发计数器 // 🎯 平衡连续误触发保护:适度学习机制 if (interrupt_duration.count() <= 20000) { // 20秒内的误触发相关 false_positive_count++; } else if (interrupt_duration.count() > 60000) { // 60秒后开始衰减 false_positive_count = std::max(false_positive_count - 1, 0); // 适度衰减 } bool consecutive_protection = (false_positive_count >= 2); // 2次误触发后保护 if (!volume_protection && !cooldown_protection && !consecutive_protection) { // 小智AI核心逻辑:StopPlayback -> SetDeviceState(Listening) ESP_LOGI(TAG, "🎯 Adaptive voice interrupt triggered (duration: %.0fms/%dms, vol: %.3f) - stopping playback", (float)duration.count(), required_duration, current_speaker_volume_); AbortSpeaking(kAbortReasonVoiceInterrupt); SetDeviceState(kDeviceStateListening); last_interrupt_time = now; } else { ESP_LOGI(TAG, "🎯 Adaptive interrupt suppressed - vol_protection: %s (%.3f), cooldown: %.0fms, consecutive: %d", volume_protection ? "true" : "false", current_speaker_volume_, 10000.0f - (float)interrupt_duration.count(), false_positive_count); } } else { ESP_LOGI(TAG, "🎯 Voice too brief (%.0fms), likely echo or noise - adaptive threshold: %dms", (float)duration.count(), required_duration); } speech_confirmation_pending = false; } }); } }); #endif #if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD if (!wake_word_detect_.Initialize(codec)) { ESP_LOGE(TAG, "Failed to initialize wake word detection"); return; } wake_word_detect_.OnWakeWordDetected([this](const std::string& wake_word) { Schedule([this, &wake_word]() { if (device_state_ == kDeviceStateIdle) { SetDeviceState(kDeviceStateConnecting); wake_word_detect_.EncodeWakeWordData(); // 将OpenAudioChannel调用移到后台任务执行,避免main任务栈溢出 background_task_->Schedule([this, wake_word]() { // 打开音频通道并发送唤醒词数据到服务器 if (!protocol_->OpenAudioChannel()) { Schedule([this]() { wake_word_detect_.Start(); }); return; } // 编码并发送唤醒词音频数据 std::vector opus; // Encode and send the wake word data to the server while (wake_word_detect_.GetWakeWordOpus(opus)) { protocol_->SendAudio(opus); } // Set the chat state to wake word detected protocol_->SendWakeWordDetected(wake_word); ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str()); // SetListeningMode需要在main task中执行,因为它涉及UI更新等 Schedule([this]() { SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeAutoStop); }); }); } else if (device_state_ == kDeviceStateSpeaking) { AbortSpeaking(kAbortReasonWakeWordDetected); } else if (device_state_ == kDeviceStateActivating) { SetDeviceState(kDeviceStateIdle); } }); }); wake_word_detect_.Start(); #endif SetDeviceState(kDeviceStateIdle); // 每次设备开机后idle状态下测试 自动检测并设置当前位置打印 //此逻辑为冗余操作,当前NVS中没有城市信息时会自动调用 位置查询API // Schedule([]() { // AutoDetectAndSetLocation(); // }); esp_timer_start_periodic(clock_timer_handle_, 1000000); #if 0 while (true) { SystemInfo::PrintRealTimeStats(pdMS_TO_TICKS(1000)); vTaskDelay(pdMS_TO_TICKS(10000)); } #endif } // 时钟定时器回调函数 void Application::OnClockTimer() { clock_ticks_++; // 每10秒打印一次调试信息 if (clock_ticks_ % 10 == 0) { int free_sram = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); int min_free_sram = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL); // ESP_LOGI(TAG, "Free internal: %u minimal internal: %u", free_sram, min_free_sram);// 打印内部内存空闲大小和最小空闲大小 // // 打印Wi-Fi的Mac地址 // ESP_LOGI(MAC_TAG, "Wi-Fi MAC Address: %s", SystemInfo::GetMacAddress().c_str());// 生产测试打印 //ESP_LOGI(TAG, "此设备角色为: %s",CONFIG_DEVICE_ROLE); // ESP_LOGI(TAG, "此设备角色为: KAKA 1028 升级成功!"); // 如果我们已经同步了服务器时间,如果设备处于空闲状态,请将状态设置为时钟“HH:MM” if (ota_.HasServerTime()) { if (device_state_ == kDeviceStateIdle) { Schedule([this]() { // Set status to clock "HH:MM" time_t now = time(NULL);// 获取当前时间 char time_str[64];// 时间字符串缓冲区 strftime(time_str, sizeof(time_str), "%H:%M ", localtime(&now));// 格式化时间字符串 Board::GetInstance().GetDisplay()->SetStatus(time_str);// 设置显示状态 }); } } } } // 添加任务到主循环 void Application::Schedule(std::function callback) { { std::lock_guard lock(mutex_);// 加锁保护任务队列 main_tasks_.push_back(std::move(callback));// 添加任务到队列 } xEventGroupSetBits(event_group_, SCHEDULE_EVENT);// 通知主循环有任务需要执行 } // 主循环控制聊天状态和Websocket连接 // 如果其他任务需要访问Websocket或聊天状态, // 它们应该使用Schedule调用此函数 void Application::MainLoop() { while (true) { auto bits = xEventGroupWaitBits(event_group_, SCHEDULE_EVENT, pdTRUE, pdFALSE, portMAX_DELAY);// 等待任务事件触发 // 检查是否有任务需要执行 if (bits & SCHEDULE_EVENT) { std::unique_lock lock(mutex_);// 加锁保护任务队列 std::list> tasks = std::move(main_tasks_);// 移动任务队列到本地 lock.unlock();// 解锁,允许其他任务添加到队列 for (auto& task : tasks) { task();// 执行任务 } } } } // 音频环路用于输入和输出音频数据 void Application::AudioLoop() { auto codec = Board::GetInstance().GetAudioCodec(); while (true) { OnAudioInput(); if (codec->output_enabled()) { OnAudioOutput(); } } } // 启动对话看门狗 void Application::StartDialogWatchdog() { if (dialog_watchdog_task_handle_ != nullptr) { return;// 如果看门狗任务已存在,直接返回 } dialog_watchdog_running_ = true;// 设置看门狗运行标志为true dialog_watchdog_last_logged_ = -1;// 重置上次记录的日志时间为-1 xTaskCreatePinnedToCore([](void* arg) { Application* app = (Application*)arg;// 获取应用实例指针 ESP_LOGI(TAG, "Dialog watchdog started, initial device state: %d", app->GetDeviceState()); while (app->dialog_watchdog_running_) { vTaskDelay(pdMS_TO_TICKS(2000));// 减少延时到2秒,更及时地检测和更新倒计时 // 检查设备状态 DeviceState current_state = app->GetDeviceState(); if (current_state != kDeviceStateDialog) { ESP_LOGD(TAG, "Dialog watchdog skipping check, not in dialog state (current: %d)", current_state); continue; } auto now = std::chrono::steady_clock::now();// 获取当前时间点 auto elapsed = std::chrono::duration_cast(now - app->last_audible_output_time_).count();// 计算自上次有音频输出以来的秒数 // 确保elapsed为非负数 if (elapsed < 0) { ESP_LOGW(TAG, "Dialog watchdog: invalid elapsed time: %lld", elapsed); continue; } int remaining = DIALOG_IDLE_COUNTDOWN_SECONDS - (int)elapsed;// 计算对话空闲倒计时剩余秒数 // 调试日志 ESP_LOGD(TAG, "Dialog watchdog: elapsed=%d, remaining=%d", (int)elapsed, remaining); // 如果剩余秒数小于等于0,说明对话空闲倒计时已到,需要重启设备 if (remaining <= 0) { ESP_LOGI(TAG, "Dialog watchdog idle reached, elapsed=%d, marking and rebooting", (int)elapsed); Settings sys("system", true); ESP_LOGI(TAG, "Dialog watchdog: preparing NVS writes (system)"); sys.SetInt("reboot_dlg_idle", 1); sys.SetInt("reboot_origin", 1); ESP_LOGI(TAG, "Dialog watchdog: committing NVS (system)"); sys.Commit(); Settings sysr("system"); int32_t verify = sysr.GetInt("reboot_dlg_idle", 0); int32_t origin_read = sysr.GetInt("reboot_origin", 0); if (verify != 1) { ESP_LOGW(TAG, "Dialog watchdog: NVS verify failed, cause=%d, origin=%d", (int)verify, (int)origin_read); ESP_LOGW(TAG, "建议: 检查NVS空间是否不足、确认nvs_flash_init成功、避免并发写入(system)"); } ESP_LOGI(TAG, "Dialog watchdog (task) set reboot_cause=1, verify=%d, restart in 2000ms", (int)verify); vTaskDelay(pdMS_TO_TICKS(2000)); esp_restart();// 重启设备 app->dialog_watchdog_running_ = false;// 设置看门狗运行标志为false } else { // 简化条件判断,移除冗余检查 // 优化桶计算逻辑,使用1秒一个桶,更精确地显示倒计时 int bucket = remaining; // 使用剩余秒数作为桶标识,实现每秒更新 if (bucket != app->dialog_watchdog_last_logged_ && remaining <= DIALOG_IDLE_COUNTDOWN_SECONDS) { ESP_LOGI(TAG, "dialog对话空闲倒计时剩余: %d 秒", remaining);// 打印剩余秒数 app->dialog_watchdog_last_logged_ = bucket;// 更新上次记录的日志时间为当前桶 } } } app->dialog_watchdog_task_handle_ = nullptr; ESP_LOGI(TAG, "Dialog watchdog stopped"); vTaskDelete(NULL); }, "dialog_watchdog", 4096, this, 5, &dialog_watchdog_task_handle_, 0); } // 停止对话看门狗 void Application::StopDialogWatchdog() { dialog_watchdog_running_ = false; if (dialog_watchdog_task_handle_ != nullptr) { vTaskDelete(dialog_watchdog_task_handle_); dialog_watchdog_task_handle_ = nullptr; ESP_LOGI(TAG, "Dialog watchdog stopped"); } dialog_watchdog_last_logged_ = -1; } // 音频输出函数 void Application::OnAudioOutput() { auto now = std::chrono::steady_clock::now();// 获取当前时间 auto codec = Board::GetInstance().GetAudioCodec();// 获取音频编解码器 const int max_silence_seconds = 10;// 最大静音秒数 std::unique_lock lock(mutex_);// 加锁保护音频队列 // 调试日志:检查设备状态和音频队列 ESP_LOGD(TAG, "🔊 OnAudioOutput called, device_state: %d, audio_queue_size: %zu, codec_output_enabled: %d", device_state_, audio_decode_queue_.size(), codec->output_enabled()); // 新增代码(小程序控制 暂停/继续播放 音频) // ========================================================= // 🔧 暂停状态下停止从队列取数据,但保留队列状态 if (audio_paused_) { // 暂停时重置音量状态,避免误判 if (current_speaker_volume_ > 0.0f) { current_speaker_volume_ = 0.0f;// 暂停时重置音量状态 #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.SetSpeakerVolume(0.0f); #endif } return; } // ========================================================= if (audio_decode_queue_.empty()) { // 重要:没有音频数据时立即重置音量状态 if (current_speaker_volume_ > 0.0f) { current_speaker_volume_ = 0.0f; #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.SetSpeakerVolume(0.0f); #endif } ESP_LOGD(TAG, "🔊 音频队列为空"); // Disable the output if there is no audio data for a long time if (device_state_ == kDeviceStateIdle) { auto duration = std::chrono::duration_cast(now - last_output_time_).count(); if (duration > max_silence_seconds) { codec->EnableOutput(false); } } return; } if (device_state_ == kDeviceStateListening && listening_mode_ != kListeningModeRealtime) { audio_decode_queue_.clear(); current_speaker_volume_ = 0.0f;// 清空音频队列后重置音量状态 #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.SetSpeakerVolume(0.0f); #endif return; } auto opus = std::move(audio_decode_queue_.front()); audio_decode_queue_.pop_front(); lock.unlock(); background_task_->Schedule([this, codec, opus = std::move(opus)]() mutable { if (aborted_) { return; } std::vector pcm; bool decoded = false; bool treat_as_pcm = (protocol_ && protocol_->downlink_is_pcm() && !ws_playback_active_.load()); if (!treat_as_pcm) { decoded = opus_decoder_->Decode(std::move(opus), pcm); } if (!decoded) { if (treat_as_pcm && !opus.empty() && (opus.size() % 2 == 0)) { pcm.resize(opus.size() / 2); memcpy(pcm.data(), opus.data(), opus.size()); int srv = protocol_ ? protocol_->server_sample_rate() : 16000; if (!player_pipeline_) { if (srv != codec->output_sample_rate()) { output_resampler_.Configure(srv, codec->output_sample_rate()); int target_size = output_resampler_.GetOutputSamples(pcm.size()); std::vector resampled(target_size); output_resampler_.Process(pcm.data(), pcm.size(), resampled.data()); pcm = std::move(resampled); } } } else { return; } } // Resample if the sample rate is different if (!treat_as_pcm && decoded && !player_pipeline_ && opus_decoder_->sample_rate() != codec->output_sample_rate()) { int target_size = output_resampler_.GetOutputSamples(pcm.size()); std::vector resampled(target_size); output_resampler_.Process(pcm.data(), pcm.size(), resampled.data()); pcm = std::move(resampled); } // 计算并同步音频输出音量到音频处理器,用于动态VAD阈值调整 if (!pcm.empty()) { // 计算音频块的RMS音量 (0.0 - 1.0) float sum_squares = 0.0f; for (const auto& sample : pcm) { float normalized = (float)sample / 32768.0f; sum_squares += normalized * normalized; } float rms_volume = std::sqrt(sum_squares / pcm.size()); // 当有可听见的音频输出时,更新最后声音输出时间戳 const float audible_volume_threshold = 0.01f; // 设置一个合理的音量阈值 if (rms_volume >= audible_volume_threshold) { this->last_audible_output_time_ = std::chrono::steady_clock::now(); ESP_LOGD(TAG, "🔊 更新last_audible_output_time_,当前音量: %.4f", rms_volume); } #if CONFIG_USE_AUDIO_PROCESSOR // 同步音量到音频处理器,用于动态阈值调整 current_speaker_volume_ = rms_volume; // 保存当前音量供打断逻辑使用 audio_processor_.SetSpeakerVolume(rms_volume); #endif } int src_rate = decoded ? opus_decoder_->sample_rate() : (protocol_ ? protocol_->server_sample_rate() : 16000); static bool first_play_logged = false; if (player_pipeline_) { player_pipeline_set_src_rate(player_pipeline_, src_rate); int bytes = (int)(pcm.size() * sizeof(int16_t)); ESP_LOGD(TAG, "写入播放管道: 采样率=%d 字节=%d", src_rate, bytes); player_pipeline_write(player_pipeline_, (char*)pcm.data(), bytes); if (bytes > 0) { this->last_audible_output_time_ = std::chrono::steady_clock::now(); } if (!first_play_logged && bytes > 0) { ESP_LOGI(TAG, "开始播放下行音频: 字节=%d 采样率=%d", bytes, src_rate); first_play_logged = true; } } else { ESP_LOGD(TAG, "直接输出PCM到编解码器: 样本=%zu", pcm.size()); codec->OutputData(pcm);// 直接输出PCM数据 if (!pcm.empty()) { this->last_audible_output_time_ = std::chrono::steady_clock::now(); } if (!first_play_logged && !pcm.empty()) { ESP_LOGI(TAG, "开始播放下行音频: 样本=%zu 采样率=%d", pcm.size(), src_rate); first_play_logged = true; } // 解决本地资源声音播放尖锐问题方案1 // // 如果是单声道,转换为立体声 // if (codec->output_channels() == 2) {// 单声道转换为立体声 // std::vector stereo(pcm.size() * 2);// 立体声PCM数据 // for (size_t i = 0, j = 0; i < pcm.size(); ++i) { // stereo[j++] = pcm[i];// 左声道 // stereo[j++] = pcm[i];// 右声道 // } // codec->OutputData(stereo);// 输出立体声PCM数据 // } else { // codec->OutputData(pcm);// 输出单声道PCM数据 // } // 解决本地资源声音播放尖锐问题方案2 // player_pipeline_ = player_pipeline_open();// 打开音频播放管道 // player_pipeline_run(player_pipeline_);// 启动音频播放管道 // player_pipeline_set_src_rate(player_pipeline_, src_rate);// 设置播放管道源采样率 // player_pipeline_write(player_pipeline_, (char*)pcm.data(), (int)(pcm.size() * sizeof(int16_t)));// 写入PCM数据到播放管道 } last_output_time_ = std::chrono::steady_clock::now();// 更新最后输出时间 }); } void Application::OnAudioInput() { std::vector data; #if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD if (wake_word_detect_.IsRunning()) { ReadAudio(data, 16000, wake_word_detect_.GetFeedSize()); wake_word_detect_.Feed(data);// 将音频数据喂给唤醒词检测器 return; } #endif #if CONFIG_USE_AUDIO_PROCESSOR if (audio_processor_.IsRunning()) { ReadAudio(data, 16000, audio_processor_.GetFeedSize()); if (!data.empty()) { int n = (int)data.size(); int64_t sum = 0; int peak = 0; for (int i = 0; i < n; ++i) { int v = data[i]; int a = v < 0 ? -v : v; sum += a; if (a > peak) peak = a; } (void)sum; // if (avg > 150 || peak > 800) { // ESP_LOGI(TAG, "🎙️ 输入幅度: 均值=%d 峰值=%d 样本=%d", avg, peak, n); // } } audio_processor_.Feed(data); return; } #else if (device_state_ == kDeviceStateListening || device_state_ == kDeviceStateDialog) { if (send_g711a_uplink_) { ReadAudio(data, 16000, 20 * 16000 / 1000); if (!data.empty()) { std::vector resampled(uplink_resampler_.GetOutputSamples((int)data.size())); if (!resampled.empty()) { uplink_resampler_.Process(data.data(), (int)data.size(), resampled.data()); } int out_samples = (int)resampled.size(); std::vector bytes(out_samples); for (int i = 0; i < out_samples; ++i) { int16_t s = resampled[i]; int sign = (s >> 8) & 0x80; if (sign) s = -s; if (s > 32635) s = 32635; int exp = 7; for (int mask = 0x4000; exp > 0 && (s & mask) == 0; mask >>= 1) exp--; int mant = (s >> ((exp == 0) ? 4 : (exp + 3))) & 0x0F; uint8_t a = (uint8_t)(sign | (exp << 4) | mant); bytes[i] = (uint8_t)(a ^ 0xD5); } Schedule([this, bytes = std::move(bytes)]() { if (protocol_ && protocol_->IsAudioChannelOpened()) { protocol_->SendG711A(bytes);// 发送G711A音频数据 } }); } } else if (send_pcm_uplink_) { ReadAudio(data, 16000, 20 * 16000 / 1000); if (!data.empty()) { int out_samples = (int)data.size() / 2; std::vector down(out_samples); for (int i = 0, j = 0; i < out_samples; ++i, j += 2) { down[i] = data[j]; } std::vector resampled(uplink_resampler_.GetOutputSamples((int)down.size())); if (!resampled.empty()) { uplink_resampler_.Process(down.data(), (int)down.size(), resampled.data()); } std::vector bytes(resampled.size() * sizeof(int16_t)); for (size_t i = 0; i < resampled.size(); ++i) { int16_t s = resampled[i]; bytes[i * 2] = (uint8_t)(s & 0xFF); bytes[i * 2 + 1] = (uint8_t)((s >> 8) & 0xFF); } Schedule([this, bytes = std::move(bytes)]() { if (protocol_ && protocol_->IsAudioChannelOpened() && (device_state_ == kDeviceStateListening || device_state_ == kDeviceStateDialog || (listening_mode_ == kListeningModeRealtime && device_state_ == kDeviceStateSpeaking))) { protocol_->SendPcm(bytes); } }); } } else { ReadAudio(data, 16000, 30 * 16000 / 1000); background_task_->Schedule([this, data = std::move(data)]() mutable { opus_encoder_->Encode(std::move(data), [this](std::vector&& opus) { Schedule([this, opus = std::move(opus)]() { if (protocol_) { protocol_->SendAudio(opus); } }); }); }); } return; } #endif vTaskDelay(pdMS_TO_TICKS(30)); } void Application::ReadAudio(std::vector& data, int sample_rate, int samples) { auto codec = Board::GetInstance().GetAudioCodec(); #if CONFIG_USE_AUDIO_PROCESSOR // 当音频处理器运行且存在参考通道时,保持原有双通道读取以支持AEC if (audio_processor_.IsRunning() && codec->input_channels() == 2) { if (codec->input_sample_rate() != sample_rate) { data.resize(samples * codec->input_sample_rate() / sample_rate); if (!codec->InputData(data)) { return; } auto mic_channel = std::vector(data.size() / 2); auto reference_channel = std::vector(data.size() / 2); for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) { mic_channel[i] = data[j]; reference_channel[i] = data[j + 1]; } auto resampled_mic = std::vector(input_resampler_.GetOutputSamples(mic_channel.size())); auto resampled_reference = std::vector(reference_resampler_.GetOutputSamples(reference_channel.size())); input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data()); reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data()); data.resize(resampled_mic.size() + resampled_reference.size()); for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) { data[j] = resampled_mic[i]; data[j + 1] = resampled_reference[i]; } return; } else { data.resize(samples * codec->input_channels()); if (!codec->InputData(data)) { return; } static bool first_equal_sr_dual_read_logged = false; if (!first_equal_sr_dual_read_logged) { ESP_LOGI(TAG, "AFE输入首包: 双通道等采样率 目标样本=%d 通道=%d 实际向量=%zu", samples, codec->input_channels(), data.size()); first_equal_sr_dual_read_logged = true; } return; } } #endif // 默认优先使用recorder管道读取(目标采样率16000),无参考通道需求 if (recorder_pipeline_ && sample_rate == 16000) { int need_bytes = samples * (int)sizeof(int16_t); int default_bytes = recorder_pipeline_get_default_read_size(recorder_pipeline_); std::vector out; out.reserve(samples);// 预分配内存空间,避免后续动态扩容 std::vector buf(default_bytes);// 内存音频缓冲区,用于存储从录音管道读取的音频数据 while ((int)out.size() < samples) { int to_read = std::min(default_bytes, (need_bytes - (int)out.size() * (int)sizeof(int16_t)));// 计算本次读取的字节数,不超过默认读取大小和剩余需要读取的字节数 if (to_read <= 0) break;// 读取到的数据大小小于等于0,跳出循环 int got = recorder_pipeline_read(recorder_pipeline_, buf.data(), to_read);// 从录音管道读取音频数据,并赋值给内存音频缓冲区 if (got <= 0) { ESP_LOGW(TAG, "🎙️ 录音管道读取失败,未收到输入数据"); break; } int got_samples = got / (int)sizeof(int16_t);// 计算本次读取的样本数,即读取到的字节数除以每个样本的字节数 int16_t* p = (int16_t*)buf.data();// 将内存音频缓冲区转换为int16_t指针,方便按样本读取 for (int i = 0; i < got_samples && (int)out.size() < samples; ++i) { out.push_back(p[i]);// 将读取到的样本添加到输出向量中,直到达到预期样本数或读取完所有数据 } } if (!out.empty()) { data.assign(out.begin(), out.end());// 将输出向量中的数据赋值给输出参数data return; } } // 回退到直接从codec读取的实现 if (codec->input_sample_rate() != sample_rate) { data.resize(samples * codec->input_sample_rate() / sample_rate); if (!codec->InputData(data)) { ESP_LOGW(TAG, "🎙️ 麦克风采样失败(重采样路径),未收到输入数据"); return; } if (codec->input_channels() == 2) { // 双通道约定:当前缓冲按 [主麦,参考] 排列;必须与 ALGORITHM_INPUT_FORMAT 的 M/R 顺序 // 以及 CHANNEL_FORMAT 的物理通道对应一致,否则可能导致 AEC 失效或增益反向 auto mic_channel = std::vector(data.size() / 2); auto reference_channel = std::vector(data.size() / 2); for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) { mic_channel[i] = data[j]; reference_channel[i] = data[j + 1]; } auto resampled_mic = std::vector(input_resampler_.GetOutputSamples(mic_channel.size())); auto resampled_reference = std::vector(reference_resampler_.GetOutputSamples(reference_channel.size())); input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data()); reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data()); data.resize(resampled_mic.size() + resampled_reference.size()); for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) { data[j] = resampled_mic[i]; data[j + 1] = resampled_reference[i]; } } else { auto resampled = std::vector(input_resampler_.GetOutputSamples(data.size())); input_resampler_.Process(data.data(), data.size(), resampled.data()); data = std::move(resampled); } } else { data.resize(samples); if (!codec->InputData(data)) { ESP_LOGW(TAG, "🎙️ 麦克风采样失败(直读路径),未收到输入数据"); return; } } } // 打断语音播报(终止播放) void Application::AbortSpeaking(AbortReason reason) { // 🔧 防止重复中止操作,避免竞态条件 bool expected = false; if (!is_aborting_.compare_exchange_strong(expected, true)) { ESP_LOGD(TAG, "AbortSpeaking already in progress, ignoring duplicate call"); return; } ESP_LOGI(TAG, "🔴 Abort speaking - immediate stop"); aborted_ = true; // 🔧 更新安全操作时间戳 last_safe_operation_.store(std::chrono::steady_clock::now()); // 🔧 修复:立即清空音频队列,确保音频播放立即停止 { std::lock_guard lock(mutex_); if (!audio_decode_queue_.empty()) { ESP_LOGI(TAG, "🔴 Clearing %zu audio frames from queue", audio_decode_queue_.size()); audio_decode_queue_.clear(); // 重置音量状态 current_speaker_volume_ = 0.0f; #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.SetSpeakerVolume(0.0f); #endif } } // ⚠️ 移除WaitForCompletion避免死锁,让后台任务通过aborted_标志自然结束 ESP_LOGI(TAG, "🔴 Audio queue cleared, background tasks will stop on next iteration"); // 🔧 修复:始终尝试发送中止消息以打断RTC下行(不受IsSafeToOperate限制) if (protocol_) { try { // 🔧 额外的连接状态验证 if (protocol_->IsAudioChannelOpened()) { protocol_->SendAbortSpeaking(reason); ESP_LOGI(TAG, "AbortSpeaking message sent successfully"); // 更新安全操作时间戳 last_safe_operation_.store(std::chrono::steady_clock::now()); } else { ESP_LOGW(TAG, "Audio channel not properly opened, skipping AbortSpeaking"); } } catch (const std::exception& e) { ESP_LOGW(TAG, "Failed to send AbortSpeaking message: %s", e.what()); } } else { ESP_LOGD(TAG, "Skipping AbortSpeaking message - protocol_ is null"); } // 🔧 确保中止窗口后恢复播放流程,避免长时间阻塞导致WS音频无法播放 Schedule([this]() { vTaskDelay(pdMS_TO_TICKS(120)); aborted_ = false; ESP_LOGI(TAG, "🔵 Abort window ended, resume playback tasks"); }); // 🔧 重置中止标志,允许后续操作 is_aborting_.store(false); } // 发送讲故事请求 webscoket协议 void Application::SendStoryRequest() { if (!websocket_protocol_) { InitializeWebsocketProtocol();// 初始化WebSocket协议 if (!websocket_protocol_) { ESP_LOGW(TAG, "WebSocket协议初始化失败"); return; } } Schedule([this]() { ws_downlink_enabled_.store(true); // 确保音频通道已打开 if (!websocket_protocol_->IsAudioChannelOpened()) { websocket_protocol_->OpenAudioChannel();// 打开音频通道 } websocket_protocol_->SendStoryRequest();// 发送故事请求 ESP_LOGI(TAG, "通过WebSocket发送的故事请求!"); }); } // 设置监听模式 void Application::SetListeningMode(ListeningMode mode) { ESP_LOGI(TAG, "Setting listening mode to %d", (int)mode);// 打印设置监听模式日志 listening_mode_ = mode; SetDeviceState(kDeviceStateListening); } // 设置设备状态 void Application::SetDeviceState(DeviceState state) { if (device_state_ == state) { return; } clock_ticks_ = 0; auto previous_state = device_state_;// 记录上一个设备状态 device_state_ = state;// 设置设备状态 if (state == kDeviceStateDialog) { StartDialogWatchdog(); } else if (previous_state == kDeviceStateDialog) { StopDialogWatchdog(); } ESP_LOGI(TAG, "打印设置设备状态日志: %s", STATE_STRINGS[device_state_]);// 打印设置设备状态日志 // The state is changed, wait for all background tasks to finish background_task_->WaitForCompletion(); auto& board = Board::GetInstance(); auto display = board.GetDisplay(); auto led = board.GetLed(); led->OnStateChanged(); // 检查是否正在进行BluFi配网,配网时禁止播放待命音效(新增代码) // ================================================================= bool is_blufi_provisioning = false; if (Board::GetInstance().GetBoardType() == "wifi") { auto& wifi_board = static_cast(Board::GetInstance()); is_blufi_provisioning = wifi_board.IsBluFiProvisioningActive(); } // ================================================================= switch (state) { case kDeviceStateUnknown: case kDeviceStateIdle: display->SetStatus(Lang::Strings::STANDBY); display->SetEmotion("neutral"); // // 只有从非待命状态进入待命状态时才播放待命音效,避免重复播放(原来的代码) // if (previous_state != kDeviceStateIdle && // previous_state != kDeviceStateUnknown && // previous_state != kDeviceStateWifiConfiguring) { // ESP_LOGI(TAG, "Entering idle state, playing standby sound"); // PlaySound(Lang::Sounds::P3_DAIMING); // } // 开机后 进入待命状态 播报 卡卡正在待命(配网模式下不播报“卡卡正在待命”)-新增代码 //===================================================================================== if (previous_state != kDeviceStateIdle && previous_state != kDeviceStateUnknown && previous_state != kDeviceStateWifiConfiguring && !is_blufi_provisioning && !IsLowBatteryTransition()) { ESP_LOGI(TAG, "Entering idle state, playing standby sound"); // PlaySound(Lang::Sounds::P3_DAIMING); 原有 待命 播报 if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){ PlaySound(Lang::Sounds::P3_KAKA_DAIMING); } else if(strcmp(CONFIG_DEVICE_ROLE, "RTC_Test") == 0){ PlaySound(Lang::Sounds::P3_LALA_DAIMING); } } //===================================================================================== #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.Stop(); #endif #if 1 if (recorder_pipeline_) { recorder_pipeline_close(recorder_pipeline_); recorder_pipeline_ = nullptr; } #endif #if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD wake_word_detect_.Start(); #endif // 设备开机后首次进入idle状态时,自动检测NVS中的位置并调用API后设置当前位置 if (!first_idle_location_checked_) { first_idle_location_checked_ = true;// 首次查询城市天气 Schedule([]() { AutoDetectAndSetLocation();// 自动检测并设置当前位置 }); } break; case kDeviceStateConnecting: display->SetStatus(Lang::Strings::CONNECTING); display->SetEmotion("neutral"); display->SetChatMessage("system", ""); break; case kDeviceStateListening: display->SetStatus(Lang::Strings::LISTENING); display->SetEmotion("neutral"); // 关键修复:只有在非音效播放状态下才重置音量,避免中断正在播放的音效 // 检查是否有音频正在播放,如果有则延迟重置音量 if (IsAudioQueueEmpty()) { current_speaker_volume_ = 0.0f; #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.SetSpeakerVolume(0.0f); #endif } else { // 如果有音频正在播放,延迟重置音量 Schedule([this]() { vTaskDelay(pdMS_TO_TICKS(500)); // 等待音效播放完成 current_speaker_volume_ = 0.0f; #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.SetSpeakerVolume(0.0f); #endif }); } // Update the IoT states before sending the start listening command UpdateIotStates(); // Make sure the audio processor is running #if CONFIG_USE_AUDIO_PROCESSOR if (!audio_processor_.IsRunning()) { #else if (true) { #endif // 🔧 关键修复:检查协议连接状态,防止发送到无效连接 if (protocol_ && protocol_->IsAudioChannelOpened()) { // Send the start listening command protocol_->SendStartListening(listening_mode_); if (listening_mode_ == kListeningModeAutoStop && previous_state == kDeviceStateSpeaking) { // FIXME: Wait for the speaker to empty the buffer vTaskDelay(pdMS_TO_TICKS(120)); } opus_encoder_->ResetState(); #if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD wake_word_detect_.Stop(); #endif #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.Start(); #endif if (!recorder_pipeline_) { recorder_pipeline_ = recorder_pipeline_open(); recorder_pipeline_run(recorder_pipeline_); } } else { ESP_LOGW(TAG, "Audio channel not available, skipping SendStartListening");// 音频通道未打开,跳过发送开始聆听命令 // 保持在聆听状态,不自动回退到idle状态 ESP_LOGI(TAG, "🔵 Staying in listening state despite audio channel unavailable"); } } break; case kDeviceStateSpeaking: display->SetStatus(Lang::Strings::SPEAKING); if (listening_mode_ != kListeningModeRealtime) { #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.Stop(); #endif #if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD wake_word_detect_.Start(); #endif } else { // 在实时模式下,保持audio_processor运行以检测语音打断 #if CONFIG_USE_AUDIO_PROCESSOR if (!audio_processor_.IsRunning()) { audio_processor_.Start(); } #endif #if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD wake_word_detect_.Stop(); #endif } ResetDecoder(); break; case kDeviceStateDialog: display->SetStatus(Lang::Strings::SPEAKING); #if CONFIG_USE_AUDIO_PROCESSOR if (!audio_processor_.IsRunning()) { audio_processor_.Start(); } #endif #if CONFIG_USE_WAKE_WORD_DETECT || CONFIG_USE_CUSTOM_WAKE_WORD wake_word_detect_.Stop(); #endif { auto codec2 = Board::GetInstance().GetAudioCodec();// 获取音频编解码器 codec2->EnableOutput(true);// 启用音频输出 last_audible_output_time_ = std::chrono::steady_clock::now();// 更新最后有声音输出的时间 } if (!recorder_pipeline_) { recorder_pipeline_ = recorder_pipeline_open(); recorder_pipeline_run(recorder_pipeline_); } break; default: // Do nothing break; } } void Application::ResetDecoder() { std::lock_guard lock(mutex_); opus_decoder_->ResetState(); audio_decode_queue_.clear(); last_output_time_ = std::chrono::steady_clock::now(); auto codec = Board::GetInstance().GetAudioCodec(); codec->EnableOutput(true); } void Application::SetDecodeSampleRate(int sample_rate, int frame_duration) { if (opus_decoder_->sample_rate() == sample_rate && opus_decoder_->duration_ms() == frame_duration) { return; } opus_decoder_.reset(); opus_decoder_ = std::make_unique(sample_rate, 1, frame_duration); auto codec = Board::GetInstance().GetAudioCodec(); if (opus_decoder_->sample_rate() != codec->output_sample_rate()) { ESP_LOGI(TAG, "Resampling audio from %d to %d", opus_decoder_->sample_rate(), codec->output_sample_rate()); output_resampler_.Configure(opus_decoder_->sample_rate(), codec->output_sample_rate()); } } void Application::UpdateIotStates() { auto& thing_manager = iot::ThingManager::GetInstance(); std::string states; if (thing_manager.GetStatesJson(states, true)) { protocol_->SendIotStates(states); } } void Application::Reboot() { ESP_LOGI(TAG, "Rebooting..."); esp_restart(); } // 唤醒词触发函数 void Application::WakeWordInvoke(const std::string& wake_word) { if (device_state_ == kDeviceStateIdle) { ToggleChatState(); Schedule([this, wake_word]() { if (protocol_) { protocol_->SendWakeWordDetected(wake_word); } }); } else if (device_state_ == kDeviceStateSpeaking) { //AbortSpeakingAndReturnToListening();// 使用唤醒词打断时立即切换到聆听状态 Schedule([this]() { AbortSpeaking(kAbortReasonNone); }); } else if (device_state_ == kDeviceStateListening) { Schedule([this]() { if (protocol_) { protocol_->CloseAudioChannel(); } }); } } bool Application::CanEnterSleepMode() { if (device_state_ != kDeviceStateIdle) { return false; } if (protocol_ && protocol_->IsAudioChannelOpened()) { return false; } // Now it is safe to enter sleep mode return true; } void Application::WaitForAudioPlayback() { // 等待 audio_decode_queue_ 清空且音频输出完成 auto codec = Board::GetInstance().GetAudioCodec(); int timeout_count = 0; const int max_timeout = 150; // 3秒超时 (150 * 20ms = 3000ms) while (timeout_count < max_timeout) { { std::lock_guard lock(mutex_); if (audio_decode_queue_.empty()) { // 检查音频输出是否已关闭或静音 if (!codec->output_enabled() || device_state_ != kDeviceStateSpeaking) { ESP_LOGI(TAG, "Audio playback completed"); break; } } } vTaskDelay(pdMS_TO_TICKS(20)); timeout_count++; } if (timeout_count >= max_timeout) { ESP_LOGW(TAG, "WaitForAudioPlayback timeout after 3 seconds"); } } bool Application::IsAudioQueueEmpty() { std::lock_guard lock(mutex_); return audio_decode_queue_.empty(); } void Application::ClearAudioQueue() { std::lock_guard lock(mutex_); audio_decode_queue_.clear(); audio_paused_ = false; // 清除暂停状态 // ESP_LOGI(TAG, "🧹 音频播放队列已清空,暂停状态已清除"); ESP_LOGI(TAG, "🎵 测试模式:音频开始播放,等待播放完成"); // 生产测试打印 // 重新启用音频编解码器输出,确保后续音频能正常播放 auto& board = Board::GetInstance(); auto codec = board.GetAudioCodec(); if (codec) { codec->EnableOutput(true); // ESP_LOGI(TAG, "🔧 音频编解码器输出已重新启用"); ESP_LOGI(TAG, "✅ 测试模式:音频播放完成"); // 生产测试打印 } } // 🔧 检查当前是否可以安全执行操作 bool Application::IsSafeToOperate() { // 检查是否正在执行中止操作 if (is_aborting_.load()) { return false; } // 检查最近是否有操作过于频繁 auto now = std::chrono::steady_clock::now(); auto last_op = last_safe_operation_.load(); auto time_diff = std::chrono::duration_cast(now - last_op); // 如果距离上次操作少于50ms,认为可能存在竞态风险 if (time_diff.count() < 50) { ESP_LOGD(TAG, "Operation too frequent, waiting for safety"); return false; } return true; } void Application::StopAudioProcessor() { #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.Stop(); #endif } // 🔴 专门处理从说话状态到空闲状态的切换 void Application::AbortSpeakingAndReturnToIdle() { ESP_LOGI(TAG, "🔴 AbortSpeakingAndReturnToIdle: Starting transition from speaking to idle state"); ESP_LOGI(TAG, "📊 当前设备状态: %s", STATE_STRINGS[device_state_]); ESP_LOGI(TAG, "🎯 目标状态: idle (空闲状态)"); // 检查当前状态是否为说话状态 if (device_state_ != kDeviceStateSpeaking) { ESP_LOGW(TAG, "🔴 AbortSpeakingAndReturnToIdle: Device not in speaking state, current state: %s", STATE_STRINGS[device_state_]); return; } ESP_LOGI(TAG, "✅ 状态检查通过,当前处于说话状态"); // 检查操作安全性 if (!IsSafeToOperate()) { ESP_LOGW(TAG, "🔴 AbortSpeakingAndReturnToIdle: Operation not safe, scheduling retry"); Schedule([this]() { vTaskDelay(pdMS_TO_TICKS(100)); AbortSpeakingAndReturnToIdle(); }); return; } // 更新安全操作时间戳 last_safe_operation_.store(std::chrono::steady_clock::now()); ESP_LOGI(TAG, "⏰ 安全操作时间戳已更新"); // 立即停止音频处理 ESP_LOGI(TAG, "🔇 开始停止音频处理"); { std::lock_guard lock(mutex_); if (!audio_decode_queue_.empty()) { ESP_LOGI(TAG, "🗑️ 清空音频队列,当前队列大小: %zu", audio_decode_queue_.size()); audio_decode_queue_.clear(); current_speaker_volume_ = 0.0f; #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.SetSpeakerVolume(0.0f); #endif ESP_LOGI(TAG, "✅ 音频队列已清空,音量已重置为0"); } else { ESP_LOGI(TAG, "ℹ️ 音频队列已为空,无需清空"); } } ESP_LOGI(TAG, "🔴 AbortSpeakingAndReturnToIdle: Sending abort message to server"); // 发送中止消息给服务器 if (protocol_ && protocol_->IsAudioChannelOpened()) { ESP_LOGI(TAG, "📡 WebSocket连接正常,发送中止消息"); try { protocol_->SendAbortSpeaking(kAbortReasonNone); ESP_LOGI(TAG, "✅ 中止消息发送成功"); } catch (const std::exception& e) { ESP_LOGW(TAG, "❌ 发送中止消息失败: %s", e.what()); } // 延迟100ms后主动关闭连接,确保服务器有时间处理中止消息 Schedule([this]() { vTaskDelay(pdMS_TO_TICKS(100)); ESP_LOGI(TAG, "⏳ 延迟100ms后开始主动关闭WebSocket连接"); ESP_LOGI(TAG, "🔌 执行主动断开WebSocket连接"); if (protocol_) { protocol_->CloseAudioChannel(); ESP_LOGI(TAG, "✅ CloseAudioChannel调用完成"); } else { ESP_LOGW(TAG, "⚠️ protocol_为空,无法关闭音频通道"); } }); } else { ESP_LOGW(TAG, "⚠️ WebSocket连接不可用,强制关闭连接"); if (protocol_) { ESP_LOGI(TAG, "🔌 强制执行WebSocket断开"); protocol_->CloseAudioChannel(); ESP_LOGI(TAG, "✅ 强制断开完成"); } else { ESP_LOGW(TAG, "❌ protocol_为空,无法执行断开操作"); } } ESP_LOGI(TAG, "🎯 主动断开流程已启动,等待OnAudioChannelClosed回调触发状态转换"); ESP_LOGI(TAG, "📋 预期流程: WebSocket断开 → 回调触发 → 转换到idle状态 → 播放待机音"); } // 🔵 专门处理从说话状态到聆听状态的切换 void Application::AbortSpeakingAndReturnToListening() { ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Starting transition from speaking to listening state (断开连接方案)"); // 检查当前状态是否为说话状态或可切换状态 // ========================================================================================= if (device_state_ != kDeviceStateSpeaking && device_state_ != kDeviceStateListening && device_state_ != kDeviceStateIdle) { ESP_LOGW(TAG, "🔵 AbortSpeakingAndReturnToListening: Device not in valid state for transition, current state: %s", STATE_STRINGS[device_state_]); return; } // 如果已经在listening状态,直接返回避免重复切换 if (device_state_ == kDeviceStateListening) { ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Already in listening state, skipping transition"); return; } // 🔧 检查并处理音频播放状态(BOOT按键优化方案) if (!audio_paused_ && device_state_ == kDeviceStateSpeaking) { ESP_LOGI(TAG, "🔵 检测到播放状态,一次按键完成暂停和状态切换"); // 第一步:禁用音频输出(立即停止播放) auto& board = Board::GetInstance();// 获取音频编解码器 auto codec = board.GetAudioCodec();// 获取音频编解码器 if (codec) { codec->EnableOutput(false);// 暂停时禁用音频编解码器输出 ESP_LOGI(TAG, "🔧 暂停时禁用音频编解码器输出"); } // 第二步:切换到暂停状态 audio_paused_ = true; ESP_LOGI(TAG, "✅ 已切换到暂停状态"); // 第三步:立即执行状态切换逻辑(不返回,继续执行下面的代码) ESP_LOGI(TAG, "🔵 继续执行状态切换到聆听状态"); } // 🔧 检查并处理音频暂停状态(BOOT按键优化方案) if (audio_paused_) { ESP_LOGI(TAG, "🔵 检测到音频暂停状态,应用BOOT按键优化方案"); audio_paused_ = false; ESP_LOGI(TAG, "✅ 音频暂停状态已清除"); // 🔧 关键优化:清空音频播放队列,避免播放暂停时残留的音频 std::unique_lock lock(mutex_); audio_decode_queue_.clear(); lock.unlock(); ESP_LOGI(TAG, "🧹 已清空音频播放队列,避免播放残留音频"); // BOOT按键切换时的优化方案:确保音频系统能正常响应状态切换 auto& board = Board::GetInstance(); auto codec = board.GetAudioCodec(); if (codec) { codec->EnableOutput(true); ESP_LOGI(TAG, "🔧 为状态切换重新启用音频编解码器输出");// 重新启用输出,后续可以播放 } // 🔧 关键修复:强制停止音频处理器,确保后续状态切换时能重新启动 #if CONFIG_USE_AUDIO_PROCESSOR if (audio_processor_.IsRunning()) { ESP_LOGI(TAG, "🔧 强制停止音频处理器以确保状态切换成功"); audio_processor_.Stop(); } #endif // 🔧 音频暂停状态下直接切换,避免复杂的异步处理 ESP_LOGI(TAG, "🔵 音频暂停状态下直接执行状态切换"); // 播放提示音 if (codec && codec->output_enabled()) { ESP_LOGI(TAG, "播放提示音:卡卡在呢"); // PlaySound(Lang::Sounds::P3_KAKAZAINNE); 原有蜡笔小新 音色播报 if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){ PlaySound(Lang::Sounds::P3_KAKA_ZAINNE); } else if(strcmp(CONFIG_DEVICE_ROLE, "RTC_Test") == 0){ PlaySound(Lang::Sounds::P3_LALA_ZAINNE); } // 简化等待逻辑 vTaskDelay(pdMS_TO_TICKS(620)); // 等待音效播放完成 ESP_LOGI(TAG, "音频播放完成"); } // 直接切换到聆听状态 SetDeviceState(kDeviceStateListening); ESP_LOGI(TAG, "🔵 音频暂停状态下状态切换完成"); return; } // ========================================================================================= // 检查操作安全性 if (!IsSafeToOperate()) { ESP_LOGW(TAG, "🔵 AbortSpeakingAndReturnToListening: Operation not safe, scheduling retry"); Schedule([this]() { vTaskDelay(pdMS_TO_TICKS(100)); AbortSpeakingAndReturnToListening(); }); return; } // 更新安全操作时间戳 last_safe_operation_.store(std::chrono::steady_clock::now()); // 立即停止音频处理器和清空音频队列 #if CONFIG_USE_AUDIO_PROCESSOR if (audio_processor_.IsRunning()) { ESP_LOGI(TAG, "🔵 停止音频处理器"); audio_processor_.Stop(); } // 清空音频队列并重置音量 if (!IsAudioQueueEmpty()) { ESP_LOGI(TAG, "🔵 清空音频队列并重置音量"); while (!IsAudioQueueEmpty()) { vTaskDelay(pdMS_TO_TICKS(10)); } current_speaker_volume_ = 0.0f; audio_processor_.SetSpeakerVolume(0.0f); ESP_LOGI(TAG, "✅ 音频队列已清空,音量已重置为0"); } else { ESP_LOGI(TAG, "ℹ️ 音频队列已为空,无需清空"); } #endif ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Sending abort message to server"); // 发送中止消息给服务器 if (protocol_ && protocol_->IsAudioChannelOpened()) { ESP_LOGI(TAG, "📡 WebSocket连接正常,发送中止消息"); try { protocol_->SendAbortSpeaking(kAbortReasonVoiceInterrupt); ESP_LOGI(TAG, "✅ 中止消息发送成功"); } catch (const std::exception& e) { ESP_LOGW(TAG, "❌ 发送中止消息失败: %s", e.what()); } // 延迟100ms后播放音效并直接切换到聆听状态,不关闭WebSocket连接 Schedule([this]() { vTaskDelay(pdMS_TO_TICKS(100)); ESP_LOGI(TAG, "⏳ 延迟100ms后播放音效并切换到聆听状态"); // 先播放"卡卡在呢"音效 ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Playing KAKAZAINNE sound"); // 🔧 修复:STATE:,确保硬件状态正确 auto& board = Board::GetInstance(); auto audio_codec = board.GetAudioCodec(); ESP_LOGI(TAG, "强制重新初始化音频输出"); audio_codec->EnableOutput(false); // 先关闭音频输出 vTaskDelay(pdMS_TO_TICKS(50)); // 短暂延迟让硬件复位 audio_codec->EnableOutput(true); // 再开启,强制硬件重新初始化 // 🔧 检查音频资源是否可用 if (audio_codec->output_enabled()) { ESP_LOGI(TAG, "播放提示音:卡卡在呢"); ResetDecoder(); // 🔧 关键修复:重置解码器状态,清除残留 // 获取当前系统音量并临时设置以确保音效能播放 float system_volume = audio_codec ? (audio_codec->output_volume() / 100.0f) : 0.7f; // 默认70% current_speaker_volume_ = system_volume; #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.SetSpeakerVolume(system_volume); ESP_LOGI(TAG, "✅ 音量设置成功: %.2f", system_volume); #endif // PlaySound(Lang::Sounds::P3_KAKAZAINNE); 原有蜡笔小新 音色播报 if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){ PlaySound(Lang::Sounds::P3_KAKA_ZAINNE); } else if(strcmp(CONFIG_DEVICE_ROLE, "RTC_Test") == 0){ PlaySound(Lang::Sounds::P3_LALA_ZAINNE); } // 🔧 修复:使用改进的等待逻辑,确保音频真正播放完成 ESP_LOGI(TAG, "等待音频播放完成..."); vTaskDelay(pdMS_TO_TICKS(100)); // 给音频足够的时间开始播放 // 等待音频队列清空 + 额外缓冲时间确保I2S硬件完成输出 int timeout_count = 0; const int max_timeout = 150; // 3秒超时 while (timeout_count < max_timeout) { if (IsAudioQueueEmpty()) { // 队列清空后,再等待500ms确保I2S硬件完成输出 ESP_LOGI(TAG, "音频队列已清空,等待硬件输出完成..."); vTaskDelay(pdMS_TO_TICKS(500)); ESP_LOGI(TAG, "音频播放完成"); break; } vTaskDelay(pdMS_TO_TICKS(20)); timeout_count++; } if (timeout_count >= max_timeout) { ESP_LOGW(TAG, "等待音频播放超时,继续状态切换"); } } else { ESP_LOGW(TAG, "音频输出无法启用,跳过提示音播放"); } // 直接切换到聆听状态,音频播放已在上面完成 ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Switching to listening state (保持WebSocket连接)"); SetDeviceState(kDeviceStateListening); }); } else { ESP_LOGW(TAG, "⚠️ WebSocket连接不可用,直接切换状态"); // 直接播放音效并切换状态 Schedule([this]() { vTaskDelay(pdMS_TO_TICKS(100)); ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Playing KAKAZAINNE sound"); // 🔧 修复:强制重新初始化音频输出,确保硬件状态正确 auto& board = Board::GetInstance(); auto audio_codec = board.GetAudioCodec(); ESP_LOGI(TAG, "强制重新初始化音频输出"); audio_codec->EnableOutput(false); // 先关闭音频输出 vTaskDelay(pdMS_TO_TICKS(50)); // 短暂延迟让硬件复位 audio_codec->EnableOutput(true); // 再开启,强制硬件重新初始化 // 🔧 检查音频资源是否可用 if (audio_codec->output_enabled()) { ESP_LOGI(TAG, "播放提示音:卡卡在呢"); ResetDecoder(); // 🔧 关键修复:重置解码器状态,清除残留 // 获取当前系统音量并临时设置以确保音效能播放 float system_volume = audio_codec ? (audio_codec->output_volume() / 100.0f) : 0.7f; // 默认70% current_speaker_volume_ = system_volume; #if CONFIG_USE_AUDIO_PROCESSOR audio_processor_.SetSpeakerVolume(system_volume); ESP_LOGI(TAG, "✅ 音量设置成功: %.2f", system_volume); #endif // PlaySound(Lang::Sounds::P3_KAKAZAINNE); 原有蜡笔小新 音色播报 if(strcmp(CONFIG_DEVICE_ROLE, "KAKA") == 0){ PlaySound(Lang::Sounds::P3_KAKA_ZAINNE); } else if(strcmp(CONFIG_DEVICE_ROLE, "RTC_Test") == 0){ PlaySound(Lang::Sounds::P3_LALA_ZAINNE); } // 🔧 修复:使用改进的等待逻辑,确保音频真正播放完成 ESP_LOGI(TAG, "等待音频播放完成..."); vTaskDelay(pdMS_TO_TICKS(100)); // 给音频足够的时间开始播放 // 等待音频队列清空 + 额外缓冲时间确保I2S硬件完成输出 int timeout_count = 0; const int max_timeout = 150; // 3秒超时 while (timeout_count < max_timeout) { if (IsAudioQueueEmpty()) { // 队列清空后,再等待500ms确保I2S硬件完成输出 ESP_LOGI(TAG, "音频队列已清空,等待硬件输出完成..."); vTaskDelay(pdMS_TO_TICKS(500)); ESP_LOGI(TAG, "音频播放完成"); break; } vTaskDelay(pdMS_TO_TICKS(20)); timeout_count++; } if (timeout_count >= max_timeout) { ESP_LOGW(TAG, "等待音频播放超时,继续状态切换"); } } else { ESP_LOGW(TAG, "音频输出无法启用,跳过提示音播放"); } // 直接切换到聆听状态,音频播放已在上面完成 ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Switching to listening state"); SetDeviceState(kDeviceStateListening); }); } ESP_LOGI(TAG, "🔵 AbortSpeakingAndReturnToListening: Transition initiated - keeping WebSocket connection and switching to listening"); } // 姿态传感器接口实现 bool Application::IsImuSensorAvailable() { auto& board = Board::GetInstance(); if (board.GetBoardType() == "movecall-moji-esp32s3") { auto& moji_board = static_cast(board); return moji_board.IsImuInitialized(); } return false; } bool Application::GetImuData(float* acc_x, float* acc_y, float* acc_z, float* gyro_x, float* gyro_y, float* gyro_z, float* temperature) { auto& board = Board::GetInstance(); if (board.GetBoardType() == "movecall-moji-esp32s3") { auto& moji_board = static_cast(board); qmi8658a_data_t imu_data; if (moji_board.GetImuData(&imu_data)) { if (acc_x) *acc_x = imu_data.acc_x; if (acc_y) *acc_y = imu_data.acc_y; if (acc_z) *acc_z = imu_data.acc_z; if (gyro_x) *gyro_x = imu_data.gyro_x; if (gyro_y) *gyro_y = imu_data.gyro_y; if (gyro_z) *gyro_z = imu_data.gyro_z; if (temperature) *temperature = imu_data.temperature; return true; } } return false; } void Application::ClearDialogIdleSkipSession() { // 清除内存中的跳过标志 skip_dialog_idle_session_ = false; // 清除NVS中的标志 Settings sys("system", true); sys.SetInt("reboot_dlg_idle", 0); sys.SetInt("reboot_origin", 0); sys.Commit(); ESP_LOGI(TAG, "跳过对话待机会话标志已清除"); } void Application::SetDialogUploadEnabled(bool enabled) { dialog_upload_enabled_ = enabled; ESP_LOGI(TAG, "对话上传状态已设置为: %s", enabled ? "启用" : "禁用"); } void Application::OnMotionDetected() { ESP_LOGI(TAG, "Motion detected by IMU sensor"); // 如果设备处于空闲状态,可以触发一些动作 if (device_state_ == kDeviceStateIdle) { // 例如:显示运动检测提示 auto& board = Board::GetInstance(); auto display = board.GetDisplay(); display->SetChatMessage("system", "检测到运动"); // 可以在这里添加更多的运动检测处理逻辑 // 比如:唤醒设备、记录运动数据等 } } void Application::SetLowBatteryTransition(bool value) { is_low_battery_transition_.store(value);// 设置低电量过渡状态 } bool Application::IsLowBatteryTransition() const { return is_low_battery_transition_.load();// 获取低电量过渡状态 } // 🌐 初始化WebSocket协议(RTC连接成功后调用) void Application::InitializeWebsocketProtocol() { ESP_LOGI(TAG, "🌐 开始初始化WebSocket协议..."); // 检查是否已经初始化过 if (websocket_protocol_) { ESP_LOGW(TAG, "⚠️ WebSocket协议已经初始化,跳过重复初始化"); return; } // 创建WebsocketProtocol实例 ESP_LOGI(TAG, "🔧 创建WebsocketProtocol实例"); websocket_protocol_ = std::make_unique(); websocket_protocol_->SetPrimary(false); websocket_protocol_->OnIncomingAudio([this](std::vector&& data) { if (!ws_downlink_enabled_.load()) { return; } ws_playback_active_.store(true); std::lock_guard lock(mutex_); size_t len = data.size(); audio_decode_queue_.emplace_back(std::move(data)); ESP_LOGD(TAG, "WS辅助音频入队: 字节=%zu 队列大小=%zu", len, audio_decode_queue_.size()); }); websocket_protocol_->OnIncomingJson([this](const cJSON* root) { auto type = cJSON_GetObjectItem(root, "type"); if (type && cJSON_IsString(type) && type->valuestring) { ESP_LOGD(TAG, "WS辅助JSON消息: %s", type->valuestring); if (strcmp(type->valuestring, "hello") == 0) { auto audio_params = cJSON_GetObjectItem(root, "audio_params"); if (audio_params && cJSON_IsObject(audio_params)) { auto sr = cJSON_GetObjectItem(audio_params, "sample_rate"); auto fd = cJSON_GetObjectItem(audio_params, "frame_duration"); int sample_rate = (sr && cJSON_IsNumber(sr)) ? sr->valueint : 16000; int frame_duration = (fd && cJSON_IsNumber(fd)) ? fd->valueint : 60; SetDecodeSampleRate(sample_rate, frame_duration); } else { SetDecodeSampleRate(16000, 60); } } } }); // 启动WebSocket协议 ESP_LOGI(TAG, "🚀 启动WebSocket协议"); websocket_protocol_->Start();// 启动WebSocket协议 ESP_LOGI(TAG, "✅ WebSocket协议初始化完成"); } // void Application::SendTextViaWebsocket(const std::string& text) { // Schedule([this, text]() { // if (websocket_protocol_ && websocket_protocol_->IsAudioChannelOpened()) { // websocket_protocol_->SendTextMessage(text); // ESP_LOGI(TAG, "WS辅助文本发送:%s", text.c_str()); // } else { // ESP_LOGW(TAG, "WS辅助未连接,丢弃文本:%s", text.c_str()); // } // }); // }