From 2a77110654c7a412d14de5679a841c6ddcd7a698 Mon Sep 17 00:00:00 2001 From: Rdzleo Date: Fri, 10 Apr 2026 10:26:52 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=AF=AD=E9=9F=B3=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E6=9C=BA=E5=99=A8=E4=BA=BA=E5=90=84=E9=83=A8=E4=BD=8D?= =?UTF-8?q?=E8=88=B5=E6=9C=BA=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. RP2040 新增一次性动作系统（oneshot）： - 支持多次来回摆动，动作完成后自动恢复原位 - 动作期间临时提升舵机速度（boost 8-15倍），确保全幅度快速响应 - 动作执行期间暂停常规状态动画和人脸追踪，避免舵机目标冲突 - 支持12个单动作（左右耳/左右眼球/点头/摇头/歪头/张嘴/眨眼/闭眼/睁眼/转身） - 支持6个组合动作（双耳/点头眨眼/摇头/张望/卖萌/跳舞） 2. ESP32 新增语音指令识别（stt关键词匹配）： - 从用户语音识别结果中匹配中文关键词，直接发送动作指令给RP2040 - 不依赖AI回复格式，用户说话后立即触发动作 - 支持ASR识别变体容错（如"耳朵"/"耳刀"/"耳"均可匹配） 3. ESP32 新增AI回复文本动作解析（备用通道）： - 支持从AI回复文本中解析全角括号包裹的动作代码 4. 新增LLM情绪标签日志，便于调试AI返回内容 Co-Authored-By: Claude Opus 4.6 (1M context) --- RP2040/animation.py | 203 +++++++++++++++++++++++++++++++++++++++++++- RP2040/main.py | 30 +++++-- main/application.cc | 104 +++++++++++++++++++++++ 3 files changed, 327 insertions(+), 10 deletions(-) diff --git a/RP2040/animation.py b/RP2040/animation.py index 875af5f..cdcda36 100644 --- a/RP2040/animation.py +++ b/RP2040/animation.py @@ -307,7 +307,7 @@ def state_limber_up(): # - listening # - speaking -state_map={ +state_map={ "neutral": state_neutral, "idle": state_sleep, "listening": state_listen, @@ -319,6 +319,207 @@ state_map={ "state_calibrate": state_calibrate } +#_________________# 一次性动作系统 #_________________# + +oneshot_active = False # 是否正在执行一次性动作 +oneshot_start_time = 0 +oneshot_duration = 0 # 每段（半程）持续时间 +oneshot_phase = 0 # 当前阶段：0,1,2,...,total_phases-1 +oneshot_total_phases = 0 # 总阶段数（来回次数×2 + 1恢复阶段） +oneshot_pos_a = {} # A 位置（摆动极限一侧） +oneshot_pos_b = {} # B 位置（摆动极限另一侧） +saved_positions = {} # 动作前各舵机位置快照 + +saved_speeds = {} # 动作前各舵机的速度/加速度备份 + +def save_servo_positions(): + """保存当前所有舵机实际位置和运动参数""" + global saved_positions, saved_speeds + saved_positions = {} + saved_speeds = {} + for name, servo in servos.items(): + saved_positions[name] = servo.pos + saved_speeds[name] = (servo.max_speed, servo.max_accel) + +def restore_servo_positions(): + """恢复舵机到保存的位置和运动参数（只恢复被动作修改过的舵机）""" + for name in oneshot_pos_a: + if name in saved_positions and name in servos: + servos[name].set_target(saved_positions[name]) + if name in saved_speeds and name in servos: + servos[name].max_speed, servos[name].max_accel = saved_speeds[name] + servos[name]._inv_2_accel = 1.0 / (2.0 * servos[name].max_accel) + +def boost_servos(names): + """临时提升指定舵机的速度和加速度，让动作又快又猛""" + for name in names: + if name in servos: + servos[name].max_speed = 2000.0 # 原来 250，提升 8 倍 + servos[name].max_accel = 3000.0 # 原来 200，提升 15 倍 + servos[name]._inv_2_accel = 1.0 / (2.0 * 3000.0) + +def start_oneshot(pos_a, pos_b, duration_ms=200, repeats=3): + """ + 启动多次来回摆动的一次性动作 + pos_a: {舵机名: 角度} — 摆动一侧极限 + pos_b: {舵机名: 角度} — 摆动另一侧极限（None 则用保存的原位） + duration_ms: 每半程持续时间（越小越快） + repeats: 来回摆动次数 + """ + global oneshot_active, oneshot_start_time, oneshot_duration + global oneshot_phase, oneshot_total_phases + global oneshot_pos_a, oneshot_pos_b + + if oneshot_active: + return + + save_servo_positions() + oneshot_pos_a = pos_a + # pos_b 为 None 时用保存的原位作为 B 位置 + if pos_b is None: + oneshot_pos_b = {} + for name in pos_a: + if name in saved_positions: + oneshot_pos_b[name] = saved_positions[name] + else: + oneshot_pos_b = pos_b + oneshot_duration = duration_ms + oneshot_phase = 0 + oneshot_total_phases = repeats * 2 + 1 + oneshot_start_time = time.ticks_ms() + oneshot_active = True + + # 临时提升涉及的舵机速度，让动作又快又猛 + boost_servos(pos_a.keys()) + if pos_b: + boost_servos(pos_b.keys()) + + # 第一阶段：去 A 位置 + for name, angle in pos_a.items(): + if name in servos: + servos[name].set_target(angle) + +def update_oneshot(): + """主循环中调用，管理多次来回摆动""" + global oneshot_active, oneshot_phase, oneshot_start_time + + if not oneshot_active: + return + + elapsed = time.ticks_diff(time.ticks_ms(), oneshot_start_time) + if elapsed < oneshot_duration: + return + + # 当前阶段完成，进入下一阶段 + oneshot_phase += 1 + oneshot_start_time = time.ticks_ms() + + if oneshot_phase >= oneshot_total_phases: + # 所有阶段完成，恢复原位 + restore_servo_positions() + oneshot_active = False + return + + # 奇数阶段去 B，偶数阶段去 A + side = "B" if (oneshot_phase % 2 == 1) else "A" + targets = oneshot_pos_b if (oneshot_phase % 2 == 1) else oneshot_pos_a + for name, angle in targets.items(): + if name in servos: + servos[name].set_target(angle) + +# --- 动作定义 --- +# duration_ms=每半程时间（越小越快），repeats=来回次数 + +def act_ear_left(): + # 左耳：全幅度快速来回 3 次（速度已 boost，250ms 足够跑满 90° 行程） + start_oneshot({"EAL": 60}, {"EAL": 150}, 250, 3) + +def act_ear_right(): + start_oneshot({"EAR": 120}, {"EAR": 30}, 250, 3) + +def act_eye_left(): + start_oneshot({"EYL": 30, "EYR": 30}, {"EYL": 150, "EYR": 150}, 250, 3) + +def act_eye_right(): + start_oneshot({"EYL": 150, "EYR": 150}, {"EYL": 30, "EYR": 30}, 250, 3) + +def act_tilt_left(): + start_oneshot({"ROL": 30}, {"ROL": 120}, 300, 2) + +def act_tilt_right(): + start_oneshot({"ROL": 120}, {"ROL": 30}, 300, 2) + +def act_nod(): + # 大幅度快速点头 3 次 + start_oneshot({"PIT": 1}, {"PIT": 80}, 250, 3) + +def act_mouth(): + start_oneshot({"MOU": 5}, {"MOU": 150}, 200, 3) + +def act_blink(): + # 快速眨眼 3 次 + start_oneshot({"LID": 30}, {"LID": 110}, 150, 3) + +def act_eyes_close(): + global oneshot_active + if oneshot_active: + return + save_servo_positions() + servos["LID"]._write_pwm(30) + +def act_eyes_open(): + global oneshot_active + if oneshot_active: + return + save_servo_positions() + servos["LID"]._write_pwm(110) + +def act_body_turn(): + start_oneshot({"YAW": 30}, {"YAW": 150}, 300, 2) + +# --- 组合动作 --- + +def act_ears_both(): + start_oneshot({"EAL": 60, "EAR": 120}, {"EAL": 150, "EAR": 30}, 250, 3) + +def act_nod_blink(): + start_oneshot({"PIT": 1, "LID": 30}, {"PIT": 80, "LID": 110}, 250, 3) + +def act_shake_head(): + start_oneshot({"ROL": 30}, {"ROL": 120}, 250, 3) + +def act_look_around(): + start_oneshot({"EYL": 30, "EYR": 30, "YAW": 50}, {"EYL": 150, "EYR": 150, "YAW": 130}, 300, 2) + +def act_cute(): + start_oneshot({"ROL": 50, "LID": 30, "EAL": 60}, {"ROL": 120, "LID": 110, "EAL": 150}, 250, 2) + +def act_dance(): + start_oneshot({"YAW": 40, "ROL": 50, "EAL": 60, "EAR": 120}, {"YAW": 140, "ROL": 120, "EAL": 150, "EAR": 30}, 300, 3) + +action_map = { + # 单动作 + "act_ear_left": act_ear_left, + "act_ear_right": act_ear_right, + "act_eye_left": act_eye_left, + "act_eye_right": act_eye_right, + "act_tilt_left": act_tilt_left, + "act_tilt_right": act_tilt_right, + "act_nod": act_nod, + "act_mouth": act_mouth, + "act_blink": act_blink, + "act_eyes_close": act_eyes_close, + "act_eyes_open": act_eyes_open, + "act_body_turn": act_body_turn, + # 组合动作 + "act_ears_both": act_ears_both, + "act_nod_blink": act_nod_blink, + "act_shake_head": act_shake_head, + "act_look_around": act_look_around, + "act_cute": act_cute, + "act_dance": act_dance, +} + #_________________# #_________________# diff --git a/RP2040/main.py b/RP2040/main.py index 76cb971..45a1549 100644 --- a/RP2040/main.py +++ b/RP2040/main.py @@ -123,7 +123,10 @@ while True: # Grab all pending commands from the ESP incoming_commands = external.esp_read() for data in incoming_commands: - if data in animation.state_map: + if data in animation.action_map: + # 一次性动作：不改变 current_state + animation.action_map[data]() + elif data in animation.state_map: animation.new_state_flag = True animation.current_state = data @@ -137,14 +140,14 @@ while True: # Snap the mouth closed when speech ends! animation.servos["MOU"].set_target(130) - # 3. Speaking timer / animation - if speaking_flag: + # 3. Speaking timer / animation（一次性动作期间暂停，避免覆盖 MOU 目标） + if speaking_flag and not animation.oneshot_active: # Set the mouth position based on the current toggle state if bool_a == False: animation.servos["MOU"].set_target(130) elif bool_a == True: animation.servos["MOU"].set_target(70) - + # Flip the boolean every 250ms to flap the mouth if time.ticks_diff(now, last_toggle_a) >= 250: bool_a = not bool_a # flip the boolean @@ -157,7 +160,10 @@ while True: blink_counter = blink_time - # Calibration mode + # 更新一次性动作状态 + animation.update_oneshot() + + # Calibration mode if (mode.value() == 1): animation.current_state = "state_calibrate" animation.apply_state("state_calibrate") # change this back to calibrate to keep calibration mode/base for testing @@ -165,7 +171,8 @@ while True: if LED_countdown <= 0: LED.toggle() LED_countdown = LED_oscillator - else: + elif not animation.oneshot_active: + # 一次性动作期间跳过常规状态动画，避免舵机目标冲突 if animation.current_state == "idle": animation.apply_state("idle") animation.servos["LID"]._write_pwm(30) @@ -173,7 +180,7 @@ while True: if animation.previous_state == "idle": animation.servos["LID"]._write_pwm(110) animation.servos["PIT"].set_target(10) - # Blink mode + # Blink mode if blinking and animation.current_state != "idle": if blink_counter > blink_time - 50: # closed @@ -184,7 +191,7 @@ while True: blink_counter -= 1 if blink_counter == 0: blinking = False - + # Update with whatever state the ESP says animation.apply_state(animation.current_state) @@ -193,7 +200,12 @@ while True: animation.servos["LID"]._write_pwm(110) animation.current_state = "neutral" - facetrack() + # 一次性动作期间暂停人脸追踪，避免覆盖 EYL/EYR/PIT/YAW 目标 + # 但仍需读取 Grove 数据防止串口缓冲区溢出 + if animation.oneshot_active: + external.grove_read() # 消费数据，不驱动舵机 + else: + facetrack() # Update all servos except eyelids for name, s in servos.items(): diff --git a/main/application.cc b/main/application.cc index ade4bfc..beb79f1 100644 --- a/main/application.cc +++ b/main/application.cc @@ -475,6 +475,57 @@ void Application::Start() { auto text = cJSON_GetObjectItem(root, "text"); if (cJSON_IsString(text)) { ESP_LOGI(TAG, "<< %s", text->valuestring); + // 从 AI 回复文本中解析动作代码，发送给 RP2040 + // 格式：（EAL）好啦好啦～ TTS 的 IgnoreBracketText 会自动跳过（）内容 + // （= UTF-8: 0xEF 0xBC 0x88 ）= UTF-8: 0xEF 0xBC 0x89 + const char* t = text->valuestring; + if ((unsigned char)t[0] == 0xEF && (unsigned char)t[1] == 0xBC && (unsigned char)t[2] == 0x88) { + // 找到全角左括号（，搜索对应的） + const char* content = t + 3; + const char* end = content; + while (*end) { + if ((unsigned char)end[0] == 0xEF && (unsigned char)end[1] == 0xBC && (unsigned char)end[2] == 0x89) { + break; + } + end++; + } + if (*end) { + int code_len = end - content; + if (code_len > 0 && code_len <= 8) { + char action_code[9] = {}; + memcpy(action_code, content, code_len); + static const struct { const char* code; const char* cmd; } action_table[] = { + // 单动作 + {"EAL", "act_ear_left"}, + {"EAR", "act_ear_right"}, + {"EYL", "act_eye_left"}, + {"EYR", "act_eye_right"}, + {"TLL", "act_tilt_left"}, + {"TLR", "act_tilt_right"}, + {"NOD", "act_nod"}, + {"MOU", "act_mouth"}, + {"BLK", "act_blink"}, + {"ECL", "act_eyes_close"}, + {"EOP", "act_eyes_open"}, + {"BTN", "act_body_turn"}, + // 组合动作 + {"EARB", "act_ears_both"}, + {"NDBK", "act_nod_blink"}, + {"SHAK", "act_shake_head"}, + {"LOOK", "act_look_around"}, + {"CUTE", "act_cute"}, + {"DANC", "act_dance"}, + }; + for (const auto& entry : action_table) { + if (strcmp(action_code, entry.code) == 0) { + ESP_LOGI(TAG, "Action: %s -> %s", action_code, entry.cmd); + uart_send_string(entry.cmd); + break; + } + } + } + } + } Schedule([this, display, message = std::string(text->valuestring)]() { display->SetChatMessage("assistant", message.c_str()); }); @@ -484,6 +535,58 @@ void Application::Start() { auto text = cJSON_GetObjectItem(root, "text"); if (cJSON_IsString(text)) { ESP_LOGI(TAG, ">> %s", text->valuestring); + // 从用户语音指令中匹配身体动作关键词，直接发送给 RP2040 + const char* s = text->valuestring; + const char* action_cmd = nullptr; + // 判断是否包含"耳"相关（"耳朵"或"耳"，ASR 可能识别为"耳刀"等） + bool has_ear = strstr(s, "耳") != nullptr; + bool has_left = strstr(s, "左") != nullptr; + bool has_right = strstr(s, "右") != nullptr; + // 优先匹配带方向的耳朵 + if (has_ear && has_left) { + action_cmd = "act_ear_left"; + } else if (has_ear && has_right) { + action_cmd = "act_ear_right"; + } else if (has_ear) { + action_cmd = "act_ears_both"; + // 头部动作 + } else if (strstr(s, "点头") || strstr(s, "点一下头") || strstr(s, "点点头")) { + action_cmd = "act_nod"; + } else if (strstr(s, "摇头") || strstr(s, "摇摇头") || strstr(s, "摇一下头")) { + action_cmd = "act_shake_head"; + } else if (strstr(s, "歪") && has_left) { + action_cmd = "act_tilt_left"; + } else if (strstr(s, "歪") && has_right) { + action_cmd = "act_tilt_right"; + } else if (strstr(s, "歪头") || strstr(s, "歪一下")) { + action_cmd = "act_tilt_left"; + // 眼睛动作 + } else if ((strstr(s, "眼") || strstr(s, "看")) && has_left) { + action_cmd = "act_eye_left"; + } else if ((strstr(s, "眼") || strstr(s, "看")) && has_right) { + action_cmd = "act_eye_right"; + } else if (strstr(s, "眨眼") || strstr(s, "眨一下")) { + action_cmd = "act_blink"; + } else if (strstr(s, "闭眼") || strstr(s, "闭上眼")) { + action_cmd = "act_eyes_close"; + } else if (strstr(s, "睁眼") || strstr(s, "睁开眼")) { + action_cmd = "act_eyes_open"; + // 嘴巴 + } else if (strstr(s, "张嘴") || strstr(s, "嘴巴") || strstr(s, "张开嘴")) { + action_cmd = "act_mouth"; + // 身体 + } else if (strstr(s, "转身") || strstr(s, "转动") || strstr(s, "转个") || strstr(s, "转一下")) { + action_cmd = "act_body_turn"; + // 组合/特殊 + } else if (strstr(s, "卖萌")) { + action_cmd = "act_cute"; + } else if (strstr(s, "跳舞") || strstr(s, "跳个舞")) { + action_cmd = "act_dance"; + } + if (action_cmd) { + ESP_LOGI(TAG, "Voice action: [%s] -> %s", s, action_cmd); + uart_send_string(action_cmd); + } Schedule([this, display, message = std::string(text->valuestring)]() { display->SetChatMessage("user", message.c_str()); }); @@ -492,6 +595,7 @@ void Application::Start() { // LLM 回复中携带的情绪标签（如 "happy"、"thinking"） auto emotion = cJSON_GetObjectItem(root, "emotion"); if (cJSON_IsString(emotion)) { + ESP_LOGI(TAG, "LLM emotion: [%s]", emotion->valuestring); Schedule([this, display, emotion_str = std::string(emotion->valuestring)]() { display->SetEmotion(emotion_str.c_str()); // 将 AI 返回的情绪标签发送给 RP2040，实时切换舵机动画表情