Rdzleo/toy-hardware
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
toy-hardware/esp-spot/example/adf/llm_touch_toy/main/main.c

411 lines
13 KiB
C
Raw Blame History

/* http client request example code
This example code is in the Public Domain (or CC0 licensed, at your option.)
Unless required by applicable law or agreed to in writing, this
software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied.
Demo video: https://www.bilibili.com/video/BV1ekRAYVEZ1/
Hardware: https://oshwhub.com/esp-college/esp-spot
*/
#include <stdio.h>
#include <string.h>
#include "freertos/FreeRTOS.h"
#include "freertos/event_groups.h"
#include "freertos/idf_additions.h"
#include "freertos/task.h"
#include "esp_log.h"
#include "nvs_flash.h"
#include "esp_wifi.h"
#include "audio_sys.h"
#include "audio_thread.h"
#include "esp_peripherals.h"
#include "periph_wifi.h"
#include "periph_sdcard.h"
#include "audio_mem.h"
#include "board.h"
#include "es8311.h"
#include "es7210.h"
#include "touch_button.h"
#include "iot_button.h"
#include "touch_sensor_lowlevel.h"
#include "audio_processor.h"
#include "coze_chat.h"
#include "led_driver.h"
#define DEFAULT_RAW_OPUS_BUFFER_SIZE (1024)
static char *TAG = "main";
#define TOUCH_CHANNEL_1 (3)
#define TOUCH_CHANNEL_2 (9)
#define TOUCH_CHANNEL_3 (13)
#define TOUCH_CHANNEL_4 (14)
#define LIGHT_TOUCH_THRESHOLD (0.15)
#define HEAVY_TOUCH_THRESHOLD (0.4)
/* To controll the audio event */
#define BIT_RECORDING_START (1 << 0)
static EventGroupHandle_t s_audio_event_group;
struct coze_ws_s {
coze_chat_handle_t chat;
audio_recorder_handle_t recorder;
audio_player_handle_t player;
char *recorder_buffer;
char *opus_raw_buffer;
int opus_raw_buffer_len;
enum {
PLAYBACK_STATE_IDLE,
PLAYBACK_STATE_PLAYING,
} player_state;
};
static struct coze_ws_s s_coze_ws;
static audio_board_handle_t board_handle = NULL;
static void audio_event_callback(coze_chat_event_t event, void *ctx)
{
if (event == COZE_CHAT_EVENT_CHAT_SPEECH_STARTED) {
ESP_LOGI(TAG, "chat start");
s_coze_ws.player_state = PLAYBACK_STATE_IDLE;
} else if (event == COZE_CHAT_EVENT_CHAT_SPEECH_STOPED) {
ESP_LOGI(TAG, "chat stop");
s_coze_ws.player_state = PLAYBACK_STATE_PLAYING;
}
}
static void audio_data_callback(char *data, int len, void *ctx)
{
#define frame_length_prefix (2)
if (len > s_coze_ws.opus_raw_buffer_len) {
s_coze_ws.opus_raw_buffer_len = len + frame_length_prefix;
s_coze_ws.opus_raw_buffer = audio_realloc(s_coze_ws.opus_raw_buffer, s_coze_ws.opus_raw_buffer_len);
}
ESP_LOGD(TAG, "data: %p, len: %d", data, len);
s_coze_ws.opus_raw_buffer[0] = (len >> 8) & 0xFF;
s_coze_ws.opus_raw_buffer[1] = len & 0xFF;
memcpy(s_coze_ws.opus_raw_buffer + frame_length_prefix, data, len);
len += frame_length_prefix;
player_pipeline_write(s_coze_ws.player, s_coze_ws.opus_raw_buffer, len);
}
static void audio_if_open()
{
s_coze_ws.recorder = recorder_pipeline_open();
s_coze_ws.player = player_pipeline_open();
recorder_pipeline_run(s_coze_ws.recorder);
player_pipeline_run();
}
/**
* @brief Temporarily mute the audio output to prevent pop or click noise
* during audio state transitions.
*
* This function briefly mutes the audio hardware for a few milliseconds
* and then unmutes it. It's typically used before starting or stopping
* playback to suppress unwanted transition noise.
*/
static void short_mute()
{
audio_hal_set_mute(board_handle->audio_hal, true);
vTaskDelay(pdMS_TO_TICKS(30));
audio_hal_set_mute(board_handle->audio_hal, false);
}
static void audio_data_read_task(void *pv)
{
#define recorder_buffer_size (640)
s_coze_ws.recorder_buffer = malloc(recorder_buffer_size);
s_audio_event_group = xEventGroupCreate();
xEventGroupSetBits(s_audio_event_group, BIT_RECORDING_START);
while (1) {
/* Stop reading audio if the WebSocket stopped */
xEventGroupWaitBits(s_audio_event_group,
BIT_RECORDING_START,
pdFALSE,
pdTRUE,
portMAX_DELAY);
int r_len = recorder_pipeline_read(s_coze_ws.recorder, s_coze_ws.recorder_buffer, recorder_buffer_size);
if (r_len > 0) {
coze_chat_send_audio_data(s_coze_ws.chat, s_coze_ws.recorder_buffer, r_len);
}
}
vTaskDelete(NULL);
}
static void audio_tone_player_event_cb(audio_element_status_t evt)
{
if (evt == AEL_STATUS_STATE_FINISHED) {
// add more functions here
}
}
static void touch_event_nose(void *arg, void *data)
{
button_event_t event = iot_button_get_event(arg);
ESP_LOGI(TAG, "touch event nose: %s", iot_button_get_event_str(event));
// Stop coze or local audio
player_pipeline_stop();
audio_tone_stop();
audio_tone_play(TONE_TYPE_TOUCH_NOSE);
}
static void touch_event_nose_long(void *arg, void *data)
{
button_event_t event = iot_button_get_event(arg);
ESP_LOGI(TAG, "touch event nose long: %s", iot_button_get_event_str(event));
// Stop coze or local audio
player_pipeline_stop();
audio_tone_stop();
audio_tone_play(TONE_TYPE_SCREAMING);
}
static void touch_event_hat(void *arg, void *data)
{
button_event_t event = iot_button_get_event(arg);
ESP_LOGI(TAG, "touch event hat: %s", iot_button_get_event_str(event));
// Stop coze or local audio
player_pipeline_stop();
audio_tone_stop();
static tone_type_t hat_state = TONE_TYPE_HAT_1;
audio_tone_play(hat_state);
// Toggle between two tone states and update LED mode
if (hat_state == TONE_TYPE_CAPYBARA_SONG_1) {
// Turn on LED
led_set_mode(4);
hat_state = TONE_TYPE_HAT_1;
} else {
hat_state = TONE_TYPE_CAPYBARA_SONG_1;
}
}
static void touch_event_belly(void *arg, void *data)
{
button_event_t event = iot_button_get_event(arg);
ESP_LOGI(TAG, "touch event hat: %s", iot_button_get_event_str(event));
// Stop coze or local audio
player_pipeline_stop();
audio_tone_stop();
static int audio_belly_count_g = 0;
audio_tone_play(audio_belly_count_g);
// Update the counter for next playback
if (audio_belly_count_g < TONE_TYPE_BELLY_4) {
audio_belly_count_g++;
} else if (audio_belly_count_g == TONE_TYPE_BELLY_4) {
audio_belly_count_g = TONE_TYPE_SCREAMING;
} else {
audio_belly_count_g = TONE_TYPE_BELLY_1;
}
}
static void touch_event_neck(void *arg, void *data)
{
button_event_t event = iot_button_get_event(arg);
ESP_LOGI(TAG, "touch event hat: %s", iot_button_get_event_str(event));
// Stop coze or local audio
player_pipeline_stop();
audio_tone_stop();
static tone_type_t neck_state = TONE_TYPE_NECK_2;
// Toggle between two tone states and update LED mode
if (neck_state == TONE_TYPE_NECK_2) {
// Turn on LED
xEventGroupClearBits(s_audio_event_group, BIT_RECORDING_START);
recorder_pipeline_stop(s_coze_ws.recorder);
coze_chat_stop(s_coze_ws.chat);
audio_tone_play(TONE_TYPE_NECK_2);
neck_state = TONE_TYPE_NECK_1;
} else {
audio_tone_play(TONE_TYPE_NECK_1);
neck_state = TONE_TYPE_NECK_2;
coze_chat_start(s_coze_ws.chat);
xEventGroupSetBits(s_audio_event_group, BIT_RECORDING_START);
recorder_pipeline_run(s_coze_ws.recorder);
}
}
static void touch_task(void *arg)
{
// Register all touch channel
uint32_t touch_channel_list[] = {TOUCH_CHANNEL_1, TOUCH_CHANNEL_2, TOUCH_CHANNEL_3, TOUCH_CHANNEL_4};
int total_channel_num = sizeof(touch_channel_list) / sizeof(touch_channel_list[0]);
// calloc channel_type for every button from the list
touch_lowlevel_type_t *channel_type = calloc(total_channel_num, sizeof(touch_lowlevel_type_t));
assert(channel_type);
for (int i = 0; i < total_channel_num; i++) {
channel_type[i] = TOUCH_LOWLEVEL_TYPE_TOUCH;
}
touch_lowlevel_config_t low_config = {
.channel_num = total_channel_num,
.channel_list = touch_channel_list,
.channel_type = channel_type,
};
esp_err_t ret = touch_sensor_lowlevel_create(&low_config);
assert(ret == ESP_OK);
free(channel_type);
const button_config_t btn_cfg = {
.short_press_time = 300,
.long_press_time = 2000,
};
/* ============================= Init touch IO3 ============================= */
button_touch_config_t touch_cfg_1 = {
.touch_channel = touch_channel_list[0],
.channel_threshold = LIGHT_TOUCH_THRESHOLD,
.skip_lowlevel_init = true,
};
/* Create button for nose */
button_handle_t btn_nose = NULL;
ret = iot_button_new_touch_button_device(&btn_cfg, &touch_cfg_1, &btn_nose);
assert(ret == ESP_OK);
/* ============================= Init touch IO9 ============================= */
button_touch_config_t touch_cfg_2 = {
.touch_channel = touch_channel_list[1],
.channel_threshold = LIGHT_TOUCH_THRESHOLD,
.skip_lowlevel_init = true,
};
/* Create button for hat */
button_handle_t btn_hat = NULL;
ret = iot_button_new_touch_button_device(&btn_cfg, &touch_cfg_2, &btn_hat);
assert(ret == ESP_OK);
/* ============================= Init touch IO13 ============================= */
button_touch_config_t touch_cfg_3 = {
.touch_channel = touch_channel_list[2],
.channel_threshold = LIGHT_TOUCH_THRESHOLD,
.skip_lowlevel_init = true,
};
/* Create light press button */
button_handle_t btn_belly = NULL;
ret = iot_button_new_touch_button_device(&btn_cfg, &touch_cfg_3, &btn_belly);
assert(ret == ESP_OK);
/* ============================= Init touch IO14 ============================= */
button_touch_config_t touch_cfg_4 = {
.touch_channel = touch_channel_list[3],
.channel_threshold = LIGHT_TOUCH_THRESHOLD,
.skip_lowlevel_init = true,
};
/* Create button for nose */
button_handle_t btn_neck = NULL;
ret = iot_button_new_touch_button_device(&btn_cfg, &touch_cfg_4, &btn_neck);
assert(ret == ESP_OK);
/* ========================== Register touch callback ========================== */
// Register touch callback for nose
iot_button_register_cb(btn_nose, BUTTON_PRESS_DOWN, NULL, touch_event_nose, NULL);
iot_button_register_cb(btn_nose, BUTTON_LONG_PRESS_START, NULL, touch_event_nose_long, NULL);
// Register touch callback for hat
iot_button_register_cb(btn_hat, BUTTON_PRESS_DOWN, NULL, touch_event_hat, NULL);
// Register touch callback for belly
iot_button_register_cb(btn_belly, BUTTON_PRESS_DOWN, NULL, touch_event_belly, NULL);
// Register touch callback for nose
iot_button_register_cb(btn_neck, BUTTON_PRESS_DOWN, NULL, touch_event_neck, NULL);
touch_sensor_lowlevel_start();
while (1) {
vTaskDelay(pdMS_TO_TICKS(1000));
}
}
static void led_task(void *arg)
{
/* Configure the LED strip and obtain a handle */
led_strip_handle_t led_strip = led_create();
led_set_mode(1);
vTaskDelay(1000);
led_set_mode(0);
while(1) {
/* Run the LED animation based on LED configuration and weather data */
ESP_ERROR_CHECK(led_animations_start(led_strip));
}
vTaskDelete(NULL);
}
void app_main(void)
{
AUDIO_MEM_SHOW(TAG);
ESP_ERROR_CHECK(nvs_flash_init());
ESP_ERROR_CHECK(esp_netif_init());
ESP_LOGI(TAG, "Initialize board peripherals");
esp_periph_config_t periph_cfg = DEFAULT_ESP_PERIPH_SET_CONFIG();
esp_periph_set_handle_t set = esp_periph_set_init(&periph_cfg);
board_handle = audio_board_init();
audio_hal_ctrl_codec(board_handle->audio_hal, AUDIO_HAL_CODEC_MODE_BOTH, AUDIO_HAL_CTRL_START);
#if CONFIG_ESP32_S3_SPOT_BOARD
audio_hal_set_volume(board_handle->audio_hal, 90);
es8311_set_mic_gain(ES8311_MIC_GAIN_0DB);
#endif
s_coze_ws.opus_raw_buffer_len = DEFAULT_RAW_OPUS_BUFFER_SIZE;
s_coze_ws.opus_raw_buffer = audio_malloc(s_coze_ws.opus_raw_buffer_len);
// Initialize SD Card peripheral
// audio_board_sdcard_init(set, SD_MODE_1_LINE);
periph_wifi_cfg_t wifi_cfg = {
.wifi_config.sta.ssid = CONFIG_ESP_WIFI_SSID,
.wifi_config.sta.password = CONFIG_ESP_WIFI_PASSWORD,
};
esp_periph_handle_t wifi_handle = periph_wifi_init(&wifi_cfg);
esp_periph_start(set, wifi_handle);
periph_wifi_wait_for_connected(wifi_handle, portMAX_DELAY);
coze_chat_config_t chat_config = COZE_CHAT_DEFAULT_CONFIG();
chat_config.bot_id = CONFIG_BOT_ID;
chat_config.access_token = CONFIG_ACCESS_TOKEN;
chat_config.audio_callback = audio_data_callback;
chat_config.event_callback = audio_event_callback;
s_coze_ws.chat = coze_chat_init(&chat_config);
coze_chat_start(s_coze_ws.chat);
audio_if_open();
audio_thread_create(NULL, "audio_data_read_task", audio_data_read_task, (void *)NULL, 1024 * 4, 12, true, 1);
xTaskCreate(touch_task, "touch_task", 1024 * 5, NULL, 5, NULL);
xTaskCreate(led_task, "led_task", 1024 * 3, NULL, 5, NULL);
audio_tone_init(audio_tone_player_event_cb);
AUDIO_MEM_SHOW(TAG);
}
Reference in New Issue View Git Blame Copy Permalink