diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/1.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/1.png new file mode 100644 index 0000000000000000000000000000000000000000..25f086a70fc884b0da0d434cb7026fcc59c612b5 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/1.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/2.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/2.png new file mode 100644 index 0000000000000000000000000000000000000000..b8b4b1d55d0ad2000d415418dd85480c3725983e Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/2.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/3.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/3.png new file mode 100644 index 0000000000000000000000000000000000000000..c611da614d463c06a6a74db83ec653af019bca0c Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/3.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/EC800MCNLER06A01M08_OCPU_QPY_TEST0810.zip b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/EC800MCNLER06A01M08_OCPU_QPY_TEST0810.zip new file mode 100644 index 0000000000000000000000000000000000000000..f81ad0721631b2f2a1694982a66461262732d17e Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/EC800MCNLER06A01M08_OCPU_QPY_TEST0810.zip differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/EVB.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/EVB.png new file mode 100644 index 0000000000000000000000000000000000000000..cb66506770269e6387842ec73d3dfd5e969d2e40 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/EVB.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/code_ai.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/code_ai.png new file mode 100644 index 0000000000000000000000000000000000000000..c8c70e96d117afb572d9980735f9a3f34bb804fe Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/code_ai.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/code_ai_audio.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/code_ai_audio.png new file mode 100644 index 0000000000000000000000000000000000000000..13aafe2026e60d8785280cd67ba8457cc946af92 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/code_ai_audio.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/comport.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/comport.png new file mode 100644 index 0000000000000000000000000000000000000000..270017b82e7cae2e731cfa8deb67bd6530ed6485 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/comport.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/connect.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/connect.png new file mode 100644 index 0000000000000000000000000000000000000000..c7a8123ed1eed4fb7a0f981473016f0da2ba3d2b Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/connect.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/encrypt.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/encrypt.png new file mode 100644 index 0000000000000000000000000000000000000000..09284bc7e3eaf518aa412ff6ca46f1182d05ffb5 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/encrypt.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/flow.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/flow.png new file mode 100644 index 0000000000000000000000000000000000000000..3d103d133950bf72382dbdbb3f3c82e172e06855 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/flow.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/mqtt.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/mqtt.png new file mode 100644 index 0000000000000000000000000000000000000000..4df590b908e8571d84c745231433a9d08b2b2946 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/mqtt.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/ota.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/ota.png new file mode 100644 index 0000000000000000000000000000000000000000..5121523606a110a43879d5651244039413913564 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/ota.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/ota_ret.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/ota_ret.png new file mode 100644 index 0000000000000000000000000000000000000000..842875ec1e058376a925e4cc35186fb1a3c22c52 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/ota_ret.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/software.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/software.png new file mode 100644 index 0000000000000000000000000000000000000000..a9b894dd1237eb172af282fae806f09ff1368ebd Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/software.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/tools_call.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/tools_call.png new file mode 100644 index 0000000000000000000000000000000000000000..b2493d0ffb4dad56de82ce2c46ab0c7faa8768b7 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/tools_call.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/tools_list.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/tools_list.png new file mode 100644 index 0000000000000000000000000000000000000000..e7801289e635ce3c68ff834ee3a5daca60736f5f Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/tools_list.png differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/udp.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/udp.png new file mode 100644 index 0000000000000000000000000000000000000000..e6465b97c0d725d5be2006fd174c3c4b113b3f01 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI_mqtt/udp.png differ diff --git a/docs/Application_guide/en/sidebar.yaml b/docs/Application_guide/en/sidebar.yaml index 29841d1438e606268a6db9770e446169c410de21..30d48252ce253b946b46033174630bde123f9428 100644 --- a/docs/Application_guide/en/sidebar.yaml +++ b/docs/Application_guide/en/sidebar.yaml @@ -277,7 +277,16 @@ items: - label: quick start file: solutions/xiaozhi_AI/quick_start.md - label: quick start - file: solutions/xiaozhi_AI/software_design.md + file: solutions/xiaozhi_AI/software_design.md + - label: XiaoZhi AI Chatbot based on MQTT and UDP + file: solutions/Xiaozhi_AI_mqtt/README.md + items: + - label: development resources + file: solutions/Xiaozhi_AI_mqtt/dev_resources.md + - label: quick start + file: solutions/Xiaozhi_AI_mqtt/quick_start.md + - label: quick start + file: solutions/Xiaozhi_AI_mqtt/software_design.md - label: EP-D200 Hua Series DTU file: solutions/DTU/README.md items: diff --git a/docs/Application_guide/en/solutions/README.md b/docs/Application_guide/en/solutions/README.md index 3e0244e6d1d5f495f71ec124ff0d85bea0a8a612..66583321890f16a84aa3cb9961027a315f7a3210 100644 --- a/docs/Application_guide/en/solutions/README.md +++ b/docs/Application_guide/en/solutions/README.md @@ -4,6 +4,7 @@ To facilitate your application development and accelerate your project implement - [Doubao RTC AI chatbot](./AIChatBot-Volcengine-webRTC/README.md) - [XiaoZhi AI chatbot](./xiaozhi_AI/README.md) +- [XiaoZhi_AI_chatbot_mqtt](./xiaozhi_AI_mqtt/README.md) - [EP-D200 Hua Series DTU](./DTU/README.md) - [Smart Agriculture Central Control Panel](./Agriculture-Control-Panel/README.md) - [Wearable Solution](./Wear/readme.md) diff --git a/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/README.md b/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/README.md new file mode 100644 index 0000000000000000000000000000000000000000..91a8a5fc6aa7afa81db7b2ab9830f2bdfb94833f --- /dev/null +++ b/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/README.md @@ -0,0 +1,16 @@ +- # Xiaozhi AI Chatbot Based on MQTT + UDP + + The Xiaozhi AI chatbot solution has the following features: + + - Supports timbre switching. + - Supports voice interruption/interruption. + - Supports voice wake-up. + - Uses Python language, facilitating secondary development. + + + +------ + +- [Summary of Development Resources](./dev_resources.md) +- [Quick Start](./quick_start.md) +- [Software Design Explanation](./software_design.md) \ No newline at end of file diff --git a/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/dev_resources.md b/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/dev_resources.md new file mode 100644 index 0000000000000000000000000000000000000000..4bc200aae7687f67bfab7a44385ada5dff8af530 --- /dev/null +++ b/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/dev_resources.md @@ -0,0 +1,36 @@ +- # Summary of Development Resources + + ## Hardware Information + + - EC800MCNLE development board (including antenna, Type-C data cable, etc.) + + > - [Click here to purchase the development board kit](https://www.quecmall.com/goods-detail/2c90800b9488359c0195efe6367303b5) + + - Computer (Windows 7, Windows 10 or Windows 11) + + - Any speaker with a power of 2-5W ([Purchase link on Quectel Mall](https://www.quecmall.com/goods-detail/2c90800c9488358b01956aa656680239)) + + - A 3.7V polymer lithium battery + + ## Software Information + + - USB driver for QuecPython module: [QuecPython_USB_Driver_Win10_ASR](https://developer.quectel.com/wp-content/uploads/2024/09/Quectel_Windows_USB_DriverA_Customer_V1.1.13.zip) + - [QuecPython firmware](../../media/solutions/xiaozhi_AI_mqtt/EC800MCNLER06A01M08_OCPU_QPY_TEST0810.zip) + + ## Development Tools + + - QPYcom - QuecPython debugging tool + - Version: V3.9.0 + - Download [QPYcom](https://developer.quectel.com/wp-content/uploads/2024/09/QPYcom_V3.9.0.zip) + - VSCode - Code editor + - Download [VSCode](https://code.visualstudio.com/) + + ## Experimental Source Code + + - Steps to obtain the source code from the Github repository: + + ```bash + git clone https://github.com/QuecPython/AIChatbot-Xiaozhi-Mqtt.git + ``` + + - [Download compressed package](https://codeload.github.com/QuecPython/AIChatbot-Xiaozhi-Mqtt/zip/refs/heads/main) \ No newline at end of file diff --git a/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/quick_start.md b/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/quick_start.md new file mode 100644 index 0000000000000000000000000000000000000000..8c7a2ed15c6abf505337584c1f428fabf94dd410 --- /dev/null +++ b/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/quick_start.md @@ -0,0 +1,68 @@ +# Quick Start + +## Hardware Preparation + +- A Windows computer, preferably with the `Win10` system. +- One set of [EC800MCNLE development board](https://www.quecmall.com/goods-detail/2c90800c9488358b0195efe4c2020409) (including antenna, Type-C data cable, etc.). +- A properly functioning SIM card. +- A speaker with a power of 2-5W. + +## Environment Setup + +- Download and install the EC800M series module driver: [QuecPython_USB_Driver_Win10_ASR](https://developer.quectel.com/wp-content/uploads/2024/09/Quectel_Windows_USB_DriverA_Customer_V1.1.13.zip). +- Download and install [VSCode](https://code.visualstudio.com/). +- Download and unzip the [QPYCom](https://images.quectel.com/python/2022/12/QPYcom_V3.6.0.zip) tool to a suitable location on your computer. +- [Download the firmware package](../../media/solutions/xiaozhi_AI_mqtt/EC800MCNLER06A01M08_OCPU_QPY_TEST0810.zip). +- Download the [experimental source code](https://codeload.github.com/QuecPython/AIChatbot-Xiaozhi-Mqtt/zip/refs/heads/main). + +## Hardware Connection + +Make hardware connections according to the following diagram: + + + +1. Insert a usable `SIM` card in the position shown in the diagram. +2. Connect the battery in the position shown in the diagram. +3. Use a Type-C data cable to connect the development board to the computer. +4. Connect the antenna to the antenna connector marked `LTE`. +5. Connect the battery in the position shown in the diagram. Connect the speaker to the pin headers marked `SPK+` and `SPK-` in the diagram. +6. Connect the LCD screen. + +## Device Development + +### Power On + +After completing the hardware connections, if a COM port containing the words `Quectel USB` appears in the port list of the computer's Device Manager, it indicates a successful power-on. + +​ + +### Firmware Package Burning + +Refer to [this chapter](https://developer.quectel.com/doc/quecpython/Application_guide/en/dev-tools/QPYcom/qpycom-dw.html#Download-Script) to burn the firmware package corresponding to the model into the development board. + +### Script Import and Execution + +1. Refer to [this chapter](https://developer.quectel.com/doc/quecpython/Getting_started/en/4G/first_python.html) to import all files in the `code` folder under the source code directory into the module file system, as shown in the following figure: +2. Refer to [this chapter](https://developer.quectel.com/doc/quecpython/Getting_started/en/4G/first_python.html#Run-Script) to execute the main program file `_main.py`. +3. Refer to [this chapter](https://developer.quectel.com/doc/quecpython/Getting_started/en/4G/first_python.html#Stop-a-Running-Program) to stop the program. + +## Business Debugging + +### Program Startup + +After executing the `_main.py` script, the program starts running. + +### Activate Xiaozhid + +The state shown in the following figure is the waiting-for-wake-up state. The development board will have a red light flashing, and you need to use the voice "Xiaozhi, Xiaozhi" to wake up Xiaozhi AI for voice conversation. + + + +After waking up, data related to connecting to the UDP server or a connection failure prompt will appear. If a connection failure occurs, please check if the network is normal. + + + +When there is no conversation for a long time, a goodbye message will be received, both mqtt and udp will be actively closed, and the program will automatically enter the waiting-for-wake-up state, waiting for the next wake-up using the wake-up word to re-establish the connection. + + + diff --git a/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/software_design.md b/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/software_design.md new file mode 100644 index 0000000000000000000000000000000000000000..c53f2f8abccc76a8727472b5c5fda42e82c8ecd4 --- /dev/null +++ b/docs/Application_guide/en/solutions/xiaozhi_AI_mqtt/software_design.md @@ -0,0 +1,227 @@ +# Software Design Explanation + +## Software Framework + +### Scheme Framework + + + +### Business Process + + + +## Code Explanation + +### Application Class Initialization + +Initialize AI objects and other hardware drivers. + +```python +class Application(object): + def __init__(self): + + Pin(Pin.GPIO33, Pin.OUT, Pin.PULL_PD, 1) + self.prev_emoj = None + + # Initialize charge management + self.charge_manager = ChargeManager() + + # Initialize audio management + self.audio_manager = AudioManager() + self.audio_manager.set_kws_cb(self.on_keyword_spotting) + self.audio_manager.set_vad_cb(self.on_voice_activity_detection) + + # Initialize network management + self.net_manager = NetManager() + + # Initialize task scheduler + self.task_manager = TaskManager() + + # Initialize protocol + self.__protocol = MqttClient() + self.__protocol.set_callback( + audio_message_handler=self.on_audio_message, + json_message_handler=self.on_json_message + ) + + self.__working_thread = None + self.__record_thread = None + self.__record_thread_stop_event = Event() + self.__voice_activity_event = Event() + self.__keyword_spotting_event = Event() +``` + +### Wake-up & Voice Detection + +When the wake-up word is detected, the working thread `self.__working_thread` will be pulled up, and the working thread will enable vad for voice activity detection + +```python +class Application(object): + def on_keyword_spotting(self, state): + logger.info("on_keyword_spotting: {}".format(state)) + if state == 0: + # Wake-up word triggered + if self.__working_thread is not None and self.__working_thread.is_running(): + return + self.__working_thread = Thread(target=self.__working_thread_handler) + self.__working_thread.start() + self.__keyword_spotting_event.clear() + else: + self.__keyword_spotting_event.set() + + def on_voice_activity_detection(self, state): + gc.collect() + logger.info("on_voice_activity_detection: {}".format(state)) + if state == 1: + self.__voice_activity_event.set() # Voice detected + else: + self.__voice_activity_event.clear() # No voice detected +``` + +### Dialogue Logic + +After the working thread is started, it will start a thread to execute `__chat_process`, and send the read audio data when voice is detected + +```python +class Application(object): + def __chat_process(self): + self.start_vad() + try: + with self.__protocol: + self.power_red_led.on() + self.__protocol.hello() + self.__protocol.wakeword_detected("Xiaozhi") + is_listen_flag = False + while True: + data = self.audio_manager.opus_read() + if self.__voice_activity_event.is_set(): + # Voice detected + if not is_listen_flag: + self.__protocol.listen("start") + is_listen_flag = True + self.__protocol.udp_send(data) + # logger.debug("send opus data to server") + else: + if is_listen_flag: + self.__protocol.listen("stop") + is_listen_flag = False + if not self.__protocol.is_state_ok(): + break + # logger.debug("read opus data length: {}".format(len(data))) + except Exception as e: + logger.debug("working thread handler got Exception: {}".format(repr(e))) + finally: + self.power_red_led.blink(250, 250) + self.stop_vad() +``` + +### Audio Management + +Unified management of the device's audio input and output, encoding and decoding, speech recognition-related functions (Keyword Spotting KWS and Voice Activity Detection VAD), and providing callback interfaces for upper-layer applications. + +```python +class AudioManager(object): + def __init__(self, channel=0, volume=11, pa_number=29): + self.aud = audio.Audio(channel) # Initialize audio playback channel + self.aud.set_pa(pa_number) + self.aud.setVolume(volume) # Set volume + self.aud.setCallback(self.audio_cb) + self.rec = audio.Record(channel) + self.__skip = 0 + # ========== Audio files ==================== + def audio_cb(self, event): + if event == 0: + # logger.info('audio play start.') + pass + elif event == 7: + # logger.info('audio play finish.') + pass + else: + pass + def play(self, file): + self.aud.play(0, 1, file) + # ========= opus ==================== + def open_opus(self): + self.pcm = audio.Audio.PCM(0, 1, 16000, 2, 1, 15) # 5 -> 25 + self.opus = Opus(self.pcm, 0, 6000) # 6000 ~ 128000 + def close_opus(self): + self.opus.close() + self.pcm.close() + del self.opus + del self.pcm + def opus_read(self): + return self.opus.read(60) + def opus_write(self, data): + return self.opus.write(data) + # ========= vad & kws ==================== + def set_kws_cb(self, cb): + self.rec.ovkws_set_callback(cb) + def set_vad_cb(self, cb): + def wrapper(state): + if self.__skip != 2: + self.__skip += 1 + return + return cb(state) + self.rec.vad_set_callback(wrapper) + def end_cb(self, para): + if(para[0] == "stream"): + if(para[2] == 1): + pass + elif (para[2] == 3): + pass + else: + pass + else: + pass + def start_kws(self): + self.rec.ovkws_start("_xiao_zhi_xiao_zhi", 0.7) + def stop_kws(self): + self.rec.ovkws_stop() + def start_vad(self): + self.__skip = 0 + self.rec.vad_start() + def stop_vad(self): + self.rec.vad_stop() +``` + +### MQTT and UDP Connections + +#### MQTT Connection + +First of all, it should be clear that to successfully connect to Xiaozhi platform's MQTT server, the corresponding MQTT connection parameters are required, and these parameters need to be obtained through Xiaozhi's OTA interface. In this scheme, we directly use Quecpython's request module to send a GET request to obtain the parameters; the specific OTA request method, request header format and request body format are all strictly implemented in accordance with the OTA documentation officially provided by Xiaozhi. + +The scheme specially designs a parameter acquisition method `ota_get()`, which is automatically called once during the initialization of the `MqttClient` class to ensure that the necessary connection parameters can be obtained at the initial stage of MQTT client instance creation, laying the foundation for the subsequent MQTT server connection process. + + + + + +After successfully obtaining the MQTT connection parameters, the scheme will temporarily store these parameters in the global list `aes_opus_info` to realize temporary storage of parameters and convenient subsequent calls. + +When the system triggers the wake-up process through the wake-up word, the program will automatically start the MQTT connection operation —— at this time, the MQTT connection parameters obtained through the OTA interface will be extracted from the global list `aes_opus_info`, and the connection configuration with Xiaozhi platform's MQTT server will be completed based on these parameters, ensuring that the MQTT connection process can be smoothly executed with valid and correct parameters. + + + +#### UDP Connection + +After the connection with Xiaozhi platform's MQTT server is successfully established, the system will further send a message through the MQTT channel to obtain the parameters required for UDP connection —— specifically, a "hello" type message specified by Xiaozhi platform needs to be sent, and the format of this message should be implemented in accordance with [the relevant specifications officially provided by Xiaozhi](https://github.com/78/xiaozhi-esp32/blob/main/docs/mqtt-udp.md). + +After successfully receiving the UDP connection parameters returned by Xiaozhi platform, the program will start the UDP connection process based on these parameters, complete the establishment of the UDP channel with Xiaozhi platform, and prepare for the subsequent transmission of audio stream data. + + + +### Audio Data Encryption + +After the UDP channel with Xiaozhi platform is successfully established, the subsequent audio stream data is transmitted using UDP. According to the clear technical specifications of Xiaozhi platform, the audio stream data sent through the UDP protocol must be encrypted to ensure the security and integrity of the data during transmission, and the specified encryption algorithm is the **AES-CTR algorithm**. + +In this scheme, we directly rely on Quecpython's encryption module `ucryptolib` to implement the AES-CTR encryption function: before sending the audio stream data, the program will call the corresponding encryption interface in the `ucryptolib` module, pass in preset parameters such as the key, complete the data encryption process; the encrypted audio stream data is then sent to Xiaozhi platform through the established UDP channel, ensuring that the entire audio transmission process conforms to the platform specifications and has reliable security guarantees. + + + +### Internal MCP + +When connecting, the mcp channel is opened in the hello message, and the server will send a request message (of type tools/list). At this time, we need to inform the local tools that can be called by Xiaozhi. During our voice interaction, when the server recognizes our intention, it will request to call our local tools. After the device receives the tool call request, it will respond and reply to the server. We can add tool information and processing logic in the response messages of tools/list and tools/call. + + + + diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/1.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/1.png new file mode 100644 index 0000000000000000000000000000000000000000..25f086a70fc884b0da0d434cb7026fcc59c612b5 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/1.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/2.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/2.png new file mode 100644 index 0000000000000000000000000000000000000000..b8b4b1d55d0ad2000d415418dd85480c3725983e Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/2.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/3.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/3.png new file mode 100644 index 0000000000000000000000000000000000000000..c611da614d463c06a6a74db83ec653af019bca0c Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/3.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/EC800MCNLER06A01M08_OCPU_QPY_TEST0810.zip b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/EC800MCNLER06A01M08_OCPU_QPY_TEST0810.zip new file mode 100644 index 0000000000000000000000000000000000000000..f81ad0721631b2f2a1694982a66461262732d17e Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/EC800MCNLER06A01M08_OCPU_QPY_TEST0810.zip differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/EVB.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/EVB.png new file mode 100644 index 0000000000000000000000000000000000000000..cb66506770269e6387842ec73d3dfd5e969d2e40 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/EVB.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/code_ai.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/code_ai.png new file mode 100644 index 0000000000000000000000000000000000000000..c8c70e96d117afb572d9980735f9a3f34bb804fe Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/code_ai.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/comport.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/comport.png new file mode 100644 index 0000000000000000000000000000000000000000..270017b82e7cae2e731cfa8deb67bd6530ed6485 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/comport.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/connect.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/connect.png new file mode 100644 index 0000000000000000000000000000000000000000..c7a8123ed1eed4fb7a0f981473016f0da2ba3d2b Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/connect.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/encrypt.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/encrypt.png new file mode 100644 index 0000000000000000000000000000000000000000..09284bc7e3eaf518aa412ff6ca46f1182d05ffb5 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/encrypt.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/flow.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/flow.png new file mode 100644 index 0000000000000000000000000000000000000000..23c06a23e989bde0efe32753f6d16d96e7218d0e Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/flow.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/mqtt.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/mqtt.png new file mode 100644 index 0000000000000000000000000000000000000000..4df590b908e8571d84c745231433a9d08b2b2946 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/mqtt.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/ota.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/ota.png new file mode 100644 index 0000000000000000000000000000000000000000..5121523606a110a43879d5651244039413913564 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/ota.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/ota_ret.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/ota_ret.png new file mode 100644 index 0000000000000000000000000000000000000000..842875ec1e058376a925e4cc35186fb1a3c22c52 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/ota_ret.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/software.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/software.png new file mode 100644 index 0000000000000000000000000000000000000000..d76e83e000ac3366d14168f8735d4298597b1d3a Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/software.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/tools_call.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/tools_call.png new file mode 100644 index 0000000000000000000000000000000000000000..b2493d0ffb4dad56de82ce2c46ab0c7faa8768b7 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/tools_call.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/tools_list.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/tools_list.png new file mode 100644 index 0000000000000000000000000000000000000000..e7801289e635ce3c68ff834ee3a5daca60736f5f Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/tools_list.png differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/udp.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/udp.png new file mode 100644 index 0000000000000000000000000000000000000000..e6465b97c0d725d5be2006fd174c3c4b113b3f01 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI_mqtt/udp.png differ diff --git a/docs/Application_guide/zh/sidebar.yaml b/docs/Application_guide/zh/sidebar.yaml index 661649c4ad2d754d50178fb25284ca57131a93d3..6adc85e512169ee1488dd310f33718ca89b7871d 100644 --- a/docs/Application_guide/zh/sidebar.yaml +++ b/docs/Application_guide/zh/sidebar.yaml @@ -280,7 +280,16 @@ items: - label: 快速上手 file: solutions/xiaozhi_AI/quick_start.md - label: 软件设计讲解 - file: solutions/xiaozhi_AI/software_design.md + file: solutions/xiaozhi_AI/software_design.md + - label: 基于MQTT+UDP的小智AI聊天机器人 + file: solutions/xiaozhi_AI_mqtt/README.md + items: + - label: 开发资源汇总 + file: solutions/xiaozhi_AI_mqtt/dev_resources.md + - label: 快速上手 + file: solutions/xiaozhi_AI_mqtt/quick_start.md + - label: 软件设计讲解 + file: solutions/xiaozhi_AI_mqtt/software_design.md - label: EP-D200 华系列 DTU file: solutions/DTU/README.md items: diff --git a/docs/Application_guide/zh/solutions/README.md b/docs/Application_guide/zh/solutions/README.md index 36eb52b1a5be930b50f8528eccce7c91c47c255f..6d5400fbca6806ae0081f7b50d703bb36a384948 100644 --- a/docs/Application_guide/zh/solutions/README.md +++ b/docs/Application_guide/zh/solutions/README.md @@ -4,6 +4,7 @@ - [豆包 RTC AI 聊天机器人](./AIChatBot-Volcengine-webRTC/README.md) - [小智 AI 聊天机器人](./xiaozhi_AI/README.md) +- [基于MQTT+UDP的小智AI聊天机器人](./xiaozhi_AI_mqtt/README.md) - [EP-D200 华系列 DTU](./DTU/README.md) - [智慧农业中控面板](./Agriculture-Control-Panel/README.md) - [可穿戴解决方案](./Wear/readme.md) diff --git a/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/README.md b/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/README.md new file mode 100644 index 0000000000000000000000000000000000000000..584d9e648b667a8ffbc477ecbc8c28759609726a --- /dev/null +++ b/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/README.md @@ -0,0 +1,22 @@ +# 基于MQTT＋UDP的小智 AI 聊天机器人 + +该小智 AI 聊天机器人方案具有以下特性： + +- 支持音色切换。 +- 支持语音中断/打断。 +- 支持语音唤醒。 +- 使用 Python 语言，便于二次开发。 + + + + + + + + + +--- + +- [开发资源汇总](./dev_resources.md) +- [快速上手](./quick_start.md) +- [软件设计讲解](./software_design.md) \ No newline at end of file diff --git a/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/dev_resources.md b/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/dev_resources.md new file mode 100644 index 0000000000000000000000000000000000000000..c8e530bf5015a80d21ef54d82df230af09fe05c5 --- /dev/null +++ b/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/dev_resources.md @@ -0,0 +1,34 @@ +# 开发资源汇总 + +## 硬件资料 + + - EC800MCNLE 开发板（含天线、Type-C 数据线等） + + > - [点此购买开发板套件](https://www.quecmall.com/goods-detail/2c90800b9488359c0195efe6367303b5) + + - 电脑（Windows 7、Windows 10 或 Windows 11） + + - 任意 2-5W 功率的喇叭即可 ([移远商城购买链接](https://www.quecmall.com/goods-detail/2c90800c9488358b01956aa656680239)) + + - 一块3.7V 聚合物锂电池 + +## 软件资料 + + - QuecPython 模块的 USB 驱动：[QuecPython_USB_Driver_Win10_ASR](https://developer.quectel.com/wp-content/uploads/2024/09/Quectel_Windows_USB_DriverA_Customer_V1.1.13.zip) + - [QuecPython 固件](../../media/solutions/xiaozhi_AI_mqtt/EC800MCNLER06A01M08_OCPU_QPY_TEST0810.zip) + +## 开发工具 + +- QPYcom - QuecPython 调试工具 + - 版本：V3.9.0 + - 下载 [QPYcom](https://developer.quectel.com/wp-content/uploads/2024/09/QPYcom_V3.9.0.zip) +- VSCode - 代码编辑器 + - 下载 [VSCode](https://code.visualstudio.com/) + +## 实验源码 + +- 从 Github 仓库获取源代码步骤如下： + ```bash + git clone https://github.com/QuecPython/AIChatbot-Xiaozhi-Mqtt.git + ``` +- [压缩包下载](https://codeload.github.com/QuecPython/AIChatbot-Xiaozhi-Mqtt/zip/refs/heads/main) \ No newline at end of file diff --git a/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/quick_start.md b/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/quick_start.md new file mode 100644 index 0000000000000000000000000000000000000000..fd00b960a735b8d61a368b5122b592abb63b7d03 --- /dev/null +++ b/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/quick_start.md @@ -0,0 +1,73 @@ +# 快速上手 + +## 硬件准备 + +- Windows 电脑一台，建议 `Win10` 系统。 +- 一套 [EC800MCNLE 开发板](https://www.quecmall.com/goods-detail/2c90800c9488358b0195efe4c2020409) （含天线、Type-C 数据线等）。 + +- 一张可正常使用的 SIM 卡。 +- 一个 2-5W 功率的喇叭。 + +## 环境搭建 + +- 下载并安装 EC800M 系列模组驱动：[QuecPython_USB_Driver_Win10_ASR](https://developer.quectel.com/wp-content/uploads/2024/09/Quectel_Windows_USB_DriverA_Customer_V1.1.13.zip)。 +- 下载并安装 [VSCode](https://code.visualstudio.com/)。 +- 下载并解压 [QPYCom](https://images.quectel.com/python/2022/12/QPYcom_V3.6.0.zip) 工具到电脑的合适位置。 +- [下载固件包](../../media/solutions/xiaozhi_AI_mqtt/EC800MCNLER06A01M08_OCPU_QPY_TEST0810.zip)。 +- 下载[实验源码](https://codeload.github.com/QuecPython/AIChatbot-Xiaozhi-Mqtt/zip/refs/heads/main)。 + + +## 硬件连接 + +按照下图进行硬件连接： + + + + +1. 在图示位置插入可用的 ` SIM` 卡。 +2. 在图示的位置接入电池。 +3. 使用 Type-C 数据线连接开发板和电脑。 +4. 将天线连接至标识有`LTE`字样的天线连接座上。 +5. 在图示的位置接入电池。将喇叭连接至图中标识有`SPK+`和`SPK-`的排针上。 +6. 连接LCD屏(可选)。 + +## 设备开发 + +### 开机 + +完成硬件连接的工作后，电脑设备管理器的端口列表中出现包含 `Quectel USB` 字样的 COM 口，表示开机成功。 + +![comport.png](../../media/solutions/xiaozhi_AI_mqtt/comport.png) + +### 烧录固件包 + +参考[此章节](https://developer.quectel.com/doc/quecpython/Application_guide/zh/dev-tools/QPYcom/qpycom-dw.html#%E4%B8%8B%E8%BD%BD%E5%9B%BA%E4%BB%B6)，烧录对应型号固件包至开发板。 + +### 脚本导入与运行 + +1. 参考[此章节](https://developer.quectel.com/doc/quecpython/Getting_started/zh/4G/first_python.html)，将源码目录下 `code` 文件夹中的所有文件导入到模组文件系统，如下图所示： + +2. 参考[此章节](https://developer.quectel.com/doc/quecpython/Getting_started/zh/4G/first_python.html#执行脚本文件)，执行主程序文件 `_main.py`。 + +3. 参考[此章节](https://developer.quectel.com/doc/quecpython/Getting_started/zh/4G/first_python.html#停止程序运行)，停止程序运行。 + +## 业务调试 + +### 程序启动 + +执行 `_main.py` 脚本后，程序开始运行。 + +### 激活小智 + +下图所示状态为待唤醒状态，开发板会出现红灯闪烁，需要语音“小智，小智”来唤醒小智AI进行语音对话。 + + + +唤醒后会出现连接UDP服务器相关的数据或者是连接失败的提示，当出现连接失败时，请检查网络是否正常。 + + + +当系统长时间未进行对话时，会收到goodbye消息，mqtt和udp都会主动关闭，程序自动进入待唤醒状态，等待下次使用唤醒词唤醒，再重新建立连接。 + + + diff --git a/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/software_design.md b/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/software_design.md new file mode 100644 index 0000000000000000000000000000000000000000..151b06f4b0c4d6761ee8db8a4c6af15d4b7d71d5 --- /dev/null +++ b/docs/Application_guide/zh/solutions/xiaozhi_AI_mqtt/software_design.md @@ -0,0 +1,228 @@ +# 软件设计讲解 + +## 软件框架 + +### 方案框架 + + + +### 业务流程 + + + + +## 代码讲解 + +### Application类初始化 + +初始化 AI 对象以及其他硬件驱动。 + +```python +class Application(object): + def __init__(self): + + Pin(Pin.GPIO33, Pin.OUT, Pin.PULL_PD, 1) + self.prev_emoj = None + + # 初始化充电管理 + self.charge_manager = ChargeManager() + + # 初始化音频管理 + self.audio_manager = AudioManager() + self.audio_manager.set_kws_cb(self.on_keyword_spotting) + self.audio_manager.set_vad_cb(self.on_voice_activity_detection) + + # 初始化网络管理 + self.net_manager = NetManager() + + # 初始化任务调度器 + self.task_manager = TaskManager() + + # 初始化协议 + self.__protocol = MqttClient() + self.__protocol.set_callback( + audio_message_handler=self.on_audio_message, + json_message_handler=self.on_json_message + ) + + self.__working_thread = None + self.__record_thread = None + self.__record_thread_stop_event = Event() + self.__voice_activity_event = Event() + self.__keyword_spotting_event = Event() +``` + +### 唤醒&人声检测 + +检测到唤醒词时，会拉起工作线程`self.__working_thread`，工作线程会开启vad进行人声检测 + +```python +class Application(object): + def on_keyword_spotting(self, state): + logger.info("on_keyword_spotting: {}".format(state)) + if state == 0: + # 唤醒词触发 + if self.__working_thread is not None and self.__working_thread.is_running(): + return + self.__working_thread = Thread(target=self.__working_thread_handler) + self.__working_thread.start() + self.__keyword_spotting_event.clear() + else: + self.__keyword_spotting_event.set() + + def on_voice_activity_detection(self, state): + gc.collect() + logger.info("on_voice_activity_detection: {}".format(state)) + if state == 1: + self.__voice_activity_event.set() # 有人声 + else: + self.__voice_activity_event.clear() # 无人声 +``` + +### 对话逻辑 + +工作线程开启后会启动线程执行`__chat_process`，检测到人声后就会发送读取的音频数据 + +```python +class Application(object): + def __chat_process(self): + self.start_vad() + try: + with self.__protocol: + self.power_red_led.on() + self.__protocol.hello() + self.__protocol.wakeword_detected("小智") + is_listen_flag = False + while True: + data = self.audio_manager.opus_read() + if self.__voice_activity_event.is_set(): + # 有人声 + if not is_listen_flag: + self.__protocol.listen("start") + is_listen_flag = True + self.__protocol.udp_send(data) + # logger.debug("send opus data to server") + else: + if is_listen_flag: + self.__protocol.listen("stop") + is_listen_flag = False + if not self.__protocol.is_state_ok(): + break + # logger.debug("read opus data length: {}".format(len(data))) + except Exception as e: + logger.debug("working thread handler got Exception: {}".format(repr(e))) + finally: + self.power_red_led.blink(250, 250) + self.stop_vad() +``` + +### 音频管理 + +统一管理设备的音频输入输出、编解码、语音识别相关功能（关键词识别 KWS 和语音活动检测 VAD），并提供回调接口供上层应用使用。 + +```python +class AudioManager(object): + def __init__(self, channel=0, volume=11, pa_number=29): + self.aud = audio.Audio(channel) # 初始化音频播放通道 + self.aud.set_pa(pa_number) + self.aud.setVolume(volume) # 设置音量 + self.aud.setCallback(self.audio_cb) + self.rec = audio.Record(channel) + self.__skip = 0 + # ========== 音频文件 ==================== + def audio_cb(self, event): + if event == 0: + # logger.info('audio play start.') + pass + elif event == 7: + # logger.info('audio play finish.') + pass + else: + pass + def play(self, file): + self.aud.play(0, 1, file) + # ========= opus ==================== + def open_opus(self): + self.pcm = audio.Audio.PCM(0, 1, 16000, 2, 1, 15) # 5 -> 25 + self.opus = Opus(self.pcm, 0, 6000) # 6000 ~ 128000 + def close_opus(self): + self.opus.close() + self.pcm.close() + del self.opus + del self.pcm + def opus_read(self): + return self.opus.read(60) + def opus_write(self, data): + return self.opus.write(data) + # ========= vad & kws ==================== + def set_kws_cb(self, cb): + self.rec.ovkws_set_callback(cb) + def set_vad_cb(self, cb): + def wrapper(state): + if self.__skip != 2: + self.__skip += 1 + return + return cb(state) + self.rec.vad_set_callback(wrapper) + def end_cb(self, para): + if(para[0] == "stream"): + if(para[2] == 1): + pass + elif (para[2] == 3): + pass + else: + pass + else: + pass + def start_kws(self): + self.rec.ovkws_start("_xiao_zhi_xiao_zhi", 0.7) + def stop_kws(self): + self.rec.ovkws_stop() + def start_vad(self): + self.__skip = 0 + self.rec.vad_start() + def stop_vad(self): + self.rec.vad_stop() +``` + +### MQTT和UDP连接 + +#### MQTT连接 + +首先需明确，要成功连接小智平台的 MQTT 服务器，必须具备对应的 MQTT 连接参数，而该参数需通过小智的 OTA 接口获取。在本方案中，我们直接采用 Quecpython 的 request 模块发送 GET 请求来完成参数获取；其中，OTA 请求的具体方式、请求头格式及请求体格式，均严格参照小智官方提供的 OTA 文档执行。 + +方案中专门设计了参数获取方法`ota_get()`，且该方法会在`MqttClient`类初始化时自动调用一次，确保在 MQTT 客户端实例创建的初始阶段，就能完成必要连接参数的获取，为后续 MQTT 服务器的连接流程奠定基础。 + + + + + +成功获取 MQTT 连接参数后，方案会将该参数暂存至全局列表`aes_opus_info`中，以实现参数的临时存储与后续便捷调用。 + +当系统通过唤醒词触发唤醒流程时，程序将自动启动 MQTT 连接操作 —— 此时会从全局列表`aes_opus_info`中提取此前通过 OTA 接口获取的 MQTT 连接参数，并基于该参数完成与小智平台 MQTT 服务器的连接配置，确保 MQTT 连接流程能基于有效、正确的参数顺利执行。 + + + +#### UDP连接 + +在与小智平台 MQTT 服务器的连接成功建立后，系统将进一步通过 MQTT 通道发送消息，以获取 UDP 连接所需的参数 —— 具体需发送小智平台规定的 “hello” 类型消息，该消息的格式需[参照小智官方相关规范执行](https://github.com/78/xiaozhi-esp32/blob/main/docs/mqtt-udp.md)。 + +当成功接收到小智平台返回的 UDP 连接参数后，程序会基于该参数启动 UDP 连接流程，完成与小智平台的 UDP 通道建立，为后续音频流数据的传输做好准备。 + + + +### 音频数据加密 + +在与小智平台的 UDP 通道成功建立后，后续音频流数据使用UDP传输。根据小智平台的明确技术规范，通过 UDP 协议发送的音频流数据必须经过加密处理，以保障数据传输过程中的安全性与完整性，而指定采用的加密算法为**AES-CTR 算法**。 + +在本方案中，我们直接依托我们 Quecpython 的加密模块`ucryptolib`来实现 AES-CTR 加密功能：在音频流数据发送前，程序会调用`ucryptolib`模块中对应的加密接口，传入预设的密钥等参数，完成数据加密处理；加密后的音频流数据再通过已建立的 UDP 通道发送至小智平台，确保整个音频传输流程既符合平台规范，又具备可靠的安全保障。 + + + +### 小智内部MCP + +连接时在hello消息中打开mcp通道，服务器会发来请求消息(tools/list类型)，此时我们需要告知我们可以给小智调用的本地工具，在我们语音交互时，服务器识别到我们的意图时会请求调用我们本地工具，设备接收到工具调用请求后会做出响应并回复服务器。我们可以在tools/list和tools/call的响应消息里添加工具信息和处理逻辑 + + + +