diff --git a/docs/Application_guide/en/README.md b/docs/Application_guide/en/README.md index f60f0156de5d7325f1eeae67d55c5d5e0faba1a1..8c6fc991ab0228cc5caff616c9dd17f5b48b8024 100644 --- a/docs/Application_guide/en/README.md +++ b/docs/Application_guide/en/README.md @@ -95,7 +95,8 @@ QuecPython Application Guide is a guide on how to use the commonly used function Product Solution - [DTU Solution](solutions/DTU/README.md) -- [XiaoZhi AI chatbot](solutions/xiaozhi_AI/README.md) +- [Xiaozhi AI chatbot based on websocket](solutions/xiaozhi_AI/README.md) +- [Xiaozhi AI chatbot based on MQTT+UDP](solutions/xiaozhi_AI_mqtt/README.md) - [Smart Agriculture Central Control Panel](solutions/Agriculture-Control-Panel/README.md) - [Wearable Solution](solutions/Wear/readme.md) - [Electricity Meter](solutions/electricity-meter/README.md) diff --git a/docs/Application_guide/en/media/solutions/AIbox/AIbox.jpg b/docs/Application_guide/en/media/solutions/AIbox/AIbox.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a7bed56946180a772fcb2c0f77be275fec13942a Binary files /dev/null and b/docs/Application_guide/en/media/solutions/AIbox/AIbox.jpg differ diff --git a/docs/Application_guide/en/media/solutions/AIbox/AIbox2.png b/docs/Application_guide/en/media/solutions/AIbox/AIbox2.png new file mode 100644 index 0000000000000000000000000000000000000000..58fa463ad70469d78886a3207fb464decb9d6a6d Binary files /dev/null and b/docs/Application_guide/en/media/solutions/AIbox/AIbox2.png differ diff --git a/docs/Application_guide/en/media/solutions/AIbox/EC800MCNLE8.10.zip b/docs/Application_guide/en/media/solutions/AIbox/EC800MCNLE8.10.zip new file mode 100644 index 0000000000000000000000000000000000000000..fad70b54432fd9ffd63372cffc401f652628d7be Binary files /dev/null and b/docs/Application_guide/en/media/solutions/AIbox/EC800MCNLE8.10.zip differ diff --git a/docs/Application_guide/en/media/solutions/AIbox/flow.png b/docs/Application_guide/en/media/solutions/AIbox/flow.png new file mode 100644 index 0000000000000000000000000000000000000000..cf2d87145b18d845637430ad936911779c9f25c9 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/AIbox/flow.png differ diff --git a/docs/Application_guide/en/media/solutions/AIbox/flow2.png b/docs/Application_guide/en/media/solutions/AIbox/flow2.png new file mode 100644 index 0000000000000000000000000000000000000000..947479568071d963ba7f2b0115741a022ec4fd85 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/AIbox/flow2.png differ diff --git a/docs/Application_guide/en/media/solutions/AIbox/run.png b/docs/Application_guide/en/media/solutions/AIbox/run.png new file mode 100644 index 0000000000000000000000000000000000000000..57382b2b9758ee4b762aa0f3a9fc5a729286c105 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/AIbox/run.png differ diff --git a/docs/Application_guide/en/media/solutions/AIbox/software.png b/docs/Application_guide/en/media/solutions/AIbox/software.png new file mode 100644 index 0000000000000000000000000000000000000000..448d1f44e1b4e5d8abc64849be7a5856571ab729 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/AIbox/software.png differ diff --git a/docs/Application_guide/en/media/solutions/AIbox/text.mp4 b/docs/Application_guide/en/media/solutions/AIbox/text.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2fb173ca684f5aecbfea0a24c3a7868d70d439f4 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/AIbox/text.mp4 differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI/EC800MCNLER01A03M08_OCPU_QPY_TEST0707A.zip b/docs/Application_guide/en/media/solutions/xiaozhi_AI/EC800MCNLER01A03M08_OCPU_QPY_TEST0707A.zip deleted file mode 100644 index cbf423fd0eef9100f2038e4abb0cad2f2a96462d..0000000000000000000000000000000000000000 Binary files a/docs/Application_guide/en/media/solutions/xiaozhi_AI/EC800MCNLER01A03M08_OCPU_QPY_TEST0707A.zip and /dev/null differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI/EC800MCNLER06A01M08_AI_WS_OCPU_QPY_BETA0820(1).zip b/docs/Application_guide/en/media/solutions/xiaozhi_AI/EC800MCNLER06A01M08_AI_WS_OCPU_QPY_BETA0820(1).zip new file mode 100644 index 0000000000000000000000000000000000000000..1eedba40c04d3ebf26bb3fa729048e54b1ee6e74 Binary files /dev/null and b/docs/Application_guide/en/media/solutions/xiaozhi_AI/EC800MCNLER06A01M08_AI_WS_OCPU_QPY_BETA0820(1).zip differ diff --git a/docs/Application_guide/en/media/solutions/xiaozhi_AI/software.png b/docs/Application_guide/en/media/solutions/xiaozhi_AI/software.png index d2bef1b682f2f63d569fba41271f4aaa94bb8778..d417744af689b44e7d0f3015c232a224d5477720 100644 Binary files a/docs/Application_guide/en/media/solutions/xiaozhi_AI/software.png and b/docs/Application_guide/en/media/solutions/xiaozhi_AI/software.png differ diff --git a/docs/Application_guide/en/sidebar.yaml b/docs/Application_guide/en/sidebar.yaml index 30d48252ce253b946b46033174630bde123f9428..4481ae505b5727fd04d87e28b17298d48f2f3ab3 100644 --- a/docs/Application_guide/en/sidebar.yaml +++ b/docs/Application_guide/en/sidebar.yaml @@ -269,7 +269,7 @@ items: file: solutions/AIChatBot-Volcengine-webRTC/quick_start.md - label: software design file: solutions/AIChatBot-Volcengine-webRTC/software_design.md - - label: XiaoZhi AI Chatbot + - label: Xiaozhi AI chatbot based on websocket file: solutions/Xiaozhi_AI/README.md items: - label: development resources @@ -286,7 +286,16 @@ items: - label: quick start file: solutions/Xiaozhi_AI_mqtt/quick_start.md - label: quick start - file: solutions/Xiaozhi_AI_mqtt/software_design.md + file: solutions/Xiaozhi_AI_mqtt/software_design.md + - label: AIbox + file: solutions/AIbox/README.md + items: + - label: development resources + file: solutions/AIbox/dev_resources.md + - label: quick start + file: solutions/AIbox/quick_start.md + - label: software design + file: solutions/AIbox/software_design.md - label: EP-D200 Hua Series DTU file: solutions/DTU/README.md items: diff --git a/docs/Application_guide/en/solutions/AIbox/README.md b/docs/Application_guide/en/solutions/AIbox/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6b0b4baee32af64c6c605060955487654edc94e6 --- /dev/null +++ b/docs/Application_guide/en/solutions/AIbox/README.md @@ -0,0 +1,22 @@ +# AIbox Chatbot + +The AIbox chatbot solution has the following features: + +- Supports tone color switching. +- Supports voice interruption/interruption. + +- Support emotional visualization +- Supports voice wake-up. +- Using the Python language is convenient for secondary development. + + + + + + + +--- + +- [Development resources](./dev_resources.md) +- [Quick start](./quick_start.md) +- [Software design](./software_design.md) diff --git a/docs/Application_guide/en/solutions/AIbox/dev_resources.md b/docs/Application_guide/en/solutions/AIbox/dev_resources.md new file mode 100644 index 0000000000000000000000000000000000000000..90417f029213b27cfe03139f04d5ce4424c695e8 --- /dev/null +++ b/docs/Application_guide/en/solutions/AIbox/dev_resources.md @@ -0,0 +1,28 @@ +# Development resources summary + +## Hardware information + + - AIbox robot kit (Including the development board, antenna, Type-C data cable, casing, etc.) + + > - [Click here to purchase the development board kit](https://www.quecmall.com/goods-detail/2c90800b8cb50925018e5a696d0104f7) + +## Software documentation + + - The USB driver of the QuecPython module：[QuecPython_USB_Driver_Win10_ASR](https://developer.quectel.com/wp-content/uploads/2024/09/Quectel_Windows_USB_DriverA_Customer_V1.1.13.zip) + - [QuecPython Firmware](../../media/solutions/AIbox/EC800MCNLE8.10.zip) + +## Development tools + +- QPYcom - QuecPython debug tool + - version: V3.6.0 + - Download: [QPYcom](https://gitee.com/link?target=https%3A%2F%2Fpython.quectel.com%2Fwp-content%2Fuploads%2F2024%2F09%2FQPYcom_V3.6.0.zip) +- VSCode + - Download: [VSCode](https://gitee.com/link?target=https%3A%2F%2Fcode.visualstudio.com%2F) + +## Source Code + +- The steps to obtain the source code from the Github repository are as follows： + ```bash + git clone https://github.com/QuecPython/AIBox.git + ``` +- [Github Download](https://codeload.github.com/QuecPython/AIBox/zip/refs/heads/main) \ No newline at end of file diff --git a/docs/Application_guide/en/solutions/AIbox/quick_start.md b/docs/Application_guide/en/solutions/AIbox/quick_start.md new file mode 100644 index 0000000000000000000000000000000000000000..954f2ed8b06bfc9350058539d37bb51dec84da03 --- /dev/null +++ b/docs/Application_guide/en/solutions/AIbox/quick_start.md @@ -0,0 +1,68 @@ +# Quick Start + +## Hardware preparation + +- A Windows computer, recommended for the Win10 system. +- a set EC800MCNLE EVB(Including antenna, Type-C data cable, etc). + +- A Nano SIM card. +- A speaker with a power of 2-5W. + +## Environment construction + +- Download and install EC800M series module driver: [QuecPython_USB_Driver_Win10_ASR](https://gitee.com/link?target=https%3A%2F%2Fimages.quectel.com%2Fpython%2F2023%2F04%2FQuectel_Windows_USB_DriverA_Customer_V1.1.13.zip). +- Download and install [VSCode](https://gitee.com/link?target=https%3A%2F%2Fcode.visualstudio.com%2F). +- Download and unzip [QPYCom](https://gitee.com/link?target=https%3A%2F%2Fimages.quectel.com%2Fpython%2F2022%2F12%2FQPYcom_V3.6.0.zip)。 +- [Download Firmware](../../media/solutions/AIbox/EC800MCNLE8.10.zip) +- Download [Source code](https://codeload.github.com/QuecPython/AIBox/zip/refs/heads/main)。 + +## Hardware connection + +Connect the hardware according to the following diagram: + + + + +1. Connect the development board to the computer using a Type-C data cable. + +## Equipment development + +### Power on + +After completing the hardware connection work, if a COM port containing the words "Quectel USB" appears in the port list of the computer device manager, it indicates that the computer has successfully booted up. + +![comport.png](../../media/solutions/AIChatBot-Volcengine-webRTC/comport.png) + +### Burn firmware package + +Refer to [this chapter](https://gitee.com/link?target=https%3A%2F%2Fdeveloper.quectel.com%2Fdoc%2Fquecpython%2FApplication_guide%2Fen%2Fdev-tools%2FQPYcom%2Fqpycom-dw.html%23%E4%B8%8B%E8%BD%BD%E5%9B%BA%E4%BB%B6) Burn the corresponding firmware package to the development board. + +### Script import and execution + +1. Refer to [this chapter](https://gitee.com/link?target=https%3A%2F%2Fdeveloper.quectel.com%2Fdoc%2Fquecpython%2FGetting_started%2Fen%2F4G%2Ffirst_python.html%23Transfer-Files-Between-PC-and-Module) Import all files from the 'code' folder in the source code directory into the module file system, as shown in the following figure: + + + +1. Refer to [this chapter](https://gitee.com/link?target=https%3A%2F%2Fdeveloper.quectel.com%2Fdoc%2Fquecpython%2FGetting_started%2Fen%2F4G%2Ffirst_python.html%23Run-Script) Execute the main program file `ai_main.py`. +2. Refer to [this chapter](https://gitee.com/link?target=https%3A%2F%2Fdeveloper.quectel.com%2Fdoc%2Fquecpython%2FGetting_started%2Fen%2F4G%2Ffirst_python.html%23Stop-a-Running-Program) Stop the program from running. + +## Debugging + +### Program startup + +After executing the `_main. py` script, the program starts running. + +### Activate Xiaozhi AI + +As shown in the following picture, it is in a state ready to be awakened. A red light will flash on the development board, and you need to use the voice `Xiaozhi, Xiaozhi` to wake up Xiaozhi AI for a voice conversation. + + + +After waking up, data related to connecting to the server or a prompt indicating connection failure will appear. When a connection failure occurs, please check if the network is normal. + + + +When the system has not engaged in a conversation for a long time or has not received any commands, it will automatically enter a sleep state and wait for the next wake-up. + + + diff --git a/docs/Application_guide/en/solutions/AIbox/software_design.md b/docs/Application_guide/en/solutions/AIbox/software_design.md new file mode 100644 index 0000000000000000000000000000000000000000..399d8be88620ebf9b723f80868d2f831834c9f59 --- /dev/null +++ b/docs/Application_guide/en/solutions/AIbox/software_design.md @@ -0,0 +1,262 @@ +# Software Design + +## Software framework + +### Framework design diagram + + + +### Business system startup process + + + +## Code Explanation + +### AI wake-up & Human voice detection + +```python +class Application(object): + def on_keyword_spotting(self, state): + logger.info("on_keyword_spotting: {}".format(state)) + if state == 0: + # Trigger the wake-up word + if self.__working_thread is not None and self.__working_thread.is_running(): + return + self.__working_thread = Thread(target=self.__working_thread_handler) + self.__working_thread.start() + self.__keyword_spotting_event.clear() + else: + self.__keyword_spotting_event.set() + + def on_voice_activity_detection(self, state): + gc.collect() + logger.info("on_voice_activity_detection: {}".format(state)) + if state == 1: + self.__voice_activity_event.set() # It is claimed that + else: + self.__voice_activity_event.clear() # No One Can Hear You +``` + +### AI initialization + +Initialize AI objects and other hardware drivers. + +```python +class Application(object): + + def __init__(self): + + Pin(Pin.GPIO33, Pin.OUT, Pin.PULL_PD, 1) + # Initialize the screen + self.lvgl=lvglManager() + # 初始化充电管理 + self.charge_manager = ChargeManager() + + # Initialize audio management + self.audio_manager = AudioManager() + self.audio_manager.set_kws_cb(self.on_keyword_spotting) + self.audio_manager.set_vad_cb(self.on_voice_activity_detection) + + # Initialize network management + self.net_manager = NetManager() + + # Initialize the task scheduler + self.task_manager = TaskManager() + + # Initialize Protocol + self.__protocol = WebSocketClient() + self.__protocol.set_callback( + audio_message_handler=self.on_audio_message, + json_message_handler=self.on_json_message + ) + + self.__working_thread = None + self.__record_thread = None + self.__record_thread_stop_event = Event() + self.__voice_activity_event = Event() + self.__keyword_spotting_event = Event() +``` + +### AI dialogue interruption logic + +```python +class Application(object): + def __chat_process(self): + self.start_vad() + try: + with self.__protocol: + self.__protocol.hello() + self.__protocol.wakeword_detected("小智") + is_listen_flag = False + while True: + data = self.audio_manager.opus_read() + if self.__voice_activity_event.is_set(): + # It is claimed that + if not is_listen_flag: + self.__protocol.listen("start") + is_listen_flag = True + self.__protocol.send(data) + # logger.debug("send opus data to server") + else: + if is_listen_flag: + self.__protocol.listen("stop") + is_listen_flag = False + if not self.__protocol.is_state_ok(): + break + # logger.debug("read opus data length: {}".format(len(data))) + except Exception as e: + logger.debug("working thread handler got Exception: {}".format(repr(e))) + finally: + self.stop_vad() + +``` + +### music organizer + +Uniformly manage the audio input and output, encoding and decoding, and voice recognition related functions of the equipment (keyword recognition KWS and voice activity detection VAD), and provide callback interfaces for upper-layer applications to use. + +```python +class AudioManager(object): + + def __init__(self, channel=0, volume=11, pa_number=29): + self.aud = audio.Audio(channel) # 初始化音频播放通道 + self.aud.set_pa(pa_number) + self.aud.setVolume(volume) # 设置音量 + self.aud.setCallback(self.audio_cb) + self.rec = audio.Record(channel) + self.__skip = 0 + + # ========== 音频文件 ==================== + + def audio_cb(self, event): + if event == 0: + # logger.info('audio play start.') + pass + elif event == 7: + # logger.info('audio play finish.') + pass + else: + pass + + def play(self, file): + self.aud.play(0, 1, file) + + # ========= opus ==================== + + def open_opus(self): + self.pcm = audio.Audio.PCM(0, 1, 16000, 2, 1, 15) # 5 -> 25 + self.opus = Opus(self.pcm, 0, 6000) # 6000 ~ 128000 + + def close_opus(self): + self.opus.close() + self.pcm.close() + del self.opus + del self.pcm + + def opus_read(self): + return self.opus.read(60) + + def opus_write(self, data): + return self.opus.write(data) + + # ========= vad & kws ==================== + + def set_kws_cb(self, cb): + self.rec.ovkws_set_callback(cb) + + def set_vad_cb(self, cb): + def wrapper(state): + if self.__skip != 2: + self.__skip += 1 + return + return cb(state) + self.rec.vad_set_callback(wrapper) + + def end_cb(self, para): + if(para[0] == "stream"): + if(para[2] == 1): + pass + elif (para[2] == 3): + pass + else: + pass + else: + pass + + def start_kws(self): + self.rec.ovkws_start("_xiao_zhi_xiao_zhi", 0.7) + + def stop_kws(self): + self.rec.ovkws_stop() + + def start_vad(self): + self.__skip = 0 + self.rec.vad_start() + + def stop_vad(self): + self.rec.vad_stop() +``` + + + +## LCD屏显 + +The display of the LCD's expressions is controlled through the session bus sys_bus. + +```python +def update_emoji(topic,msg): + screen_gif.set_style_opa(lv.OPA.TRANSP, 0) + if msg == "happy": + screen_gif.set_src("U:/media/happy_min.gif") + #print("update_emoji: happy screen") + elif msg == "cool": + screen_gif.set_src("U:/media/cool_min.gif") + #print("update_emoji: cool screen") + elif msg == "thinking": + screen_gif.set_src("U:/media/think_min.gif") + #print("update_emoji: think screen") + elif msg == "angry": + screen_gif.set_src("U:/media/angry_min.gif") + #print("update_emoji: angry screen") + elif msg == "sleep": + screen_gif.set_src("U:/media/sleep_min.gif") + #print("update_emoji: happy screen") + utime.sleep_ms(20) + screen_gif.set_style_opa(lv.OPA.COVER, 0) + +def update_status(topic, msg): + screen_word.set_text(msg+"...") + +def update_screen(topic, msg): + try: + lcd.lcd_clear(0x0000) + if msg == "init_screen": + lv.scr_load(init_screen) + logger.info("change init_screen") + elif msg == "main_screen": + lv.scr_load(screen) + logger.info("change main_screen") + except Exception as e: + logger.error("update_screen error: {}".format(e)) + +def update_time(arg): + year, month, day, hour, minute, second = utime.localtime()[0:6] + toptime.set_text("{}:{}".format(hour,minute)) + #logger.info("time update") + +# subscribe topic to update emoji and status +sys_bus.subscribe("update_emoji",update_emoji) +sys_bus.subscribe("update_status",update_status) +sys_bus.subscribe("update_screen",update_screen) + +class lvglManager: + #@staticmethod + def __init__(self): + #lv.scr_load(init_screen) + update_time(None) + timer1 = Timer(Timer.Timer1) + timer1.start(period=10000, mode=timer1.PERIODIC, callback= update_time) + utime.sleep(3) + #lv.scr_load(screen) +``` + diff --git a/docs/Application_guide/en/solutions/README.md b/docs/Application_guide/en/solutions/README.md index 66583321890f16a84aa3cb9961027a315f7a3210..13186def85e552a4322e2f1060659f12db85cf27 100644 --- a/docs/Application_guide/en/solutions/README.md +++ b/docs/Application_guide/en/solutions/README.md @@ -3,8 +3,9 @@ To facilitate your application development and accelerate your project implementation, QuecPython team has launched software solutions for some common industry applications, including cloud speakers, DTUs, public-network intercoms, electricity meters, network cameras, CPE telephones, smart trackers, watches, electronic student ID cards, and charging piles. - [Doubao RTC AI chatbot](./AIChatBot-Volcengine-webRTC/README.md) -- [XiaoZhi AI chatbot](./xiaozhi_AI/README.md) +- [Xiaozhi AI chatbot based on websocket](./xiaozhi_AI/README.md) - [XiaoZhi_AI_chatbot_mqtt](./xiaozhi_AI_mqtt/README.md) +- [AIbox](./AIbox/README.md) - [EP-D200 Hua Series DTU](./DTU/README.md) - [Smart Agriculture Central Control Panel](./Agriculture-Control-Panel/README.md) - [Wearable Solution](./Wear/readme.md) diff --git a/docs/Application_guide/en/solutions/xiaozhi_AI/README.md b/docs/Application_guide/en/solutions/xiaozhi_AI/README.md index 023104512fb0d46e0095f310845e0d7b378b8b47..f8f69cc4e61c1e2399afd2887daf9f9ed74b19a1 100644 --- a/docs/Application_guide/en/solutions/xiaozhi_AI/README.md +++ b/docs/Application_guide/en/solutions/xiaozhi_AI/README.md @@ -1,4 +1,4 @@ -# XiaoZhi AI Chatbot +# Xiaozhi AI chatbot based on websocket This small intelligent AI chatbot solution has the following features: - Support for sound tone switching. diff --git a/docs/Application_guide/en/solutions/xiaozhi_AI/quick_start.md b/docs/Application_guide/en/solutions/xiaozhi_AI/quick_start.md index 6bfc57a3dc27c515c7bb770be6ae23f6b5ee273e..6667cc93c2240c7fc0e2c6b516128d64c6545789 100644 --- a/docs/Application_guide/en/solutions/xiaozhi_AI/quick_start.md +++ b/docs/Application_guide/en/solutions/xiaozhi_AI/quick_start.md @@ -8,12 +8,12 @@ - A Nano SIM card. - A speaker with a power of 2-5W. -## 环境搭建 +## Environment construction - Download and install EC800M series module driver: [QuecPython_USB_Driver_Win10_ASR](https://gitee.com/link?target=https%3A%2F%2Fimages.quectel.com%2Fpython%2F2023%2F04%2FQuectel_Windows_USB_DriverA_Customer_V1.1.13.zip). - Download and install [VSCode](https://gitee.com/link?target=https%3A%2F%2Fcode.visualstudio.com%2F). - Download and unzip [QPYCom](https://gitee.com/link?target=https%3A%2F%2Fimages.quectel.com%2Fpython%2F2022%2F12%2FQPYcom_V3.6.0.zip)。 -- [Download Firmware](../../media/solutions/xiaozhi_AI/EC800MCNLER01A03M08_OCPU_QPY_TEST0707A.zip) +- [Download Firmware](../../media/solutions/xiaozhi_AI/EC800MCNLER06A01M08_AI_WS_OCPU_QPY_BETA0820(1).zip) - Download [Source code](https://github.com/QuecPython/solution-xiaozhiAI/archive/refs/heads/master.zip)。 ## Hardware connection diff --git a/docs/Application_guide/en/solutions/xiaozhi_AI/software_design.md b/docs/Application_guide/en/solutions/xiaozhi_AI/software_design.md index 7424ac0679459abc3bd542ed9cab0447f4c42819..f159d3b1a024830591f45e299692263e1928cd00 100644 --- a/docs/Application_guide/en/solutions/xiaozhi_AI/software_design.md +++ b/docs/Application_guide/en/solutions/xiaozhi_AI/software_design.md @@ -87,6 +87,8 @@ class Application(object): ### AI dialogue interruption logic +After waking up the AI, start the thread for detecting and uploading the human voice audio. 'self.start_vad()' is used to start the recording function. When there is a human voice, the "Start listening" flag bit 'start' will be sent through websocket. Then execute 'self.__protocol.abort()' to end the current speech and thereby interrupt the AI speaking. + ```python class Application(object): def __chat_process(self): @@ -207,3 +209,137 @@ class AudioManager(object): self.rec.vad_stop() ``` +### MCP Management + +The message construction and sending function based on the MCP protocol covers scenarios such as initialization, tool list query, tool call response, and device notification. All messages follow the JSON-RPC 2.0 format and are sent uniformly through the 'send_mcp' method, ensuring the modularization and consistency of the code. + +```python +class WebSocketClient(object): + def send_mcp(self, payload, session_id=""): + """ + 发送标准MCP消息,payload为JSON-RPC 2.0格式字典 + """ + with self.__resp_helper: + self.send( + JsonMessage( + { + "session_id": session_id, + "type": "mcp", + "payload": payload + } + ).to_bytes() + ) + def mcp_initialize(self, capabilities=None, session_id="", req_id=1): + """ + 发送MCP initialize响应 + """ + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2025-9-03", + "capabilities": { + "tools":{}, + "notifications": {} + }, + "serverInfo": { + "name": 'xiaozhi-mqtt-client', + "version": "1.0.0" + } + } + } + self.send_mcp(payload, session_id) + def mcp_tools_list(self, cursor="", session_id="", req_id=2): + """ + 发送MCP tools/list响应请求 + """ + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "tools": [ + { + "name": "self.setvolume_down()", + "description": "只通过调用setvolume_down方法来控制音量变小,接收到回应后会播报当前音量大小", + "inputSchema": {} + }, + { + "name": "self.setvolume_up()", + "description": "只通过调用setvolume_up方法来控制音量变大,接收到回应后会播报当前音量大小", + "inputSchema": {} + }, + { + "name": "self.setvolume_close()", + "description": "只通过调用setvolume_close方法来静音,接收到回应后会播报当前音量大小", + "inputSchema": {} + }, + ], + } + } + + self.send_mcp(payload, session_id) + + def mcp_tools_call(self, session_id="", req_id="", error=None, tool_name=""): + """ + 发送MCP tools/call响应 + :param error: 如果为None则返回成功响应,否则返回错误响应(字典,包含code和message) + """ + if error is None: + if tool_name == "self.setvolume_down()": + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [ + { "type": "text", "text": "音量已调小 "} + ], + "isError": False + } + } + elif tool_name == "self.setvolume_up()": + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [ + { "type": "text", "text": "音量已调大" } + ], + "isError": False + } + } + elif tool_name == "self.setvolume_close()": + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [ + { "type": "text", "text": "已静音" } + ], + "isError": False + } + } + else: + payload = { + "jsonrpc": "2.0", + "id": req_id, + "error": { + "code": error.get("code", -32601), + "message": error.get("message", "Unknown error") + } + self.send_mcp(payload, session_id) + + def mcp_notify(self, method, params, session_id=""): + """ + 设备主动发送MCP通知 + """ + payload = { + "jsonrpc": "2.0", + "method": "notifications/state_changed", + "params": { + "newState": "idle", + "oldState": "connecting" + } + } + self.send_mcp(payload, session_id) +``` + diff --git a/docs/Application_guide/zh/README.md b/docs/Application_guide/zh/README.md index 378c7c815323aeab0b10e70811d83d71614b862d..eaf29976b0c61db2c154cd88d341b23b77a30d5c 100644 --- a/docs/Application_guide/zh/README.md +++ b/docs/Application_guide/zh/README.md @@ -84,7 +84,9 @@ QuecPython 开发指南，是对QuecPython常用功能模块如何使用的指 产品方案 - [豆包 RTC AI 聊天机器人](solutions/AIChatBot-Volcengine-webRTC/README.md) -- [小智 AI 聊天机器人](solutions/xiaozhi_AI/README.md) +- [基于websocket的小智 AI 聊天机器人](solutions/xiaozhi_AI/README.md) +- [基于MQTT+UDP的小智 AI 聊天机器人](solutions/xiaozhi_AI_mqtt/README.md) +- [AIbox聊天机器人](solutions/AIbox/README.md) - [DTU 方案](solutions/DTU/README.md) - [智慧农业中控面板](solutions/Agriculture-Control-Panel/README.md) - [可穿戴解决方案](solutions/Wear/readme.md) diff --git a/docs/Application_guide/zh/media/solutions/AIbox/AIbox.jpg b/docs/Application_guide/zh/media/solutions/AIbox/AIbox.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a7bed56946180a772fcb2c0f77be275fec13942a Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/AIbox/AIbox.jpg differ diff --git a/docs/Application_guide/zh/media/solutions/AIbox/AIbox1.jpg b/docs/Application_guide/zh/media/solutions/AIbox/AIbox1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c4a73d523fee737c1129c5cb008a1292fe26c757 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/AIbox/AIbox1.jpg differ diff --git a/docs/Application_guide/zh/media/solutions/AIbox/AIbox2.png b/docs/Application_guide/zh/media/solutions/AIbox/AIbox2.png new file mode 100644 index 0000000000000000000000000000000000000000..58fa463ad70469d78886a3207fb464decb9d6a6d Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/AIbox/AIbox2.png differ diff --git a/docs/Application_guide/zh/media/solutions/AIbox/EC800MCNLE8.10.zip b/docs/Application_guide/zh/media/solutions/AIbox/EC800MCNLE8.10.zip new file mode 100644 index 0000000000000000000000000000000000000000..fad70b54432fd9ffd63372cffc401f652628d7be Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/AIbox/EC800MCNLE8.10.zip differ diff --git a/docs/Application_guide/zh/media/solutions/AIbox/flow.png b/docs/Application_guide/zh/media/solutions/AIbox/flow.png new file mode 100644 index 0000000000000000000000000000000000000000..cf2d87145b18d845637430ad936911779c9f25c9 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/AIbox/flow.png differ diff --git a/docs/Application_guide/zh/media/solutions/AIbox/flow2.png b/docs/Application_guide/zh/media/solutions/AIbox/flow2.png new file mode 100644 index 0000000000000000000000000000000000000000..947479568071d963ba7f2b0115741a022ec4fd85 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/AIbox/flow2.png differ diff --git a/docs/Application_guide/zh/media/solutions/AIbox/run.png b/docs/Application_guide/zh/media/solutions/AIbox/run.png new file mode 100644 index 0000000000000000000000000000000000000000..57382b2b9758ee4b762aa0f3a9fc5a729286c105 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/AIbox/run.png differ diff --git a/docs/Application_guide/zh/media/solutions/AIbox/software.png b/docs/Application_guide/zh/media/solutions/AIbox/software.png new file mode 100644 index 0000000000000000000000000000000000000000..11f6f47f64cc1038ae2f853725482c2fae7a18ce Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/AIbox/software.png differ diff --git a/docs/Application_guide/zh/media/solutions/AIbox/text.mp4 b/docs/Application_guide/zh/media/solutions/AIbox/text.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2fb173ca684f5aecbfea0a24c3a7868d70d439f4 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/AIbox/text.mp4 differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI/EC800MCNLER01A03M08_OCPU_QPY_TEST0707A.zip b/docs/Application_guide/zh/media/solutions/xiaozhi_AI/EC800MCNLER01A03M08_OCPU_QPY_TEST0707A.zip deleted file mode 100644 index cbf423fd0eef9100f2038e4abb0cad2f2a96462d..0000000000000000000000000000000000000000 Binary files a/docs/Application_guide/zh/media/solutions/xiaozhi_AI/EC800MCNLER01A03M08_OCPU_QPY_TEST0707A.zip and /dev/null differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI/EC800MCNLER06A01M08_AI_WS_OCPU_QPY_BETA0820(1).zip b/docs/Application_guide/zh/media/solutions/xiaozhi_AI/EC800MCNLER06A01M08_AI_WS_OCPU_QPY_BETA0820(1).zip new file mode 100644 index 0000000000000000000000000000000000000000..1eedba40c04d3ebf26bb3fa729048e54b1ee6e74 Binary files /dev/null and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI/EC800MCNLER06A01M08_AI_WS_OCPU_QPY_BETA0820(1).zip differ diff --git a/docs/Application_guide/zh/media/solutions/xiaozhi_AI/software.png b/docs/Application_guide/zh/media/solutions/xiaozhi_AI/software.png index d2bef1b682f2f63d569fba41271f4aaa94bb8778..747c85bc0a3996012cbc7c6fb0dc34689709e2f1 100644 Binary files a/docs/Application_guide/zh/media/solutions/xiaozhi_AI/software.png and b/docs/Application_guide/zh/media/solutions/xiaozhi_AI/software.png differ diff --git a/docs/Application_guide/zh/sidebar.yaml b/docs/Application_guide/zh/sidebar.yaml index 6adc85e512169ee1488dd310f33718ca89b7871d..da0130656ce2ffabbd3ebc8c600b7d7180c4fb23 100644 --- a/docs/Application_guide/zh/sidebar.yaml +++ b/docs/Application_guide/zh/sidebar.yaml @@ -272,7 +272,7 @@ items: file: solutions/AIChatBot-Volcengine-webRTC/quick_start.md - label: 软件设计讲解 file: solutions/AIChatBot-Volcengine-webRTC/software_design.md - - label: 小智 AI 聊天机器人 + - label: 基于websocket的小智 AI 聊天机器人 file: solutions/xiaozhi_AI/README.md items: - label: 开发资源汇总 @@ -280,7 +280,7 @@ items: - label: 快速上手 file: solutions/xiaozhi_AI/quick_start.md - label: 软件设计讲解 - file: solutions/xiaozhi_AI/software_design.md + file: solutions/xiaozhi_AI/software_design.md - label: 基于MQTT+UDP的小智AI聊天机器人 file: solutions/xiaozhi_AI_mqtt/README.md items: @@ -289,7 +289,16 @@ items: - label: 快速上手 file: solutions/xiaozhi_AI_mqtt/quick_start.md - label: 软件设计讲解 - file: solutions/xiaozhi_AI_mqtt/software_design.md + file: solutions/xiaozhi_AI_mqtt/software_design.md + - label: AIbox + file: solutions/AIbox/README.md + items: + - label: 开发资源汇总 + file: solutions/AIbox/dev_resources.md + - label: 快速上手 + file: solutions/AIbox/quick_start.md + - label: 软件设计讲解 + file: solutions/AIbox/software_design.md - label: EP-D200 华系列 DTU file: solutions/DTU/README.md items: diff --git a/docs/Application_guide/zh/solutions/AIbox/README.md b/docs/Application_guide/zh/solutions/AIbox/README.md new file mode 100644 index 0000000000000000000000000000000000000000..083286e284724c0ec832f00f8c37ea557dc2a218 --- /dev/null +++ b/docs/Application_guide/zh/solutions/AIbox/README.md @@ -0,0 +1,23 @@ +# AIbox 聊天机器人 + +该AIbox 聊天机器人方案具有以下特性： + +- 支持音色切换。 +- 支持语音中断/打断。 +- 支持情感可视化 +- 支持语音唤醒。 +- 使用 Python 语言，便于二次开发。 + + + + + + + + + +--- + +- [开发资源汇总](./dev_resources.md) +- [快速上手](./quick_start.md) +- [软件设计讲解](./software_design.md) diff --git a/docs/Application_guide/zh/solutions/AIbox/dev_resources.md b/docs/Application_guide/zh/solutions/AIbox/dev_resources.md new file mode 100644 index 0000000000000000000000000000000000000000..542e8eb6c41df01be578e18433569bea0fdbf22b --- /dev/null +++ b/docs/Application_guide/zh/solutions/AIbox/dev_resources.md @@ -0,0 +1,28 @@ +# 开发资源汇总 + +## 硬件资料 + + - AIbox成品（含开发板、天线、Type-C 数据线等） + + > - [点此购买AIbox套件](https://www.quecmall.com/goods-detail/2c90800b9488359c0195efe6367303b5) + +## 软件资料 + + - QuecPython 模块的 USB 驱动：[QuecPython_USB_Driver_Win10_ASR](https://developer.quectel.com/wp-content/uploads/2024/09/Quectel_Windows_USB_DriverA_Customer_V1.1.13.zip) + - [QuecPython 固件](../../media/solutions/AIbox/EC800MCNLE8.10.zip) + +## 开发工具 + +- QPYcom - QuecPython 调试工具 + - 版本：V3.6.0 + - 下载 [QPYcom](https://python.quectel.com/wp-content/uploads/2024/09/QPYcom_V3.6.0.zip) +- VSCode - 代码编辑器 + - 下载 [VSCode](https://code.visualstudio.com/) + +## 实验源码 + +- 从 Github 仓库获取源代码步骤如下： + ```bash + git clone https://github.com/QuecPython/AIBox.git + ``` +- [压缩包下载](https://codeload.github.com/QuecPython/AIBox/zip/refs/heads/main) \ No newline at end of file diff --git a/docs/Application_guide/zh/solutions/AIbox/quick_start.md b/docs/Application_guide/zh/solutions/AIbox/quick_start.md new file mode 100644 index 0000000000000000000000000000000000000000..533b29c82dd17f1392558fa9bbe15aae4e81188c --- /dev/null +++ b/docs/Application_guide/zh/solutions/AIbox/quick_start.md @@ -0,0 +1,67 @@ +# 快速上手 + +## 硬件准备 + +- AIbox成品（含开发板、天线、Type-C 数据线等） + + > - [点此购买AIbox套件](https://www.quecmall.com/goods-detail/2c90800b8cb50925018e5a696d0104f7) + +## 环境搭建 + +- 下载并安装 EC800M 系列模组驱动：[QuecPython_USB_Driver_Win10_ASR](https://developer.quectel.com/wp-content/uploads/2024/09/Quectel_Windows_USB_DriverA_Customer_V1.1.13.zip)。 +- 下载并安装 [VSCode](https://code.visualstudio.com/)。 +- 下载并解压 [QPYCom](https://images.quectel.com/python/2022/12/QPYcom_V3.6.0.zip) 工具到电脑的合适位置。 +- [下载固件包](../../media/solutions/AIbox/EC800MCNLE8.10.zip)。 +- 下载[实验源码](https://codeload.github.com/QuecPython/AIBox/zip/refs/heads/main)。 + + +## 硬件连接 + +按照下图进行硬件连接： + + + + +1. 使用 Type-C 数据线连接AIbox和电脑。 + +## 设备开发 + +### 开机 + +完成硬件连接的工作后，电脑设备管理器的端口列表中出现包含 `Quectel USB` 字样的 COM 口，表示开机成功。 + +![comport.png](../../media/solutions/AIChatBot-Volcengine-webRTC/comport.png) + +### 烧录固件包 + +参考[此章节](https://developer.quectel.com/doc/quecpython/Application_guide/zh/dev-tools/QPYcom/qpycom-dw.html#%E4%B8%8B%E8%BD%BD%E5%9B%BA%E4%BB%B6)，烧录对应型号固件包至开发板。 + +### 脚本导入与运行 + +1. 参考[此章节](https://developer.quectel.com/doc/quecpython/Getting_started/zh/4G/first_python.html)，将源码目录下 `code` 文件夹中的所有文件导入到模组文件系统，如下图所示： + + +参考[此章节](https://developer.quectel.com/doc/quecpython/Getting_started/zh/4G/first_python.html#执行脚本文件)，执行主程序文件 `_main.py`。 + +3. 参考[此章节](https://developer.quectel.com/doc/quecpython/Getting_started/zh/4G/first_python.html#停止程序运行)，停止程序运行。 + +## 业务调试 + +### 程序启动 + +执行 `_main.py` 脚本后，程序开始运行。 + +### 激活小智 + +下图所示状态为待唤醒状态，开发板会出现红灯闪烁，需要语音“小智，小智”来唤醒小智AI进行语音对话。 + + + +唤醒后会出现连接服务器相关的数据或者是连接失败的提示，当出现连接失败时，请检查网络是否正常。 + + + +当系统长时间未进行对话或未接收到命令时会自动断开连接，等待下次唤醒。 + + + diff --git a/docs/Application_guide/zh/solutions/AIbox/software_design.md b/docs/Application_guide/zh/solutions/AIbox/software_design.md new file mode 100644 index 0000000000000000000000000000000000000000..a39c5c14999d60f804b0b71d1e74e957a2d4cdbe --- /dev/null +++ b/docs/Application_guide/zh/solutions/AIbox/software_design.md @@ -0,0 +1,433 @@ +# 软件设计讲解 + +## 软件框架 + +### 框架设计图 + + + +### 业务系统启动流程 + + + +## 代码讲解 + +### AI 唤醒&人声检测 + +```python +class Application(object): + def on_keyword_spotting(self, state): + logger.info("on_keyword_spotting: {}".format(state)) + if state == 0: + # 唤醒词触发 + if self.__working_thread is not None and self.__working_thread.is_running(): + return + self.__working_thread = Thread(target=self.__working_thread_handler) + self.__working_thread.start() + self.__keyword_spotting_event.clear() + else: + self.__keyword_spotting_event.set() + + def on_voice_activity_detection(self, state): + gc.collect() + logger.info("on_voice_activity_detection: {}".format(state)) + if state == 1: + self.__voice_activity_event.set() # 有人声 + else: + self.__voice_activity_event.clear() # 无人声 +``` + +### AI 初始化 + +初始化 AI 对象以及其他硬件驱动。 + +```python +class Application(object): + + def __init__(self): + + Pin(Pin.GPIO33, Pin.OUT, Pin.PULL_PD, 1) + #初始化屏幕 + self.lvgl=lvglManager() + # 初始化充电管理 + self.charge_manager = ChargeManager() + + # 初始化音频管理 + self.audio_manager = AudioManager() + self.audio_manager.set_kws_cb(self.on_keyword_spotting) + self.audio_manager.set_vad_cb(self.on_voice_activity_detection) + + # 初始化网络管理 + self.net_manager = NetManager() + + # 初始化任务调度器 + self.task_manager = TaskManager() + + # 初始化协议 + self.__protocol = WebSocketClient() + self.__protocol.set_callback( + audio_message_handler=self.on_audio_message, + json_message_handler=self.on_json_message + ) + + self.__working_thread = None + self.__record_thread = None + self.__record_thread_stop_event = Event() + self.__voice_activity_event = Event() + self.__keyword_spotting_event = Event() +``` + +### AI 对话中断逻辑 + +在唤醒AI之后启动该人声音频检测与上传的线程，`self.start_vad()`用于启动录音功能，当有人声时会通过websocket发送“开始监听”的标志位`start`，然后执行`self.__protocol.abort()`结束当前的语音从而实现打断AI说话。 + +```python +class Application(object): + def __chat_process(self): + self.start_vad() + try: + with self.__protocol: + self.power_red_led.on() + self.__protocol.hello() + self.__protocol.wakeword_detected("小智") + is_listen_flag = False + while True: + data = self.audio_manager.opus_read() + if self.__voice_activity_event.is_set(): + # 有人声 + if not is_listen_flag: + self.__protocol.abort() + self.__protocol.listen("start") + is_listen_flag = True + self.__protocol.send(data) + # logger.debug("send opus data to server") + else: + if is_listen_flag: + self.__protocol.listen("stop") + is_listen_flag = False + if not self.__protocol.is_state_ok(): + break + # logger.debug("read opus data length: {}".format(len(data))) + except Exception as e: + logger.debug("working thread handler got Exception: {}".format(repr(e))) + finally: + self.stop_vad() + +``` + +### 音频管理 + +统一管理设备的音频输入输出、编解码、语音识别相关功能（关键词识别 KWS 和语音活动检测 VAD），并提供回调接口供上层应用使用。 + +```python +class AudioManager(object): + + def __init__(self, channel=0, volume=11, pa_number=29): + self.aud = audio.Audio(channel) # 初始化音频播放通道 + self.aud.set_pa(pa_number) + self.aud.setVolume(volume) # 设置音量 + self.aud.setCallback(self.audio_cb) + self.rec = audio.Record(channel) + self.__skip = 0 + + # ========== 音频文件 ==================== + + def audio_cb(self, event): + if event == 0: + # logger.info('audio play start.') + pass + elif event == 7: + # logger.info('audio play finish.') + pass + else: + pass + + def play(self, file): + self.aud.play(0, 1, file) + + # ========= opus ==================== + + def open_opus(self): + self.pcm = audio.Audio.PCM(0, 1, 16000, 2, 1, 15) # 5 -> 25 + self.opus = Opus(self.pcm, 0, 6000) # 6000 ~ 128000 + + def close_opus(self): + self.opus.close() + self.pcm.close() + del self.opus + del self.pcm + + def opus_read(self): + return self.opus.read(60) + + def opus_write(self, data): + return self.opus.write(data) + + # ========= vad & kws ==================== + + def set_kws_cb(self, cb): + self.rec.ovkws_set_callback(cb) + + def set_vad_cb(self, cb): + def wrapper(state): + if self.__skip != 2: + self.__skip += 1 + return + return cb(state) + self.rec.vad_set_callback(wrapper) + + def end_cb(self, para): + if(para[0] == "stream"): + if(para[2] == 1): + pass + elif (para[2] == 3): + pass + else: + pass + else: + pass + + def start_kws(self): + self.rec.ovkws_start("_xiao_zhi_xiao_zhi", 0.7) + + def stop_kws(self): + self.rec.ovkws_stop() + + def start_vad(self): + self.__skip = 0 + self.rec.vad_start() + + def stop_vad(self): + self.rec.vad_stop() +``` + + + +## LCD屏显 + +通过会话总线`sys_bus`来控制LCD的表情显示 + +```python +import lvgl as lv +import utime +import sys_bus +from usr.lcd import * +from machine import Timer +import log + +log.basicConfig(level=log.INFO) +logger = log.getLogger("UI") + + +screen = lv.obj() +screen.set_size(240,240) +screen.set_scrollbar_mode(lv.SCROLLBAR_MODE.OFF) + +# Set style for screen, Part: lv.PART.MAIN, State: lv.STATE.DEFAULT. +screen.set_style_bg_opa(255, lv.PART.MAIN|lv.STATE.DEFAULT) +screen.set_style_bg_color(lv.color_hex(0x000000), lv.PART.MAIN|lv.STATE.DEFAULT) +screen.set_style_bg_grad_dir(lv.GRAD_DIR.NONE, lv.PART.MAIN|lv.STATE.DEFAULT) + +# Create flex flow +screen.center() +screen.set_flex_align(lv.FLEX_ALIGN.SPACE_EVENLY, lv.FLEX_ALIGN.CENTER, lv.FLEX_ALIGN.CENTER) +screen.set_flex_flow(lv.FLEX_FLOW.COLUMN) +# Create screen_gif +screen_gif = lv.gif(screen) +screen_gif.set_src("U:/media/happy.gif") +screen_gif.set_style_bg_color(lv.color_hex(0x000000), 0) # 黑色背景 +screen_gif.set_style_bg_opa(lv.OPA.COVER, 0) +screen_gif.set_size(240, 240) + +def update_emoji(topic,msg): + screen_gif.set_style_opa(lv.OPA.TRANSP, 0) + if msg == "happy": + screen_gif.set_src("U:/media/happy.gif") + elif msg == "cool": + screen_gif.set_src("U:/media/cool.gif") + elif msg == "thinking": + screen_gif.set_src("U:/media/thinking.gif") + elif msg == "angry": + screen_gif.set_src("U:/media/angry.gif") + elif msg == "sleep": + screen_gif.set_src("U:/media/sleep.gif") + elif msg == "confident": + screen_gif.set_src("U:/media/confident.gif") + elif msg == "crying": + screen_gif.set_src("U:/media/crying.gif") + elif msg == "delicious": + screen_gif.set_src("U:/media/delicious.gif") + elif msg == "funny": + screen_gif.set_src("U:/media/funny.gif") + elif msg == "kissy": + screen_gif.set_src("U:/media/kissy.gif") + elif msg == "laughing": + screen_gif.set_src("U:/media/laughing.gif") + elif msg == "loving": + screen_gif.set_src("U:/media/loving.gif") + elif msg == "neutral": + screen_gif.set_src("U:/media/neutral.gif") + elif msg == "sleepy": + screen_gif.set_src("U:/media/sleep.gif") + elif msg == "sad": + screen_gif.set_src("U:/media/sad.gif") + elif msg == "surprised": + screen_gif.set_src("U:/media/surprised.gif") + elif msg == "winking": + screen_gif.set_src("U:/media/winking.gif") + elif msg == "silly": + screen_gif.set_src("U:/media/silly.gif") + elif msg == "relaxed": + screen_gif.set_src("U:/media/relaxed.gif") + elif msg == "embarrassed": + screen_gif.set_src("U:/media/embarrassed.gif") + else: + pass + utime.sleep_ms(20) + screen_gif.set_style_opa(lv.OPA.COVER, 0) + +sys_bus.subscribe("update_emoji", update_emoji) + +class lvglManager: + #@staticmethod + def __init__(self): + lv.scr_load(screen) + +``` + + + +## MCP管理 + +基于 MCP 协议的消息构造与发送功能，涵盖了初始化、工具列表查询、工具调用响应以及设备通知等场景。所有消息均遵循 JSON-RPC 2.0 格式，并通过 `send_mcp` 方法统一发送，确保了代码的模块化和一致性。 + +```python +class WebSocketClient(object): + def send_mcp(self, payload, session_id=""): + """ + 发送标准MCP消息,payload为JSON-RPC 2.0格式字典 + """ + with self.__resp_helper: + self.send( + JsonMessage( + { + "session_id": session_id, + "type": "mcp", + "payload": payload + } + ).to_bytes() + ) + def mcp_initialize(self, capabilities=None, session_id="", req_id=1): + """ + 发送MCP initialize响应 + """ + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2025-9-03", + "capabilities": { + "tools":{}, + "notifications": {} + }, + "serverInfo": { + "name": 'xiaozhi-mqtt-client', + "version": "1.0.0" + } + } + } + self.send_mcp(payload, session_id) + def mcp_tools_list(self, cursor="", session_id="", req_id=2): + """ + 发送MCP tools/list响应请求 + """ + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "tools": [ + { + "name": "self.setvolume_down()", + "description": "只通过调用setvolume_down方法来控制音量变小,接收到回应后会播报当前音量大小", + "inputSchema": {} + }, + { + "name": "self.setvolume_up()", + "description": "只通过调用setvolume_up方法来控制音量变大,接收到回应后会播报当前音量大小", + "inputSchema": {} + }, + { + "name": "self.setvolume_close()", + "description": "只通过调用setvolume_close方法来静音,接收到回应后会播报当前音量大小", + "inputSchema": {} + }, + ], + } + } + + self.send_mcp(payload, session_id) + + def mcp_tools_call(self, session_id="", req_id="", error=None, tool_name=""): + """ + 发送MCP tools/call响应 + :param error: 如果为None则返回成功响应,否则返回错误响应(字典,包含code和message) + """ + if error is None: + if tool_name == "self.setvolume_down()": + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [ + { "type": "text", "text": "音量已调小 "} + ], + "isError": False + } + } + elif tool_name == "self.setvolume_up()": + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [ + { "type": "text", "text": "音量已调大" } + ], + "isError": False + } + } + elif tool_name == "self.setvolume_close()": + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [ + { "type": "text", "text": "已静音" } + ], + "isError": False + } + } + else: + payload = { + "jsonrpc": "2.0", + "id": req_id, + "error": { + "code": error.get("code", -32601), + "message": error.get("message", "Unknown error") + } + self.send_mcp(payload, session_id) + + def mcp_notify(self, method, params, session_id=""): + """ + 设备主动发送MCP通知 + """ + payload = { + "jsonrpc": "2.0", + "method": "notifications/state_changed", + "params": { + "newState": "idle", + "oldState": "connecting" + } + } + self.send_mcp(payload, session_id) +``` + diff --git a/docs/Application_guide/zh/solutions/README.md b/docs/Application_guide/zh/solutions/README.md index 6d5400fbca6806ae0081f7b50d703bb36a384948..afd35410caf6bc3e1f056dfaaac3e6b54087e87b 100644 --- a/docs/Application_guide/zh/solutions/README.md +++ b/docs/Application_guide/zh/solutions/README.md @@ -3,8 +3,9 @@ 为了方便客户应用开发，加快客户项目快速落地，QuecPython 团队针对一些常见的行业应用推出了自己的软件解决方案，包括云喇叭、DTU、公网对讲机、电表、网络摄像头、CPE 电话机、智能定位器、手表、电子学生证、充电桩等。 - [豆包 RTC AI 聊天机器人](./AIChatBot-Volcengine-webRTC/README.md) -- [小智 AI 聊天机器人](./xiaozhi_AI/README.md) +- [基于websocket的小智 AI 聊天机器人](./xiaozhi_AI/README.md) - [基于MQTT+UDP的小智AI聊天机器人](./xiaozhi_AI_mqtt/README.md) +- [AIbox](./AIbox/README.md) - [EP-D200 华系列 DTU](./DTU/README.md) - [智慧农业中控面板](./Agriculture-Control-Panel/README.md) - [可穿戴解决方案](./Wear/readme.md) diff --git a/docs/Application_guide/zh/solutions/xiaozhi_AI/README.md b/docs/Application_guide/zh/solutions/xiaozhi_AI/README.md index 5d27db1a141febab308d0394f07ec892b56067ee..29eb102069866f2d44f413b2f415f02af95f1a29 100644 --- a/docs/Application_guide/zh/solutions/xiaozhi_AI/README.md +++ b/docs/Application_guide/zh/solutions/xiaozhi_AI/README.md @@ -1,4 +1,4 @@ -# 小智 AI 聊天机器人 +# 基于websocket的小智 AI 聊天机器人 该小智 AI 聊天机器人方案具有以下特性： diff --git a/docs/Application_guide/zh/solutions/xiaozhi_AI/quick_start.md b/docs/Application_guide/zh/solutions/xiaozhi_AI/quick_start.md index 2024e2a383a9096f79e4b6c90b44914974b84ef4..dea8489126531c78e4efe0642fefbe695e944bb7 100644 --- a/docs/Application_guide/zh/solutions/xiaozhi_AI/quick_start.md +++ b/docs/Application_guide/zh/solutions/xiaozhi_AI/quick_start.md @@ -13,7 +13,7 @@ - 下载并安装 EC800M 系列模组驱动：[QuecPython_USB_Driver_Win10_ASR](https://developer.quectel.com/wp-content/uploads/2024/09/Quectel_Windows_USB_DriverA_Customer_V1.1.13.zip)。 - 下载并安装 [VSCode](https://code.visualstudio.com/)。 - 下载并解压 [QPYCom](https://images.quectel.com/python/2022/12/QPYcom_V3.6.0.zip) 工具到电脑的合适位置。 -- [下载固件包](../../media/solutions/xiaozhi_AI/EC800MCNLER01A03M08_OCPU_QPY_TEST0707A.zip)。 +- [下载固件包](../../media/solutions/xiaozhi_AI/EC800MCNLER06A01M08_AI_WS_OCPU_QPY_BETA0820(1).zip)。 - 下载[实验源码](https://github.com/QuecPython/solution-xiaozhiAI/archive/refs/heads/master.zip)。 diff --git a/docs/Application_guide/zh/solutions/xiaozhi_AI/software_design.md b/docs/Application_guide/zh/solutions/xiaozhi_AI/software_design.md index b9a36d3d50863c292fb044ee7374e4c0584c2155..e236c27b91cc5c166c330eb287dc4b70b5925d71 100644 --- a/docs/Application_guide/zh/solutions/xiaozhi_AI/software_design.md +++ b/docs/Application_guide/zh/solutions/xiaozhi_AI/software_design.md @@ -4,7 +4,7 @@ ### 框架设计图 - + ### 业务系统启动流程 @@ -87,6 +87,8 @@ class Application(object): ### AI 对话中断逻辑 +在唤醒AI之后启动该人声音频检测与上传的线程，`self.start_vad()`用于启动录音功能，当有人声时会通过websocket发送“开始监听”的标志位`start`，然后执行`self.__protocol.abort()`结束当前的语音从而实现打断AI说话。 + ```python class Application(object): def __chat_process(self): @@ -102,6 +104,7 @@ class Application(object): if self.__voice_activity_event.is_set(): # 有人声 if not is_listen_flag: + self.__protocol.abort() self.__protocol.listen("start") is_listen_flag = True self.__protocol.send(data) @@ -128,13 +131,33 @@ class Application(object): ```python class AudioManager(object): - def __init__(self, channel=0, volume=11, pa_number=29): + def __init__(self, channel=0, volume=5, pa_number=29): self.aud = audio.Audio(channel) # 初始化音频播放通道 self.aud.set_pa(pa_number) self.aud.setVolume(volume) # 设置音量 self.aud.setCallback(self.audio_cb) self.rec = audio.Record(channel) + self.rec.gain_set(4,10) self.__skip = 0 + + def setvolume_down(self): + global volume + volume -= 1 + if volume < 0: volume = 0 + self.aud.setVolume(volume) + return volume + + def setvolume_up(self): + global volume + volume += 1 + if volume > 11: volume = 11 + self.aud.setVolume(volume) + return volume + + def setvolume_close(self): + self.aud.setVolume(0) + volume = 0 + return volume # ========== 音频文件 ==================== @@ -207,3 +230,139 @@ class AudioManager(object): self.rec.vad_stop() ``` + + +## MCP管理 + +基于 MCP 协议的消息构造与发送功能，涵盖了初始化、工具列表查询、工具调用响应以及设备通知等场景。所有消息均遵循 JSON-RPC 2.0 格式，并通过 `send_mcp` 方法统一发送，确保了代码的模块化和一致性。 + +```python +class WebSocketClient(object): + def send_mcp(self, payload, session_id=""): + """ + 发送标准MCP消息,payload为JSON-RPC 2.0格式字典 + """ + with self.__resp_helper: + self.send( + JsonMessage( + { + "session_id": session_id, + "type": "mcp", + "payload": payload + } + ).to_bytes() + ) + def mcp_initialize(self, capabilities=None, session_id="", req_id=1): + """ + 发送MCP initialize响应 + """ + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2025-9-03", + "capabilities": { + "tools":{}, + "notifications": {} + }, + "serverInfo": { + "name": 'xiaozhi-mqtt-client', + "version": "1.0.0" + } + } + } + self.send_mcp(payload, session_id) + def mcp_tools_list(self, cursor="", session_id="", req_id=2): + """ + 发送MCP tools/list响应请求 + """ + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "tools": [ + { + "name": "self.setvolume_down()", + "description": "只通过调用setvolume_down方法来控制音量变小,接收到回应后会播报当前音量大小", + "inputSchema": {} + }, + { + "name": "self.setvolume_up()", + "description": "只通过调用setvolume_up方法来控制音量变大,接收到回应后会播报当前音量大小", + "inputSchema": {} + }, + { + "name": "self.setvolume_close()", + "description": "只通过调用setvolume_close方法来静音,接收到回应后会播报当前音量大小", + "inputSchema": {} + }, + ], + } + } + + self.send_mcp(payload, session_id) + + def mcp_tools_call(self, session_id="", req_id="", error=None, tool_name=""): + """ + 发送MCP tools/call响应 + :param error: 如果为None则返回成功响应,否则返回错误响应(字典,包含code和message) + """ + if error is None: + if tool_name == "self.setvolume_down()": + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [ + { "type": "text", "text": "音量已调小 "} + ], + "isError": False + } + } + elif tool_name == "self.setvolume_up()": + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [ + { "type": "text", "text": "音量已调大" } + ], + "isError": False + } + } + elif tool_name == "self.setvolume_close()": + payload = { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [ + { "type": "text", "text": "已静音" } + ], + "isError": False + } + } + else: + payload = { + "jsonrpc": "2.0", + "id": req_id, + "error": { + "code": error.get("code", -32601), + "message": error.get("message", "Unknown error") + } + self.send_mcp(payload, session_id) + + def mcp_notify(self, method, params, session_id=""): + """ + 设备主动发送MCP通知 + """ + payload = { + "jsonrpc": "2.0", + "method": "notifications/state_changed", + "params": { + "newState": "idle", + "oldState": "connecting" + } + } + self.send_mcp(payload, session_id) +``` +