From 401dd7a5d6992098b1fc23d33c66975594441bd7 Mon Sep 17 00:00:00 2001 From: chenzheng Date: Wed, 25 Sep 2024 15:12:04 +0800 Subject: [PATCH 1/4] new patch adapte --- .../Host/itrustee_tzdriver.patch | 113 +++++++++--------- 1 file changed, 55 insertions(+), 58 deletions(-) diff --git a/trustzone-awared-vm/Host/itrustee_tzdriver.patch b/trustzone-awared-vm/Host/itrustee_tzdriver.patch index 2712033..2023a5c 100644 --- a/trustzone-awared-vm/Host/itrustee_tzdriver.patch +++ b/trustzone-awared-vm/Host/itrustee_tzdriver.patch @@ -1,6 +1,6 @@ diff -uprN itrustee_tzdriver/auth/auth_base_impl.c itrustee_tzdriver_new/auth/auth_base_impl.c ---- itrustee_tzdriver/auth/auth_base_impl.c 2024-07-12 11:08:59.353629380 +0800 -+++ itrustee_tzdriver_new/auth/auth_base_impl.c 2024-07-12 10:51:32.889629380 +0800 +--- itrustee_tzdriver/auth/auth_base_impl.c 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/auth/auth_base_impl.c 2024-09-25 14:56:17.511977710 +0800 @@ -336,11 +336,25 @@ int check_teecd_auth(void) return CHECK_ACCESS_SUCC; @@ -28,8 +28,8 @@ diff -uprN itrustee_tzdriver/auth/auth_base_impl.c itrustee_tzdriver_new/auth/au #ifdef CONFIG_TEE_TELEPORT_AUTH diff -uprN itrustee_tzdriver/auth/auth_base_impl.h itrustee_tzdriver_new/auth/auth_base_impl.h ---- itrustee_tzdriver/auth/auth_base_impl.h 2024-07-12 11:08:59.353629380 +0800 -+++ itrustee_tzdriver_new/auth/auth_base_impl.h 2024-07-12 10:51:32.885629380 +0800 +--- itrustee_tzdriver/auth/auth_base_impl.h 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/auth/auth_base_impl.h 2024-09-25 14:56:17.511977710 +0800 @@ -84,6 +84,7 @@ void mutex_crypto_hash_lock(void); void mutex_crypto_hash_unlock(void); int check_hidl_auth(void); @@ -51,8 +51,8 @@ diff -uprN itrustee_tzdriver/auth/auth_base_impl.h itrustee_tzdriver_new/auth/au #endif /* CLIENT_AUTH || TEECD_AUTH */ diff -uprN itrustee_tzdriver/core/agent.c itrustee_tzdriver_new/core/agent.c ---- itrustee_tzdriver/core/agent.c 2024-07-12 11:08:59.357629380 +0800 -+++ itrustee_tzdriver_new/core/agent.c 2024-07-12 10:51:32.865629380 +0800 +--- itrustee_tzdriver/core/agent.c 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/core/agent.c 2024-09-25 14:56:17.511977710 +0800 @@ -298,7 +298,8 @@ int tc_ns_set_native_hash(unsigned long return ret; } @@ -129,8 +129,8 @@ diff -uprN itrustee_tzdriver/core/agent.c itrustee_tzdriver_new/core/agent.c if (is_agent_already_exist(agent_id, nsid, &event_data, dev_file, &find_flag)) diff -uprN itrustee_tzdriver/core/agent.h itrustee_tzdriver_new/core/agent.h ---- itrustee_tzdriver/core/agent.h 2024-07-12 11:08:59.357629380 +0800 -+++ itrustee_tzdriver_new/core/agent.h 2024-07-12 10:51:32.861629380 +0800 +--- itrustee_tzdriver/core/agent.h 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/core/agent.h 2024-09-25 14:56:17.511977710 +0800 @@ -118,7 +118,8 @@ int agent_process_work(const struct tc_n unsigned int agent_id, unsigned int nsid); int is_agent_alive(unsigned int agent_id, unsigned int nsid); @@ -152,8 +152,8 @@ diff -uprN itrustee_tzdriver/core/agent.h itrustee_tzdriver_new/core/agent.h int tee_agent_clear_work(struct tc_ns_client_context *context, unsigned int dev_file_id); diff -uprN itrustee_tzdriver/core/gp_ops.c itrustee_tzdriver_new/core/gp_ops.c ---- itrustee_tzdriver/core/gp_ops.c 2024-07-12 11:08:59.357629380 +0800 -+++ itrustee_tzdriver_new/core/gp_ops.c 2024-07-12 10:51:32.845629380 +0800 +--- itrustee_tzdriver/core/gp_ops.c 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/core/gp_ops.c 2024-09-25 14:56:17.511977710 +0800 @@ -231,6 +231,84 @@ int write_to_client(void __user *dest, s return 0; } @@ -455,7 +455,7 @@ diff -uprN itrustee_tzdriver/core/gp_ops.c itrustee_tzdriver_new/core/gp_ops.c if (index >= TEE_PARAM_NUM) return -EINVAL; -@@ -513,19 +733,57 @@ static int transfer_shared_mem(const str +@@ -518,19 +738,57 @@ static int transfer_shared_mem(const str buffer_addr = client_param->memref.buffer | ((uint64_t)client_param->memref.buffer_h_addr << ADDR_TRANS_NUM); @@ -525,7 +525,7 @@ diff -uprN itrustee_tzdriver/core/gp_ops.c itrustee_tzdriver_new/core/gp_ops.c } op_params->local_tmpbuf[index].temp_buffer = buff; -@@ -695,13 +953,24 @@ static int update_tmp_mem(const struct t +@@ -707,13 +965,24 @@ static int update_tmp_mem(const struct t if (buffer_size == 0) return 0; /* Only update the buffer when the buffer size is valid in complete case */ @@ -557,21 +557,22 @@ diff -uprN itrustee_tzdriver/core/gp_ops.c itrustee_tzdriver_new/core/gp_ops.c } return 0; } -@@ -871,7 +1140,10 @@ static void free_operation_params(const - #ifdef CONFIG_NOCOPY_SHAREDMEM - tlogd("free_operation_params release nocopy or register shm\n"); - temp_buf = local_tmpbuf[index].temp_buffer; -- if (temp_buf != NULL) { -+ if (temp_buf != NULL && call_params->dev->isVM) { -+ release_vm_shared_mem_page(temp_buf, local_tmpbuf[index].size, call_params->dev->vm_page_size); -+ mailbox_free(temp_buf); -+ } else if (temp_buf != NULL && !call_params->dev->isVM) { - release_shared_mem_page(temp_buf, local_tmpbuf[index].size); - mailbox_free(temp_buf); - } +@@ -863,7 +1132,11 @@ static void free_operation_sharedmem_par + } + } + #endif +- release_shared_mem_page(temp_buf, temp_buf_sz); ++ if (call_params->dev->isVM) { ++ release_vm_shared_mem_page(temp_buf, temp_buf_sz, call_params->dev->vm_page_size); ++ } else { ++ release_shared_mem_page(temp_buf, temp_buf_sz); ++ } + mailbox_free(temp_buf); + } + #endif diff -uprN itrustee_tzdriver/core/gp_ops.h itrustee_tzdriver_new/core/gp_ops.h ---- itrustee_tzdriver/core/gp_ops.h 2024-07-12 11:08:59.357629380 +0800 -+++ itrustee_tzdriver_new/core/gp_ops.h 2024-07-12 10:51:32.841629380 +0800 +--- itrustee_tzdriver/core/gp_ops.h 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/core/gp_ops.h 2024-09-25 14:56:17.511977710 +0800 @@ -28,5 +28,9 @@ int tc_client_call(const struct tc_call_ bool is_tmp_mem(uint32_t param_type); bool is_ref_mem(uint32_t param_type); @@ -583,9 +584,9 @@ diff -uprN itrustee_tzdriver/core/gp_ops.h itrustee_tzdriver_new/core/gp_ops.h #endif diff -uprN itrustee_tzdriver/core/session_manager.c itrustee_tzdriver_new/core/session_manager.c ---- itrustee_tzdriver/core/session_manager.c 2024-07-12 11:08:59.357629380 +0800 -+++ itrustee_tzdriver_new/core/session_manager.c 2024-07-12 10:51:32.801629380 +0800 -@@ -589,7 +589,7 @@ static struct tc_ns_service *tc_ref_serv +--- itrustee_tzdriver/core/session_manager.c 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/core/session_manager.c 2024-09-25 14:56:17.511977710 +0800 +@@ -605,7 +605,7 @@ static struct tc_ns_service *tc_ref_serv } static int tc_ns_service_init(const unsigned char *uuid, uint32_t uuid_len, @@ -594,7 +595,7 @@ diff -uprN itrustee_tzdriver/core/session_manager.c itrustee_tzdriver_new/core/s { int ret = 0; struct tc_ns_service *service = NULL; -@@ -610,7 +610,7 @@ static int tc_ns_service_init(const unsi +@@ -626,7 +626,7 @@ static int tc_ns_service_init(const unsi } #ifdef CONFIG_CONFIDENTIAL_CONTAINER @@ -603,7 +604,7 @@ diff -uprN itrustee_tzdriver/core/session_manager.c itrustee_tzdriver_new/core/s #else service->nsid = PROC_PID_INIT_INO; #endif -@@ -648,7 +648,11 @@ static struct tc_ns_service *find_servic +@@ -664,7 +664,11 @@ static struct tc_ns_service *find_servic struct tc_ns_service *service = NULL; bool is_full = false; #ifdef CONFIG_CONFIDENTIAL_CONTAINER @@ -616,7 +617,7 @@ diff -uprN itrustee_tzdriver/core/session_manager.c itrustee_tzdriver_new/core/s #else unsigned int nsid = PROC_PID_INIT_INO; #endif -@@ -677,7 +681,7 @@ static struct tc_ns_service *find_servic +@@ -693,7 +697,7 @@ static struct tc_ns_service *find_servic goto add_service; } /* Create a new service if we couldn't find it in list */ @@ -625,7 +626,7 @@ diff -uprN itrustee_tzdriver/core/session_manager.c itrustee_tzdriver_new/core/s /* unlock after init to make sure find service from all is correct */ mutex_unlock(&g_service_list_lock); if (ret != 0) { -@@ -791,10 +795,19 @@ static int32_t load_image_copy_file(stru +@@ -807,10 +811,19 @@ static int32_t load_image_copy_file(stru } return 0; } @@ -649,7 +650,7 @@ diff -uprN itrustee_tzdriver/core/session_manager.c itrustee_tzdriver_new/core/s } return 0; } -@@ -1387,10 +1400,109 @@ find_session: +@@ -1409,10 +1422,109 @@ find_session: return ret; } @@ -759,7 +760,7 @@ diff -uprN itrustee_tzdriver/core/session_manager.c itrustee_tzdriver_new/core/s ret = tc_ns_send_cmd(dev_file, context); if (ret != 0) -@@ -1399,6 +1511,11 @@ static int ioctl_session_send_cmd(struct +@@ -1421,6 +1533,11 @@ static int ioctl_session_send_cmd(struct if (ret == 0) ret = -EFAULT; } @@ -772,8 +773,8 @@ diff -uprN itrustee_tzdriver/core/session_manager.c itrustee_tzdriver_new/core/s } diff -uprN itrustee_tzdriver/core/shared_mem.c itrustee_tzdriver_new/core/shared_mem.c ---- itrustee_tzdriver/core/shared_mem.c 2024-07-12 11:08:59.357629380 +0800 -+++ itrustee_tzdriver_new/core/shared_mem.c 2024-07-12 10:51:32.793629380 +0800 +--- itrustee_tzdriver/core/shared_mem.c 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/core/shared_mem.c 2024-09-25 14:56:17.511977710 +0800 @@ -120,6 +120,36 @@ void release_shared_mem_page(uint64_t bu put_page(page); } @@ -812,8 +813,8 @@ diff -uprN itrustee_tzdriver/core/shared_mem.c itrustee_tzdriver_new/core/shared int fill_shared_mem_info(uint64_t start_vaddr, uint32_t pages_no, diff -uprN itrustee_tzdriver/core/shared_mem.h itrustee_tzdriver_new/core/shared_mem.h ---- itrustee_tzdriver/core/shared_mem.h 2024-07-12 11:08:59.357629380 +0800 -+++ itrustee_tzdriver_new/core/shared_mem.h 2024-07-12 10:51:32.789629380 +0800 +--- itrustee_tzdriver/core/shared_mem.h 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/core/shared_mem.h 2024-09-25 14:56:17.511977710 +0800 @@ -64,5 +64,5 @@ void free_spi_mem(uint64_t spi_vaddr); int fill_shared_mem_info(uint64_t start_vaddr, uint32_t pages_no, uint32_t offset, uint32_t buffer_size, uint64_t info_addr); @@ -822,8 +823,8 @@ diff -uprN itrustee_tzdriver/core/shared_mem.h itrustee_tzdriver_new/core/shared +void release_vm_shared_mem_page(uint64_t buf, uint32_t buf_size, uint32_t vm_page_size); #endif diff -uprN itrustee_tzdriver/core/tc_client_driver.c itrustee_tzdriver_new/core/tc_client_driver.c ---- itrustee_tzdriver/core/tc_client_driver.c 2024-07-12 11:08:59.357629380 +0800 -+++ itrustee_tzdriver_new/core/tc_client_driver.c 2024-07-12 10:51:32.749629380 +0800 +--- itrustee_tzdriver/core/tc_client_driver.c 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/core/tc_client_driver.c 2024-09-25 14:56:17.511977710 +0800 @@ -186,6 +186,8 @@ static int tc_ns_get_tee_version(const s smc_cmd.operation_h_phys = (uint64_t)mailbox_virt_to_phys((uintptr_t)&mb_pack->operation) >> ADDR_TRANS_NUM; @@ -1032,8 +1033,8 @@ diff -uprN itrustee_tzdriver/core/tc_client_driver.c itrustee_tzdriver_new/core/ return ret; } diff -uprN itrustee_tzdriver/core/tc_client_driver.h itrustee_tzdriver_new/core/tc_client_driver.h ---- itrustee_tzdriver/core/tc_client_driver.h 2024-07-12 11:08:59.357629380 +0800 -+++ itrustee_tzdriver_new/core/tc_client_driver.h 2024-07-12 10:51:32.741629380 +0800 +--- itrustee_tzdriver/core/tc_client_driver.h 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/core/tc_client_driver.h 2024-09-25 14:56:17.511977710 +0800 @@ -47,6 +47,7 @@ int tc_ns_client_open(struct tc_ns_dev_f int tc_ns_client_close(struct tc_ns_dev_file *dev); int is_agent_alive(unsigned int agent_id, unsigned int nsid); @@ -1043,8 +1044,8 @@ diff -uprN itrustee_tzdriver/core/tc_client_driver.h itrustee_tzdriver_new/core/ struct class *driver_class, const struct file_operations *fops); void destory_dev_node(struct dev_node *node, struct class *driver_class); diff -uprN itrustee_tzdriver/core/tc_cvm_driver.c itrustee_tzdriver_new/core/tc_cvm_driver.c ---- itrustee_tzdriver/core/tc_cvm_driver.c 2024-07-12 11:08:59.357629380 +0800 -+++ itrustee_tzdriver_new/core/tc_cvm_driver.c 2024-07-12 10:51:32.737629380 +0800 +--- itrustee_tzdriver/core/tc_cvm_driver.c 2024-09-25 14:59:38.675977710 +0800 ++++ itrustee_tzdriver_new/core/tc_cvm_driver.c 2024-09-25 14:56:17.511977710 +0800 @@ -104,6 +104,10 @@ static long tc_cvm_ioctl(struct file *fi { int ret = -EFAULT; @@ -1056,9 +1057,10 @@ diff -uprN itrustee_tzdriver/core/tc_cvm_driver.c itrustee_tzdriver_new/core/tc_ handle_cmd_prepare(cmd); switch (cmd) { +Binary files itrustee_tzdriver/.git/index and itrustee_tzdriver_new/.git/index differ diff -uprN itrustee_tzdriver/Makefile itrustee_tzdriver_new/Makefile ---- itrustee_tzdriver/Makefile 2024-07-12 11:08:59.353629380 +0800 -+++ itrustee_tzdriver_new/Makefile 2024-07-12 10:51:32.641629380 +0800 +--- itrustee_tzdriver/Makefile 2024-09-25 14:59:38.671977710 +0800 ++++ itrustee_tzdriver_new/Makefile 2024-09-25 14:56:17.511977710 +0800 @@ -54,6 +54,7 @@ EXTRA_CFLAGS += -DCONFIG_CPU_AFF_NR=0 -D EXTRA_CFLAGS += -DCONFIG_TEE_LOG_ACHIVE_PATH=\"/var/log/tee/last_teemsg\" EXTRA_CFLAGS += -DNOT_TRIGGER_AP_RESET -DLAST_TEE_MSG_ROOT_GID -DCONFIG_NOCOPY_SHAREDMEM -DCONFIG_REGISTER_SHAREDMEM -DCONFIG_TA_AFFINITY=y -DCONFIG_TA_AFFINITY_CPU_NUMS=128 @@ -1067,14 +1069,9 @@ diff -uprN itrustee_tzdriver/Makefile itrustee_tzdriver_new/Makefile EXTRA_CFLAGS += -DCONFIG_AUTH_SUPPORT_UNAME -DCONFIG_AUTH_HASH -std=gnu99 EXTRA_CFLAGS += -DCONFIG_TEE_UPGRADE -DCONFIG_TEE_REBOOT -DCONFIG_CONFIDENTIAL_TEE EXTRA_CFLAGS += -I$(PWD)/tzdriver_internal/tee_reboot -diff -uprN itrustee_tzdriver/modules.order itrustee_tzdriver_new/modules.order ---- itrustee_tzdriver/modules.order 1970-01-01 08:00:00.000000000 +0800 -+++ itrustee_tzdriver_new/modules.order 2024-07-12 10:54:21.957629380 +0800 -@@ -0,0 +1 @@ -+kernel//home/z50040113/target/itrustee_tzdriver/tzdriver.ko diff -uprN itrustee_tzdriver/tc_ns_client.h itrustee_tzdriver_new/tc_ns_client.h ---- itrustee_tzdriver/tc_ns_client.h 2024-07-12 11:08:59.361629380 +0800 -+++ itrustee_tzdriver_new/tc_ns_client.h 2024-07-12 10:51:32.625629380 +0800 +--- itrustee_tzdriver/tc_ns_client.h 2024-09-25 14:59:38.675977710 +0800 ++++ itrustee_tzdriver_new/tc_ns_client.h 2024-09-25 14:56:17.511977710 +0800 @@ -230,6 +230,8 @@ struct tc_ns_log_pool { #define TC_NS_CLIENT_IOCTL_GET_TEE_INFO \ _IOWR(TC_NS_CLIENT_IOC_MAGIC, 26, struct tc_ns_tee_info) @@ -1085,8 +1082,8 @@ diff -uprN itrustee_tzdriver/tc_ns_client.h itrustee_tzdriver_new/tc_ns_client.h _IOWR(TC_NS_CLIENT_IOC_MAGIC, 32, unsigned int) diff -uprN itrustee_tzdriver/teek_ns_client.h itrustee_tzdriver_new/teek_ns_client.h ---- itrustee_tzdriver/teek_ns_client.h 2024-07-12 11:08:59.361629380 +0800 -+++ itrustee_tzdriver_new/teek_ns_client.h 2024-07-12 10:51:32.597629380 +0800 +--- itrustee_tzdriver/teek_ns_client.h 2024-09-25 14:59:38.675977710 +0800 ++++ itrustee_tzdriver_new/teek_ns_client.h 2024-09-25 14:56:17.515977710 +0800 @@ -129,6 +129,9 @@ struct tc_ns_dev_file { int load_app_flag; #ifdef CONFIG_CONFIDENTIAL_CONTAINER @@ -1098,8 +1095,8 @@ diff -uprN itrustee_tzdriver/teek_ns_client.h itrustee_tzdriver_new/teek_ns_clie struct completion close_comp; /* for kthread close unclosed session */ #ifdef CONFIG_TEE_TELEPORT_SUPPORT diff -uprN itrustee_tzdriver/tlogger/tlogger.c itrustee_tzdriver_new/tlogger/tlogger.c ---- itrustee_tzdriver/tlogger/tlogger.c 2024-07-12 11:08:59.361629380 +0800 -+++ itrustee_tzdriver_new/tlogger/tlogger.c 2024-07-12 10:51:32.581629380 +0800 +--- itrustee_tzdriver/tlogger/tlogger.c 2024-09-25 14:59:38.675977710 +0800 ++++ itrustee_tzdriver_new/tlogger/tlogger.c 2024-09-25 14:56:17.511977710 +0800 @@ -61,6 +61,7 @@ #define SET_TLOGCAT_STAT_BASE 7 #define GET_TLOGCAT_STAT_BASE 8 -- Gitee From d4bb4d3cdf0a3b23920f7393b42469d6b2c9cdee Mon Sep 17 00:00:00 2001 From: chenzheng Date: Tue, 24 Sep 2024 20:34:12 +0800 Subject: [PATCH 2/4] add debug log and bugfix 1. virtio_console: change ioctl to read_iter 2. vtz_proxy: destroy vm, agent and work thread should close. --- .../Host/itrustee_tzdriver.patch | 6 +- trustzone-awared-vm/Host/vtzb_proxy/agent.c | 52 +++-- trustzone-awared-vm/Host/vtzb_proxy/agent.h | 1 + .../Host/vtzb_proxy/include/tee_sys_log.h | 33 ++-- .../Host/vtzb_proxy/process_data.c | 20 +- .../Host/vtzb_proxy/process_data.h | 7 + .../Host/vtzb_proxy/serial_port.c | 54 +++-- .../Host/vtzb_proxy/serial_port.h | 5 +- .../Host/vtzb_proxy/thread_pool.c | 15 +- .../Host/vtzb_proxy/thread_pool.h | 2 +- trustzone-awared-vm/Host/vtzb_proxy/tlogcat.c | 2 +- trustzone-awared-vm/Host/vtzb_proxy/vm.c | 102 +++++++--- trustzone-awared-vm/Host/vtzb_proxy/vm.h | 5 +- .../Host/vtzb_proxy/vtzb_proxy.c | 80 +++++--- .../VM/virtio/char/virtio_console.c | 186 +++--------------- trustzone-awared-vm/VM/vtzdriver/serialport.c | 60 ++---- trustzone-awared-vm/VM/vtzdriver/vtzf.c | 5 +- 17 files changed, 302 insertions(+), 333 deletions(-) diff --git a/trustzone-awared-vm/Host/itrustee_tzdriver.patch b/trustzone-awared-vm/Host/itrustee_tzdriver.patch index 2023a5c..09b073b 100644 --- a/trustzone-awared-vm/Host/itrustee_tzdriver.patch +++ b/trustzone-awared-vm/Host/itrustee_tzdriver.patch @@ -562,11 +562,11 @@ diff -uprN itrustee_tzdriver/core/gp_ops.c itrustee_tzdriver_new/core/gp_ops.c } #endif - release_shared_mem_page(temp_buf, temp_buf_sz); -+ if (call_params->dev->isVM) { ++ if (call_params->dev->isVM) + release_vm_shared_mem_page(temp_buf, temp_buf_sz, call_params->dev->vm_page_size); -+ } else { ++ else + release_shared_mem_page(temp_buf, temp_buf_sz); -+ } ++ mailbox_free(temp_buf); } #endif diff --git a/trustzone-awared-vm/Host/vtzb_proxy/agent.c b/trustzone-awared-vm/Host/vtzb_proxy/agent.c index d508f72..5f67c75 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/agent.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/agent.c @@ -20,37 +20,51 @@ extern ThreadPool g_pool; +void do_free_agent(struct_agent_args *agent_args) +{ + int ret = -1; + + if (agent_args == NULL) { + tloge("agent args is null\n"); + return; + } + + tlogv("free agent fd %u\n", agent_args->dev_fd); + ListRemoveEntry(&(agent_args->node)); + ret = ioctl(agent_args->dev_fd, TC_NS_CLIENT_IOCTL_UNREGISTER_AGENT, agent_args->args.id); + if (ret) { + tloge("ioctl failed\n"); + } + close(agent_args->dev_fd); + agent_args->dev_fd = -1; + + if (agent_args->thd!= 0) { + thread_pool_submit(&g_pool, Kill_useless_thread, (void *)(agent_args->thd)); + } + pthread_spin_destroy(&agent_args->spinlock); + free(agent_args); +} + void free_agent_buf(int ptzfd, struct vm_file *vm_fp) { - int ret; struct ListNode *ptr = NULL; struct ListNode *n = NULL; - struct_agent_args *agent_args = NULL; - unsigned long buf[2]; - if (!vm_fp) + if (!vm_fp) { + tloge("vm file is NULL\n"); return; + } pthread_mutex_lock(&vm_fp->agents_lock); - if (LIST_EMPTY(&vm_fp->agents_head)) + if (LIST_EMPTY(&vm_fp->agents_head)) { + // when teecd init, this is possible + tlogd("agent list is empty\n"); goto END; + } LIST_FOR_EACH_SAFE(ptr, n, &vm_fp->agents_head) { struct_agent_args *tmp = CONTAINER_OF(ptr, struct_agent_args, node); if (tmp->dev_fd == ptzfd) { - ListRemoveEntry(&(tmp->node)); - agent_args = tmp; - if (agent_args) { - buf[0] = agent_args->args.id; - ret = ioctl(ptzfd, TC_NS_CLIENT_IOCTL_UNREGISTER_AGENT, buf); - if (ret) { - tloge("ioctl failed\n"); - } - if (agent_args->thd!= 0) { - thread_pool_submit(&g_pool, Kill_useless_thread, (void *)(agent_args->thd)); - } - pthread_spin_destroy(&agent_args->spinlock); - free(agent_args); - } + do_free_agent(tmp); } } END: diff --git a/trustzone-awared-vm/Host/vtzb_proxy/agent.h b/trustzone-awared-vm/Host/vtzb_proxy/agent.h index f7ee9b6..3064375 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/agent.h +++ b/trustzone-awared-vm/Host/vtzb_proxy/agent.h @@ -16,6 +16,7 @@ typedef struct { pthread_t thd; } struct_agent_args; +void do_free_agent(struct_agent_args *agent_args); void free_agent_buf(int ptzfd, struct vm_file *vm_fp); void register_agent(struct_packet_cmd_regagent *packet_cmd, struct serial_port_file *serial_port); void wait_event(struct_packet_cmd_event *packet_cmd, struct serial_port_file *serial_port); diff --git a/trustzone-awared-vm/Host/vtzb_proxy/include/tee_sys_log.h b/trustzone-awared-vm/Host/vtzb_proxy/include/tee_sys_log.h index 1fa0c3d..ffdb797 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/include/tee_sys_log.h +++ b/trustzone-awared-vm/Host/vtzb_proxy/include/tee_sys_log.h @@ -14,44 +14,45 @@ #define TEEC_SYS_LOG_H #include - -// #define TEE_LOG_MASK TZ_LOG_ERROR -#define TEE_LOG_MASK TZ_LOG_VERBOSE +#include +#include +#define TEE_LOG_MASK TZ_LOG_INFO #define TZ_LOG_VERBOSE 0 -#define TZ_LOG_INFO 1 -#define TZ_LOG_WARN 2 -#define TZ_LOG_DEBUG 3 +#define TZ_LOG_DEBUG 1 +#define TZ_LOG_INFO 2 +#define TZ_LOG_WARN 3 #define TZ_LOG_ERROR 4 -#define tlogv(...) \ +#define log_format(lev, fmt, ...) syslog(lev, "[%lu][%s] " fmt, (unsigned long)syscall(SYS_gettid), __func__, ##__VA_ARGS__) +#define tlogv(fmt, ...) \ do { \ if (TZ_LOG_VERBOSE == TEE_LOG_MASK) \ - syslog(LOG_USER | LOG_NOTICE, __VA_ARGS__); \ + log_format(LOG_USER | LOG_NOTICE, fmt, ##__VA_ARGS__); \ } while (0) -#define tlogd(...) \ +#define tlogd(fmt, ...) \ do { \ if (TZ_LOG_DEBUG >= TEE_LOG_MASK) \ - syslog(LOG_USER | LOG_DEBUG, __VA_ARGS__); \ + log_format(LOG_USER | LOG_DEBUG, fmt, ##__VA_ARGS__); \ } while (0) -#define tlogi(...) \ +#define tlogi(fmt, ...) \ do { \ if (TZ_LOG_INFO >= TEE_LOG_MASK) \ - syslog(LOG_USER | LOG_INFO, __VA_ARGS__); \ + log_format(LOG_USER | LOG_INFO, fmt, ##__VA_ARGS__); \ } while (0) -#define tlogw(...) \ +#define tlogw(fmt, ...) \ do { \ if (TZ_LOG_WARN >= TEE_LOG_MASK) \ - syslog(LOG_USER | LOG_WARNING, __VA_ARGS__); \ + log_format(LOG_USER | LOG_WARNING, fmt, ##__VA_ARGS__); \ } while (0) -#define tloge(...) \ +#define tloge(fmt, ...) \ do { \ if (TZ_LOG_ERROR >= TEE_LOG_MASK) \ - syslog(LOG_USER | LOG_ERR, __VA_ARGS__); \ + log_format(LOG_USER | LOG_ERR, fmt, ##__VA_ARGS__); \ } while (0) #endif diff --git a/trustzone-awared-vm/Host/vtzb_proxy/process_data.c b/trustzone-awared-vm/Host/vtzb_proxy/process_data.c index 734edd3..a224b4c 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/process_data.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/process_data.c @@ -6,22 +6,27 @@ #include "securec.h" #include "vm.h" -static void *malloc_copy(void *buf, int buf_len , int size, int *poffset) +static void *malloc_copy(void *buf, int buf_len, int size, int *poffset) { void *res; int offset = *poffset; - if (buf_len < offset + size || size < 4) { - memmove_s(buf, buf_len, buf + offset, buf_len - offset); + // packet date not write complete, just move date to front + if (buf_len < offset + size || size < (int)sizeof(uint32_t)) { + if (offset) { + memmove_s(buf, buf_len, buf + offset, buf_len - offset); + } *poffset = buf_len - offset; return NULL; } - res = malloc(size + sizeof(uint64_t)); + res = malloc(size + sizeof(vm_trace_data)); if (!res) { tloge("failed malloc\n"); return NULL; } - if (memcpy_s(res + sizeof(uint64_t), size, buf + offset, size)) { - tloge("memcpy_s err\n"); + if (memcpy_s(res + sizeof(vm_trace_data), size, buf + offset, size)) { + tloge("memcpy_s failed, just skip this packet\n"); + free(res); + res = NULL; } *poffset = offset + size; return res; @@ -31,12 +36,13 @@ void *get_packet_item(void *buf, int buf_len, int *poffset) { uint32_t packet_size; void *res = NULL; + // packet data deal over, offset reset zero if (buf_len == *poffset) { *poffset = 0; return NULL; } - if (buf_len < *poffset + (int)sizeof(int)) { + if (buf_len < *poffset + (int)sizeof(uint32_t)) { return malloc_copy(buf, buf_len, buf_len - *poffset, poffset); } packet_size = *(uint32_t *)(buf + *poffset); diff --git a/trustzone-awared-vm/Host/vtzb_proxy/process_data.h b/trustzone-awared-vm/Host/vtzb_proxy/process_data.h index 119eac2..2cffa48 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/process_data.h +++ b/trustzone-awared-vm/Host/vtzb_proxy/process_data.h @@ -12,6 +12,13 @@ #ifndef PROCESS_DATA_H #define PROCESS_DATA_H +#include + +typedef struct { + uint64_t serial_port_ptr; + uint32_t vmid; +} vm_trace_data; + void *get_packet_item(void *buf, int buf_len, int *poffset); #endif \ No newline at end of file diff --git a/trustzone-awared-vm/Host/vtzb_proxy/serial_port.c b/trustzone-awared-vm/Host/vtzb_proxy/serial_port.c index 1187d25..9c68b94 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/serial_port.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/serial_port.c @@ -20,7 +20,6 @@ struct serial_port_list g_serial_list; struct pollfd g_pollfd[SERIAL_PORT_NUM]; -int g_pollfd_len = 0; struct timeval g_last_time, g_cur_time; struct serial_port_file *g_serial_array[SERIAL_PORT_NUM]; @@ -32,8 +31,7 @@ int serial_port_list_init() gettimeofday(&g_cur_time, NULL); pthread_mutex_init(&g_serial_list.lock, NULL); ListInit(&g_serial_list.head); - for ( i = 0; i < SERIAL_PORT_NUM; i++) - { + for (i = 0; i < SERIAL_PORT_NUM; i++) { serial_port = (struct serial_port_file *)malloc(sizeof(struct serial_port_file)); if (!serial_port) { tloge("Failed to allocate memory for serial_port\n"); @@ -45,6 +43,7 @@ int serial_port_list_init() serial_port->offset = 0; serial_port->rd_buf = (char *)malloc(BUF_LEN_MAX_RD); serial_port->vm_file = NULL; + serial_port->index = i; g_pollfd[i].fd = -1; if (!serial_port->rd_buf) { tloge("Failed to allocate memory for rd_buf\n"); @@ -76,6 +75,7 @@ void serial_port_list_destroy() } ListRemoveEntry(&serial_port->head); (void)pthread_mutex_destroy(&serial_port->lock); + release_vm_file(serial_port, serial_port->index); free(serial_port); } (void)pthread_mutex_unlock(&g_serial_list.lock); @@ -118,12 +118,37 @@ static int connect_domsock_chardev(char *dev_path, int *sock) if (memcpy_s(&sock_addr.sun_path, sizeof(sock_addr.sun_path), dev_path, sizeof(sock_addr.sun_path))) { tloge("memcpy_s err\n"); + goto CLOSE; } ret = connect(*sock, (struct sockaddr *)&sock_addr, sizeof(sock_addr)); if (ret < 0) { tloge("connect domain socket %s failed \n", dev_path); + goto CLOSE; } return ret; + +CLOSE: + close(*sock); + *sock = -1; + return ret; +} + +void release_vm_file(struct serial_port_file *serial_port, int i) +{ + if (!serial_port) { + tloge("vm %d 's serial_port is null\n", i); + return; + } + close(serial_port->sock); + serial_port->sock = -1; + g_pollfd[i].fd = -1; + g_serial_array[i] = NULL; + serial_port->opened = false; + serial_port->offset = 0; + if (serial_port->vm_file) { + destroy_vm_file(serial_port->vm_file); + } + serial_port->vm_file = NULL; } static void do_check_stat_serial_port() @@ -132,7 +157,7 @@ static void do_check_stat_serial_port() int i = 0; struct serial_port_file *serial_port; (void)pthread_mutex_lock(&g_serial_list.lock); - LIST_FOR_EACH_ENTRY(serial_port, &g_serial_list.head, head){ + LIST_FOR_EACH_ENTRY(serial_port, &g_serial_list.head, head) { if (serial_port->opened == false) { ret = access(serial_port->path, R_OK | W_OK); if (ret == 0) { @@ -145,20 +170,14 @@ static void do_check_stat_serial_port() g_serial_array[i] = serial_port; g_pollfd[i].fd = serial_port->sock; g_pollfd[i].events = POLLIN; + tlogd("vm %d started, connect fd %d\n", i, serial_port->sock); } } } else { ret = access(serial_port->path, R_OK | W_OK); if (ret) { - close(serial_port->sock); - serial_port->sock = -1; - g_serial_array[i] = NULL; - g_pollfd[i].fd = -1; - serial_port->opened = false; - if (serial_port->vm_file) { - destroy_vm_file(serial_port->vm_file); - } - serial_port->vm_file = NULL; + tlogd("vm %d closed, fd %d is invalid, should close\n", i, serial_port->sock); + release_vm_file(serial_port, i); } } i++; @@ -182,18 +201,17 @@ static int clean_dirty_data() struct timeval start, end; void *tmp_buf; (void)ret; - if (!g_pollfd_len) - return 0; tmp_buf = malloc(BUF_LEN_MAX_RD); if (!tmp_buf) return -ENOMEM; gettimeofday(&start, NULL); gettimeofday(&end, NULL); - while(end.tv_sec - start.tv_sec < 1) { - ret = safepoll(g_pollfd, g_pollfd_len, 0); - for (i = 0; i < g_pollfd_len; i++) { + while (end.tv_sec - start.tv_sec < 1) { + ret = safepoll(g_pollfd, SERIAL_PORT_NUM, 0); + for (i = 0; i < SERIAL_PORT_NUM; i++) { if (g_pollfd[i].revents & POLLIN) { ret = read(g_pollfd[i].fd, tmp_buf, BUF_LEN_MAX_RD); + tlogd("clean vm %d dirty data %d\n", i, ret); } } gettimeofday(&end, NULL); diff --git a/trustzone-awared-vm/Host/vtzb_proxy/serial_port.h b/trustzone-awared-vm/Host/vtzb_proxy/serial_port.h index 9a74d39..2d1222e 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/serial_port.h +++ b/trustzone-awared-vm/Host/vtzb_proxy/serial_port.h @@ -20,7 +20,7 @@ #include "vm.h" #define VTZB_CHAR_DEV "/tmp/vm_vtzb_sock" -#define SERIAL_PORT_NUM 15 +#define SERIAL_PORT_NUM 33 #define BUF_LEN_MAX_RD 1024 *512 #define UNIX_PATH_MAX 108 #define CHECK_TIME_SEC 2 @@ -50,4 +50,5 @@ void *get_rd_buf(int serial_port_fd); void *get_serial_port_file(int serial_port_fd); void check_stat_serial_port(); int check_stat_serial_port_first(); -#endif \ No newline at end of file +void release_vm_file(struct serial_port_file *serial_port, int i); +#endif diff --git a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c index 9b399d4..0048859 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c @@ -15,6 +15,7 @@ TimeOut g_time_out[THREAD_POOL_SIZE]; /* Custom signal handler for killing zombie threads. */ void signal_handler(int signum) { (void)signum; + tlogd("thread %lu got sig exited\n", pthread_self()); pthread_exit(NULL); } @@ -54,9 +55,11 @@ void replenish_thread_pool(ThreadPool *pool, pthread_t thd) pthread_create(&pool->threads[i], NULL, thread_func, &g_thd_args[i]); pthread_detach(pool->threads[i]); pool->kill_flag[i] = false; - break; + tlogv("old id %lu, new id %lu\n", thd, pool->threads[i]); + return; } } + tloge("can't found the killed thread %lu\n", thd); } /* Thread function */ @@ -82,7 +85,7 @@ void *thread_func(void *arg) pthread_mutex_unlock(&pool->task_mutex); break; } - + if (pool->kill_flag[index]) { pthread_cond_signal(&pool->queue_not_empty); pthread_mutex_unlock(&pool->task_mutex); @@ -98,6 +101,7 @@ void *thread_func(void *arg) pthread_mutex_lock(&pool->busy_mutex); pool->busy_cnt++; + tlogv("start work, thread cnt: %d, task cnt: %d\n", pool->busy_cnt, pool->task_cnt); if (pool->task_args[index]) free(pool->task_args[index]); pool->task_args[index] = task.arg; @@ -108,6 +112,7 @@ void *thread_func(void *arg) pthread_mutex_lock(&pool->busy_mutex); pool->busy_cnt--; pool->task_args[index] = NULL; + tlogv("end work, thread cnt: %d, task cnt: %d\n", pool->busy_cnt, pool->task_cnt); pthread_mutex_unlock(&pool->busy_mutex); } @@ -120,15 +125,14 @@ void *admin_thread(void *arg) ThreadPool *pool = (ThreadPool *)arg; struct timeval cur_time; long time_sec = 0; - while (!pool->destroying) - { + while (!pool->destroying) { sleep(DEFAULT_TIME_SEC); gettimeofday(&cur_time, NULL); time_sec = cur_time.tv_sec; pthread_mutex_lock(&pool->time_mutex); for (i = 0; i < THREAD_POOL_SIZE; i++) { if (g_time_out[i].flag != 0 && (time_sec - g_time_out[i].start_time) > DEFAULT_TIME_SEC) { - kill_open_session_thd(g_time_out[i]); + kill_open_session_thd(&g_time_out[i]); g_time_out[i].flag = 0; } } @@ -158,6 +162,7 @@ void thread_pool_submit(ThreadPool *pool, void *(*task_func)(void *), void *arg) pool->task_queue[pool->rear].arg = arg; pool->rear = (pool->rear + 1) % TASK_QUEUE_SIZE; pool->task_cnt++; + tlogv("add task to task queue cnt: %d\n", pool->task_cnt); /* Notify waiting threads of a new task. */ pthread_cond_signal(&pool->queue_not_empty); diff --git a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h index 18b7aeb..0603d01 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h +++ b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h @@ -64,4 +64,4 @@ void thread_pool_submit(ThreadPool *pool, void *(*task_func)(void *), void *arg) void replenish_thread_pool(ThreadPool *pool, pthread_t thd); void set_thread_session_id(ThreadPool *pool, pthread_t thd, unsigned int id); unsigned int get_thread_session_id(ThreadPool *pool, pthread_t thd, unsigned int session_id); -#endif \ No newline at end of file +#endif diff --git a/trustzone-awared-vm/Host/vtzb_proxy/tlogcat.c b/trustzone-awared-vm/Host/vtzb_proxy/tlogcat.c index 50fd311..b899115 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/tlogcat.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/tlogcat.c @@ -92,7 +92,7 @@ static void tlog_get_log(struct_packet_cmd_get_log *packet_cmd, tv.tv_usec = 0; FD_ZERO(&readset); FD_SET(packet_cmd->ptzfd, &readset); - tlogd("while select\n"); + tlogv("while select\n"); result = select((packet_cmd->ptzfd + 1), &readset, NULL, NULL, &tv); } while (result == -1 && errno == EINTR); if (result <= 0) { diff --git a/trustzone-awared-vm/Host/vtzb_proxy/vm.c b/trustzone-awared-vm/Host/vtzb_proxy/vm.c index 41c0ab4..1fca1c3 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/vm.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/vm.c @@ -1,5 +1,6 @@ #include "errno.h" #include "vm.h" +#include "agent.h" #include "thread_pool.h" #include "comm_structs.h" #include "serial_port.h" @@ -15,11 +16,15 @@ void add_session_list(int ptzfd, struct vm_file *vm_fp, TC_NS_ClientContext *cli struct fd_file *fd_p = NULL; struct session *sessionp = NULL; fd_p = find_fd_file(ptzfd, vm_fp); - if (!fd_p) + if (!fd_p) { + tloge("found the fd %d 's fd_file failed\n", ptzfd); return; + } sessionp = (struct session *)malloc(sizeof(struct session)); - if (!sessionp) + if (!sessionp) { + tloge("malloc session for fd %d failed\n", ptzfd); return; + } sessionp->session_id = clicontext->session_id; ListInit(&sessionp->head); pthread_mutex_lock(&fd_p->session_lock); @@ -30,8 +35,12 @@ void add_session_list(int ptzfd, struct vm_file *vm_fp, TC_NS_ClientContext *cli void *Kill_useless_thread(void *args) { pthread_t tid = (pthread_t)args; + tlogv("try to kill thread %lu\n", tid); int result = pthread_kill(tid, SIGUSR1); if (result == 0) { + pthread_mutex_lock(&g_pool.busy_mutex); + g_pool.busy_cnt--; + pthread_mutex_unlock(&g_pool.busy_mutex); replenish_thread_pool(&g_pool, tid); } return NULL; @@ -39,8 +48,10 @@ void *Kill_useless_thread(void *args) static void try_kill_thread(struct session *sp) { - if (!sp) + if (!sp) { + tloge("session is null\n"); return; + } if (sp->thread_id != 0 && get_thread_session_id(&g_pool, sp->thread_id, sp->session_id)) { thread_pool_submit(&g_pool, Kill_useless_thread, (void *)(sp->thread_id)); } @@ -50,8 +61,10 @@ static void do_remove_session(unsigned int session_id, struct fd_file *fd_p) { struct ListNode *ptr = NULL; struct ListNode *n = NULL; - if (!fd_p) + if (!fd_p) { + tloge("fd_file is null\n"); return ; + } pthread_mutex_lock(&fd_p->session_lock); if (!LIST_EMPTY(&fd_p->session_head)) { LIST_FOR_EACH_SAFE(ptr, n, &fd_p->session_head) { @@ -69,12 +82,16 @@ static void do_remove_session(unsigned int session_id, struct fd_file *fd_p) void remove_session(int ptzfd, int session_id, struct vm_file *vm_fp) { struct fd_file *fd_p = NULL; - if (!vm_fp) + if (!vm_fp) { + tloge("vm_file is null\n"); return; + } fd_p = find_fd_file(ptzfd, vm_fp); - if (fd_p) { - do_remove_session(session_id, fd_p); + if (!fd_p) { + tloge("found the fd %d 's fd_file failed\n", ptzfd); + return; } + do_remove_session(session_id, fd_p); } struct fd_file *find_fd_file(int ptzfd, struct vm_file *vm_fp) @@ -82,8 +99,10 @@ struct fd_file *find_fd_file(int ptzfd, struct vm_file *vm_fp) struct ListNode *ptr = NULL; struct fd_file *fd_p = NULL; int bfind = 0; - if (!vm_fp) + if (!vm_fp) { + tloge("vm_file is null\n"); return NULL; + } pthread_mutex_lock(&vm_fp->fd_lock); if (!LIST_EMPTY(&vm_fp->fds_head)) { LIST_FOR_EACH(ptr, &vm_fp->fds_head) { @@ -97,18 +116,24 @@ struct fd_file *find_fd_file(int ptzfd, struct vm_file *vm_fp) pthread_mutex_unlock(&vm_fp->fd_lock); if (bfind) return fd_p; + return NULL; } -void add_fd_list(int fd, struct vm_file *vm_fp) +void add_fd_list(int fd, uint32_t fd_type, struct vm_file *vm_fp) { struct fd_file *fd_p; - if (!vm_fp) + if (!vm_fp) { + tloge("add fd_file failed, vm_fp is NULL\n"); return; + } fd_p = (struct fd_file *)malloc(sizeof(struct fd_file)); - if (!fd_p) + if (!fd_p) { + tloge("malloc fd_file failed\n"); return; + } fd_p->ptzfd = fd; + fd_p->fd_type = fd_type; pthread_mutex_init(&fd_p->session_lock, NULL); ListInit(&fd_p->session_head); ListInit(&fd_p->head); @@ -124,8 +149,12 @@ static void do_remove_fd(struct fd_file *fd_p) struct ListNode *n = NULL; unsigned int session_id; (void)session_id; - if (!fd_p) + if (!fd_p) { + tloge("fd_file is NULL\n"); return; + } + + tlogv("remove fd %u\n", fd_p->ptzfd); pthread_mutex_lock(&fd_p->session_lock); if (!LIST_EMPTY(&fd_p->session_head)) { LIST_FOR_EACH_SAFE(ptr, n, &fd_p->session_head) { @@ -136,22 +165,31 @@ static void do_remove_fd(struct fd_file *fd_p) } } pthread_mutex_unlock(&fd_p->session_lock); + //if (fd_p->fd_type != TC_PRIVATE_DEV_FLAG) { + close(fd_p->ptzfd); + fd_p->ptzfd = -1; + //} } + int remove_fd(int ptzfd, struct vm_file *vm_fp) { - if (!vm_fp) + if (!vm_fp) { + tloge("vm_file is null\n"); return -EINVAL; + } struct fd_file *fd_p = find_fd_file(ptzfd, vm_fp); - if (fd_p) { - pthread_mutex_lock(&vm_fp->fd_lock); - ListRemoveEntry(&fd_p->head); - pthread_mutex_unlock(&vm_fp->fd_lock); - do_remove_fd(fd_p); - free(fd_p); - return 0; + if (!fd_p) { + tloge("found the fd %d 's fd_file failed\n", ptzfd); + return -EBADF; } - return -EINVAL; + + pthread_mutex_lock(&vm_fp->fd_lock); + ListRemoveEntry(&fd_p->head); + pthread_mutex_unlock(&vm_fp->fd_lock); + do_remove_fd(fd_p); + free(fd_p); + return 0; } struct vm_file *create_vm_file(uint32_t vmid) @@ -171,6 +209,7 @@ struct vm_file *create_vm_file(uint32_t vmid) } if (!isfind) { + tlogd("create new vm_file for vmid %d\n", vmid); tmp = (struct vm_file *)malloc(sizeof(struct vm_file)); if (!tmp) { tloge("Failed to allocate memory for vm_file\n"); @@ -199,6 +238,18 @@ int destroy_vm_file(struct vm_file *vm_file) struct fd_file *fd_p = NULL; if (!vm_file) return 0; + + // release agent in vm + pthread_mutex_lock(&vm_file->agents_lock); + if (!LIST_EMPTY(&vm_file->agents_head)) { + LIST_FOR_EACH_SAFE(ptr, n, &vm_file->agents_head) { + struct_agent_args *tmp = CONTAINER_OF(ptr, struct_agent_args, node); + do_free_agent(tmp); + } + } + pthread_mutex_unlock(&vm_file->agents_lock); + + // release session in vm pthread_mutex_lock(&vm_file->fd_lock); if (!LIST_EMPTY(&vm_file->fds_head)) { LIST_FOR_EACH_SAFE(ptr, n, &vm_file->fds_head) { @@ -209,6 +260,7 @@ int destroy_vm_file(struct vm_file *vm_file) } } pthread_mutex_unlock(&vm_file->fd_lock); + pthread_mutex_lock(&g_mutex_vm); ListRemoveEntry(&(vm_file->head)); free(vm_file); @@ -216,15 +268,15 @@ int destroy_vm_file(struct vm_file *vm_file) return ret; } -void kill_open_session_thd(TimeOut t_out) +void kill_open_session_thd(TimeOut *t_out) { struct_packet_rsp_session packet_rsp; - pthread_t tid = t_out.tid; + pthread_t tid = t_out->tid; packet_rsp.packet_size = sizeof(packet_rsp); - packet_rsp.seq_num = t_out.seq_num + 1; + packet_rsp.seq_num = t_out->seq_num + 1; packet_rsp.ret = -1; thread_pool_submit(&g_pool, Kill_useless_thread, (void *)tid); - (void)send_to_vm(t_out.serial_port, &packet_rsp, sizeof(packet_rsp)); + (void)send_to_vm(t_out->serial_port, &packet_rsp, sizeof(packet_rsp)); } int set_start_time(pthread_t tid, int seq_num, diff --git a/trustzone-awared-vm/Host/vtzb_proxy/vm.h b/trustzone-awared-vm/Host/vtzb_proxy/vm.h index 77914d0..679cc37 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/vm.h +++ b/trustzone-awared-vm/Host/vtzb_proxy/vm.h @@ -34,6 +34,7 @@ struct vm_file { struct fd_file { int32_t ptzfd; + uint32_t fd_type; struct ListNode head; pthread_mutex_t session_lock; struct ListNode session_head; @@ -55,7 +56,7 @@ typedef struct { struct fd_file *find_fd_file(int ptzfd, struct vm_file *vm_fp); int remove_fd(int ptzfd, struct vm_file *vm_fp); -void add_fd_list(int fd, struct vm_file *vm_fp); +void add_fd_list(int fd, uint32_t fd_type, struct vm_file *vm_fp); void remove_session(int ptzfd, int session_id, struct vm_file *vm_fp); void add_session_list(int ptzfd, struct vm_file *vm_fp, TC_NS_ClientContext *clicontext); int destroy_vm_file(struct vm_file *vm_file); @@ -63,5 +64,5 @@ struct vm_file *create_vm_file(uint32_t vmid); void *Kill_useless_thread(void *args); int set_start_time(pthread_t tid, int seq_num, struct serial_port_file *serial_port); void remove_start_time(int i); -void kill_open_session_thd(TimeOut t_out); +void kill_open_session_thd(TimeOut *t_out); #endif \ No newline at end of file diff --git a/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.c b/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.c index 05d48e5..2155fcf 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.c @@ -31,7 +31,6 @@ #include "tlogcat.h" ThreadPool g_pool = {0}; -extern int g_pollfd_len; extern struct pollfd g_pollfd[SERIAL_PORT_NUM]; extern struct serial_port_file *g_serial_array[SERIAL_PORT_NUM]; @@ -45,6 +44,7 @@ static void open_tzdriver(struct_packet_cmd_open_tzd *packet_cmd, packet_rsp.seq_num = packet_cmd->seq_num + 1; packet_rsp.packet_size = sizeof(packet_rsp); packet_rsp.vmid = packet_cmd->vmid; + if (packet_cmd->flag == TLOG_DEV_THD_FLAG) { if (!serial_port->vm_file || !serial_port->vm_file->log_fd) { fd = open(TC_LOGGER_DEV_NAME, O_RDONLY); @@ -74,6 +74,7 @@ static void open_tzdriver(struct_packet_cmd_open_tzd *packet_cmd, ret = ioctl(fd, TC_NS_CLIENT_IOCTL_SET_VM_FLAG, packet_cmd->vmid); } + tlogv("vmid %d flag %d open tzdriver, fd %d\n", packet_cmd->vmid, packet_cmd->flag, fd); packet_rsp.ptzfd = fd; if (fd < 0) { tloge("open tee client dev failed, fd is %d\n", fd); @@ -90,7 +91,7 @@ END: } else { vm_fp = serial_port->vm_file; } - add_fd_list(fd, vm_fp); + add_fd_list(fd, packet_cmd->flag, vm_fp); if (packet_cmd->flag == TLOG_DEV_THD_FLAG) { vm_fp->log_fd = fd; } @@ -98,7 +99,6 @@ END: ret = send_to_vm(serial_port, &packet_rsp, sizeof(packet_rsp)); if (ret != sizeof(packet_rsp) && fd > 0) { remove_fd(fd, vm_fp); - (void)close(fd); } } @@ -111,14 +111,17 @@ static void close_tzdriver(struct_packet_cmd_close_tzd *packet_cmd, packet_rsp.packet_size = sizeof(packet_rsp); packet_rsp.ret = 0; (void)ret; - if (!serial_port->vm_file) + if (!serial_port->vm_file) { + tloge("serial_port->vm_file is null\n"); return; - - if (packet_cmd->ptzfd > 2) { - free_agent_buf(packet_cmd->ptzfd, serial_port->vm_file); - if (remove_fd(packet_cmd->ptzfd, serial_port->vm_file) == 0) - ret = close(packet_cmd->ptzfd); } + if (packet_cmd->ptzfd <= 2) { + tloge("invalid ptzfd %d\n", packet_cmd->ptzfd); + return; + } + + free_agent_buf(packet_cmd->ptzfd, serial_port->vm_file); + ret = remove_fd(packet_cmd->ptzfd, serial_port->vm_file); if (send_to_vm(serial_port, &packet_rsp, sizeof(packet_rsp)) != sizeof(packet_rsp)) tloge("close_tzdriver send to VM failed \n"); @@ -519,12 +522,16 @@ static void do_set_thread_id(struct fd_file *fd_p, unsigned int session_id, int static void set_thread_id(int ptzfd, unsigned int session_id, int flag, struct vm_file *vm_fp) { struct fd_file *fd_p; - if (!vm_fp) + if (!vm_fp) { + tloge("vm_file is null\n"); return; + } fd_p = find_fd_file(ptzfd, vm_fp); - if (fd_p) { - do_set_thread_id(fd_p, session_id, flag); + if (!fd_p) { + tloge("found the fd %d 's fd_file failed\n", ptzfd); + return; } + do_set_thread_id(fd_p, session_id, flag); } static void send_cmd(struct_packet_cmd_send_cmd *packet_cmd, @@ -673,15 +680,19 @@ static void vtz_nothing(struct_packet_cmd_nothing *packet_cmd, } } + void *thread_entry(void *args) { struct_packet_cmd_general *packet_general = NULL; uint32_t ui32_cmd = 0; - uint64_t u64 = *(uint64_t *)(args); - struct serial_port_file *serial_port = (struct serial_port_file *)u64; - char *rd_buf = (char *)(args) + sizeof(uint64_t); + vm_trace_data *data = (vm_trace_data *)args; + struct serial_port_file *serial_port = (struct serial_port_file *)data->serial_port_ptr; + char *rd_buf = (char *)(args) + sizeof(vm_trace_data); ui32_cmd = *(uint32_t *)(rd_buf + sizeof(uint32_t)); + struct_packet_cmd_nothing *p = (struct_packet_cmd_nothing *)rd_buf; + tlogd("vm %u cmd %u, size %u, seq %u\n", data->vmid, p->cmd, p->packet_size, p->seq_num); + if (ui32_cmd == VTZ_OPEN_TZD) { (void)open_tzdriver((struct_packet_cmd_open_tzd *)rd_buf, serial_port); goto END; @@ -695,11 +706,11 @@ void *thread_entry(void *args) packet_general = (struct_packet_cmd_general *)rd_buf; if (!serial_port || !packet_general || !find_fd_file(packet_general->ptzfd, serial_port->vm_file)) { + tloge("invalid params\n"); goto END; } - switch (ui32_cmd) - { + switch (ui32_cmd) { case VTZ_CLOSE_TZD: (void)close_tzdriver((struct_packet_cmd_close_tzd *)rd_buf, serial_port); break; @@ -753,6 +764,7 @@ void *thread_entry(void *args) (void)tlog(ui32_cmd, (void *)rd_buf, serial_port); break; default: + tloge("invalid cmd %d\n", ui32_cmd); break; } @@ -768,13 +780,13 @@ void proc_event(struct serial_port_file *serial_port) int offset = 0; int buf_len; int fd; - if (!serial_port || !serial_port->rd_buf || serial_port->sock <= 0){ + + if (!serial_port || !serial_port->rd_buf || serial_port->sock <= 0) { tloge("serial_port ptr or rd_buf is NULL!\n"); return; } fd = serial_port->sock; ret = read(fd, serial_port->rd_buf + serial_port->offset, BUF_LEN_MAX_RD - serial_port->offset); - if (ret < 0) { tloge("read domain socket failed \n"); return; @@ -782,12 +794,15 @@ void proc_event(struct serial_port_file *serial_port) if (ret == 0) return; buf_len = ret + serial_port->offset; - while(1) { + while (1) { void *packet = NULL; packet = get_packet_item(serial_port->rd_buf, buf_len, &offset); if (packet == NULL) break; - *(uint64_t *)(packet) = (uint64_t)serial_port; + + vm_trace_data *data = (vm_trace_data *)packet; + data->serial_port_ptr = (uint64_t)serial_port; + data->vmid = serial_port->index; thread_pool_submit(&g_pool, thread_entry, (void *)((uint64_t)packet)); } serial_port->offset = offset; @@ -805,28 +820,31 @@ int main() { while (1) { check_stat_serial_port(); ret = safepoll(g_pollfd, SERIAL_PORT_NUM, 20*1000); - if (ret == -1) { - tloge("pollfd failed, ret = %d \n", ret); - return -1; - } - if (ret == 0) { + if (ret <= 0) { + tlogv("pollfd no event or failed, ret = %d\n", ret); continue; } - + + tlogv("poll receive event %d\n", ret); for (i = 0; i < SERIAL_PORT_NUM; i++) { + if (g_pollfd[i].revents == 0) { + continue; + } + + tlogv("vm %d, event %x, fd %d\n", i, g_pollfd[i].revents, g_pollfd[i].fd); if (g_pollfd[i].revents & POLLIN) { proc_event(g_serial_array[i]); } if (g_pollfd[i].revents & POLLERR || g_pollfd[i].revents & POLLNVAL) { + tloge("vm %d got error event\n", i); continue; } if (g_pollfd[i].revents & POLLHUP) { - g_serial_array[i]->opened = false; - close(g_serial_array[i]->sock); - g_pollfd[i].fd = -1; + tloge("vm %d got POLLHUP event\n", i); + release_vm_file(g_serial_array[i], i); } } } @@ -836,4 +854,4 @@ END1: END2: serial_port_list_destroy(); return 0; -} \ No newline at end of file +} diff --git a/trustzone-awared-vm/VM/virtio/char/virtio_console.c b/trustzone-awared-vm/VM/virtio/char/virtio_console.c index ee9e9d3..52a1ec0 100644 --- a/trustzone-awared-vm/VM/virtio/char/virtio_console.c +++ b/trustzone-awared-vm/VM/virtio/char/virtio_console.c @@ -670,7 +670,8 @@ static ssize_t fill_readbuf(struct port *port, char __user *out_buf, buf = port->inbuf; out_count = min(out_count, buf->len - buf->offset); - if (to_user) { + (void)to_user; + if (access_ok(out_buf, out_count)) { ssize_t ret; ret = copy_to_user(out_buf, buf->buf + buf->offset, out_count); @@ -827,7 +828,12 @@ static ssize_t port_fops_write(struct file *filp, const char __user *ubuf, if (!buf) return -ENOMEM; - ret = copy_from_user(buf->buf, ubuf, count); + if (access_ok(ubuf, count)) { + ret = copy_from_user(buf->buf, ubuf, count); + } else { + memcpy(buf->buf, ubuf, count); + ret = 0; + } if (ret) { ret = -EFAULT; goto free_buf; @@ -853,156 +859,6 @@ out: return ret; } - -struct vtz_buf_struct{ - size_t buf_size; - void * buf; -}; - - -#define VTZ_IOC_MAGIC 'v' -#define TC_NS_CLIENT_IOCTL_READ_REQ \ - _IOWR(VTZ_IOC_MAGIC, 1, struct vtz_buf_struct) -#define TC_NS_CLIENT_IOCTL_WRITE_REQ \ - _IOWR(VTZ_IOC_MAGIC, 2, struct vtz_buf_struct) - -static int vtz_read_ioctl(struct file *filp, unsigned int cmd, struct vtz_buf_struct *vtz_buf) -{ - int ret = -EINVAL; - char *ubuf = vtz_buf->buf; - size_t count = vtz_buf->buf_size; - struct port *port; - - port = filp->private_data; - - /* Port is hot-unplugged. */ - if (!port->guest_connected) - return -ENODEV; - - if (!port_has_data(port)) { - /* - * If nothing's connected on the host just return 0 in - * case of list_empty; this tells the userspace app - * that there's no connection - */ - if (!port->host_connected) - return 0; - if (filp->f_flags & O_NONBLOCK) - return -EAGAIN; - ret = wait_event_freezable(port->waitqueue, - !will_read_block(port)); - if (ret < 0) - return ret; - } - /* Port got hot-unplugged while we were waiting above. */ - if (!port->guest_connected) - return -ENODEV; - /* - * We could've received a disconnection message while we were - * waiting for more data. - * - * This check is not clubbed in the if() statement above as we - * might receive some data as well as the host could get - * disconnected after we got woken up from our wait. So we - * really want to give off whatever data we have and only then - * check for host_connected. - */ - if (!port_has_data(port) && !port->host_connected) { - return 0; - } - - ret = fill_readbuf(port, ubuf, count, false); - return ret; -} - -static int vtz_write_ioctl(struct file *filp, unsigned int cmd, struct vtz_buf_struct *vtz_buf) -{ - int ret = -EINVAL; - char *ubuf = vtz_buf->buf; - size_t count = vtz_buf->buf_size; - - struct port *port; - struct port_buffer *buf; - bool nonblock; - struct scatterlist sg[1]; - /* Userspace could be out to fool us */ - if (!count) - return 0; - - port = filp->private_data; - - nonblock = filp->f_flags & O_NONBLOCK; - - ret = wait_port_writable(port, nonblock); - if (ret < 0) - return ret; - - count = min((size_t)(32 * 1024), count); - - buf = alloc_buf(port->portdev->vdev, count, 0); - if (!buf) - return -ENOMEM; - - memcpy(buf->buf, ubuf, count); - //ret = copy_from_user(buf->buf, ubuf, count); - //if (ret) { - // ret = -EFAULT; - // goto free_buf; - //} - - /* - * We now ask send_buf() to not spin for generic ports -- we - * can re-use the same code path that non-blocking file - * descriptors take for blocking file descriptors since the - * wait is already done and we're certain the write will go - * through to the host. - */ - nonblock = true; - sg_init_one(sg, buf->buf, count); - ret = __send_to_port(port, sg, 1, count, buf, nonblock); - - if (nonblock && ret > 0) - goto out; - -free_buf: - free_buf(buf, true); -out: - return ret; -} - -static long vtz_ioctl(struct file *file, unsigned int cmd, unsigned long arg) -{ - int ret = -EFAULT; - void *argp = (void *)(uintptr_t)arg; - struct vtz_buf_struct vtz_buf; - memcpy(&vtz_buf, argp, sizeof(vtz_buf)); - switch (cmd) { - case TC_NS_CLIENT_IOCTL_READ_REQ: - ret = vtz_read_ioctl(file, cmd, &vtz_buf); - break; - case TC_NS_CLIENT_IOCTL_WRITE_REQ: - ret = vtz_write_ioctl(file, cmd, &vtz_buf); - break; - default: - break; - } - return ret; -} - -#ifdef CONFIG_COMPAT -long vtz_compat_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) -{ - long ret; - - if (!file) - return -EINVAL; - - ret = vtz_ioctl(file, cmd, (unsigned long)(uintptr_t)compat_ptr(arg)); - return ret; -} -#endif - struct sg_list { unsigned int n; unsigned int size; @@ -1236,6 +1092,24 @@ static int port_fops_fasync(int fd, struct file *filp, int mode) return fasync_helper(fd, filp, mode, &port->async_queue); } + +static ssize_t port_fops_read_iter(struct kiocb *kiocbp, struct iov_iter *iovp) +{ + struct file *filp = kiocbp->ki_filp; + char *ubuf = iovp->iov->iov_base; + size_t count = iovp->iov->iov_len; + loff_t offp = 0; + return port_fops_read(filp, ubuf, count, &offp); +} + +static ssize_t port_fops_write_iter(struct kiocb *kiocbp, struct iov_iter *iovp) +{ + struct file *filp = kiocbp->ki_filp; + char *ubuf = iovp->iov->iov_base; + size_t count = iovp->iov->iov_len; + loff_t offp = 0; + return port_fops_write(filp, ubuf, count, &offp); +} /* * The file operations that we support: programs in the guest can open * a console device, read from it, write to it, poll for data and @@ -1245,17 +1119,13 @@ static int port_fops_fasync(int fd, struct file *filp, int mode) static const struct file_operations port_fops = { .owner = THIS_MODULE, .open = port_fops_open, - .read = port_fops_read, - .write = port_fops_write, + .read_iter = port_fops_read_iter, + .write_iter = port_fops_write_iter, .splice_write = port_fops_splice_write, .poll = port_fops_poll, .release = port_fops_release, .fasync = port_fops_fasync, .llseek = no_llseek, - .unlocked_ioctl = vtz_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = vtz_compat_ioctl, -#endif }; /* diff --git a/trustzone-awared-vm/VM/vtzdriver/serialport.c b/trustzone-awared-vm/VM/vtzdriver/serialport.c index 663af40..6650180 100644 --- a/trustzone-awared-vm/VM/vtzdriver/serialport.c +++ b/trustzone-awared-vm/VM/vtzdriver/serialport.c @@ -239,6 +239,7 @@ void put_event_data(void *packet, int packet_size, uint32_t seq_num) { struct vhc_event_data *event_data; struct vhc_event_data *tmp; + tlogd("put event: len %d, seq %d\n", packet_size, seq_num); if (!packet) return; spin_lock(&g_event_data_list.spinlock); @@ -266,10 +267,8 @@ int rd_thread_func(void *arg) uint32_t seq_num; int buf_len = 0; int offset = 0; -#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 0, 0) - struct vtz_buf_struct vtz_buf = {0}; -#endif struct file *fp_serialport = NULL; + fp_serialport = file->filep; while (!kthread_should_stop()) { ret = wait_event_interruptible(file->rd_wait_event_wq, file->rd_flag); @@ -280,15 +279,8 @@ int rd_thread_func(void *arg) if (g_destroy_rd_thread) break; off = 0; -#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 0, 0) - vtz_buf.buf = file->buffer + file->offset; - vtz_buf.buf_size = SERIAL_PORT_BUF_LEN - file->offset; - ssize_ret = fp_serialport->f_op->unlocked_ioctl(fp_serialport, - TC_NS_CLIENT_IOCTL_READ_REQ, (unsigned long)(&vtz_buf)); -#else ssize_ret = kernel_read(file->filep, file->buffer + file->offset, SERIAL_PORT_BUF_LEN - file->offset, &off); -#endif tlogd("kernel_read, ret value = %d, offset = %ld \n", (int)ssize_ret, (long)off); if (ssize_ret <= 0) { tloge("kernel_read failed, ret = %d \n", (int)ssize_ret); @@ -345,7 +337,6 @@ struct wr_data *get_wr_data(void) spin_unlock(&g_wr_data_list.spinlock); if (write_data && !write_data->wr_buf) { tloge("write_data->wr_buf NULL\n"); - return NULL; } return write_data; } @@ -362,27 +353,12 @@ void destroy_wr_data(struct wr_data *write_data) static int do_write(struct file *fp_serialport, void *buf, uint32_t buf_size) { int ret = 0; -#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 0, 0) - struct vtz_buf_struct vtz_buf = {0}; -#else loff_t off =0; -#endif if (!fp_serialport || !buf || buf_size > 32 * 1024) return -EINVAL; -#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 0, 0) - if (!fp_serialport->f_op->unlocked_ioctl) { - tloge("kernel version > 5.0.0, f_op->unlocked_ioctl is NULL, check virtio-console\n"); - return -EINVAL; - } - vtz_buf.buf = buf; - vtz_buf.buf_size = buf_size; - ret = fp_serialport->f_op->unlocked_ioctl(fp_serialport, - TC_NS_CLIENT_IOCTL_WRITE_REQ, (unsigned long)(&vtz_buf)); -#else ret = kernel_write(fp_serialport, buf, buf_size, &off); -#endif return ret < 0 ? ret : 0; } @@ -570,16 +546,6 @@ int create_thread(int pos, struct vtzf_serial_port_file *file) (void)snprintf(thread_name, 32, "vtz_wr_thread_%d", pos); file->wr_thread_name = thread_name; - thread_name = kzalloc(32, GFP_KERNEL); - if (!thread_name) { - tloge("Failed to allocate memory for thread name\n"); - kfree(file->rd_thread_name); - file->rd_thread_name = NULL; - return -ENOMEM; - } - (void)snprintf(thread_name, 32, "vtz_log_thread_%d", pos); - file->log_thread_name = thread_name; - tmp_thread = kthread_run(rd_thread_func, file, file->rd_thread_name); if (tmp_thread) { file->rd_thread = tmp_thread; @@ -598,6 +564,16 @@ int create_thread(int pos, struct vtzf_serial_port_file *file) return -EFAULT; } +#ifdef AUTO_LOG_THREAD + thread_name = kzalloc(32, GFP_KERNEL); + if (!thread_name) { + tloge("Failed to allocate memory for thread name\n"); + kfree(file->rd_thread_name); + file->rd_thread_name = NULL; + return -ENOMEM; + } + (void)snprintf(thread_name, 32, "vtz_log_thread_%d", pos); + file->log_thread_name = thread_name; tmp_thread = kthread_run(log_thread_func, file, file->log_thread_name); if (tmp_thread) { file->log_thread = tmp_thread; @@ -606,6 +582,7 @@ int create_thread(int pos, struct vtzf_serial_port_file *file) tloge("Failed to create kernel thread\n"); return -EFAULT; } +#endif return 0; } @@ -670,13 +647,6 @@ int serial_port_init(void) init_waitqueue_head(&(serial_port_file->log_wait_event_wq)); list_add_tail(&serial_port_file->head, &g_serial_port_list.head); g_serial_port_file = serial_port_file; -#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 0, 0) - if (!file->f_op->unlocked_ioctl) { - tloge("waring! file->f_op->unlocked_ioctl undefine!\n"); - ret = -EFAULT; - goto err; - } -#endif if (create_thread(i, serial_port_file)) goto err; } @@ -756,6 +726,8 @@ int send_to_proxy(void * wrt_buf, size_t size_wrt_buf, void * rd_buf, size_t siz int ret = -1; struct vhc_event_data *event_data; + tlogd("send data, wr_len %ld, rd_len %ld, seq %d\n", size_wrt_buf, size_rd_buf, seq_num); + ret = creat_wr_data(wrt_buf, size_wrt_buf); if (ret != 0) { tloge("creat_wr_data failed\n"); @@ -780,4 +752,4 @@ int send_to_proxy(void * wrt_buf, size_t size_wrt_buf, void * rd_buf, size_t siz err: return ret; -} \ No newline at end of file +} diff --git a/trustzone-awared-vm/VM/vtzdriver/vtzf.c b/trustzone-awared-vm/VM/vtzdriver/vtzf.c index 33719da..8c79e50 100644 --- a/trustzone-awared-vm/VM/vtzdriver/vtzf.c +++ b/trustzone-awared-vm/VM/vtzdriver/vtzf.c @@ -860,6 +860,7 @@ static int alloc_for_tmp_mem(struct tc_ns_client_context *clicontext, user_buf_size = (uintptr_t)(clicontext->params[index].memref.size_addr | (uint64_t)clicontext->params[index].memref.size_h_addr << ADDR_TRANS_NUM); + tlogv("buf_addr %lx, size_addr %lx\n", user_buf, user_buf_size); if (copy_from_user(&buf_size, (void *)user_buf_size, sizeof(uint32_t)) != 0) { tloge("copy from user failed\n"); return -EFAULT; @@ -895,6 +896,7 @@ static int alloc_for_val_mem(struct tc_ns_client_context *clicontext, | (uint64_t)clicontext->params[index].value.a_h_addr << ADDR_TRANS_NUM); user_val_b = (uintptr_t)(clicontext->params[index].value.b_addr | (uint64_t)clicontext->params[index].value.b_h_addr << ADDR_TRANS_NUM); + tlogv("a_val_addr %lx, b_val_addr %lx\n", user_val_a, user_val_b); if (copy_from_user(&val_a, (void *)user_val_a, sizeof(uint32_t)) != 0) { tloge("copy from user failed\n"); return -EFAULT; @@ -925,7 +927,8 @@ static int alloc_for_ref_mem(struct vtzf_dev_file *dev_file, | (uint64_t)clicontext->params[index].memref.size_h_addr << ADDR_TRANS_NUM); user_buffer = (void *)(clicontext->params[index].memref.buffer | (uint64_t)clicontext->params[index].memref.buffer_h_addr << ADDR_TRANS_NUM); - + + tlogv("buf_addr %p, size_addr %lx\n", user_buffer, user_size_addr); if (copy_from_user(&buf_size, (void *)user_size_addr, sizeof(uint32_t)) != 0) { tloge("copy from user failed\n"); return -EFAULT; -- Gitee From 191103a388f4a071f88deea739f07e3ff3aed1ee Mon Sep 17 00:00:00 2001 From: chenzheng Date: Thu, 12 Dec 2024 16:24:50 +0000 Subject: [PATCH 3/4] speed vm packet deal --- .../Host/vtzb_proxy/serial_port.c | 9 +- .../Host/vtzb_proxy/thread_pool.c | 101 ++++++++++++++++++ .../Host/vtzb_proxy/thread_pool.h | 4 + .../Host/vtzb_proxy/vtzb_proxy.c | 64 ----------- .../Host/vtzb_proxy/vtzb_proxy.h | 2 + trustzone-awared-vm/VM/vtzdriver/vtzf.c | 4 + 6 files changed, 116 insertions(+), 68 deletions(-) diff --git a/trustzone-awared-vm/Host/vtzb_proxy/serial_port.c b/trustzone-awared-vm/Host/vtzb_proxy/serial_port.c index 9c68b94..dc25b03 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/serial_port.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/serial_port.c @@ -16,6 +16,7 @@ #include "comm_structs.h" #include "vm.h" #include "debug.h" +#include "thread_pool.h" #include "virt.h" struct serial_port_list g_serial_list; @@ -165,12 +166,10 @@ static void do_check_stat_serial_port() if (ret < 0) { tloge("connect_domsock_chardev(%s) failed, ret = %d \n", serial_port->path, ret); } else { + tlogd("vm %d started, connect fd %d, create read thread\n", i, serial_port->sock); serial_port->opened = true; serial_port->offset = 0; - g_serial_array[i] = serial_port; - g_pollfd[i].fd = serial_port->sock; - g_pollfd[i].events = POLLIN; - tlogd("vm %d started, connect fd %d\n", i, serial_port->sock); + create_reader_thread(serial_port, i); } } } else { @@ -191,6 +190,8 @@ void check_stat_serial_port() if (g_cur_time.tv_sec - g_last_time.tv_sec > CHECK_TIME_SEC) { do_check_stat_serial_port(); gettimeofday(&g_last_time, NULL); + } else { + sleep(CHECK_TIME_SEC); } } diff --git a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c index 0048859..c818bdb 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c @@ -1,3 +1,4 @@ +#define _GNU_SOURCE #include #include #include @@ -5,12 +6,20 @@ #include #include #include +#include +#include +#include #include "thread_pool.h" +#include "serial_port.h" +#include "process_data.h" +#include "vtzb_proxy.h" #include "debug.h" #include "vm.h" +extern ThreadPool g_pool; ThreadFuncArgs g_thd_args[THREAD_POOL_SIZE]; TimeOut g_time_out[THREAD_POOL_SIZE]; +static cpu_set_t g_cpuset; /* Custom signal handler for killing zombie threads. */ void signal_handler(int signum) { @@ -19,9 +28,27 @@ void signal_handler(int signum) { pthread_exit(NULL); } +static void init_cpu_set() +{ + int cpu_num = get_nprocs(); + CPU_ZERO(&g_cpuset); + for (int i = 1; i <= CPU_SET_NUM && i < cpu_num; i++) { + CPU_SET(cpu_num - i, &g_cpuset); + } +} + +#define CPU_SET_AFFINITY() \ +do { \ + if (pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &g_cpuset)) { \ + tloge("set cpu affinity failed\n"); \ + } \ +} while(0) + + /* Initialize the thread pool. */ int thread_pool_init(ThreadPool *pool) { + char name[20] = {0}; pool->task_cnt = 0; pool->busy_cnt = 0; pool->front = pool->rear = 0; @@ -29,7 +56,10 @@ int thread_pool_init(ThreadPool *pool) memset(pool->task_queue, 0, sizeof(Task) * TASK_QUEUE_SIZE); memset(pool->kill_flag, 0, sizeof(bool) * THREAD_POOL_SIZE); memset(pool->session_ids, 0, sizeof(unsigned int) * THREAD_POOL_SIZE); + init_cpu_set(); + CPU_SET_AFFINITY(); pthread_create(&pool->admin_tid, NULL, admin_thread, pool); + pthread_setname_np(pool->admin_tid, "adminer"); pthread_mutex_init(&pool->task_mutex, NULL); pthread_mutex_init(&pool->session_mutex, NULL); pthread_mutex_init(&pool->time_mutex, NULL); @@ -40,6 +70,8 @@ int thread_pool_init(ThreadPool *pool) g_thd_args[i].index = i; g_thd_args[i].pool = pool; pthread_create(&pool->threads[i], NULL, thread_func, &g_thd_args[i]); + sprintf(name, "worker_%d", i); + pthread_setname_np(pool->threads[i], name); pthread_detach(pool->threads[i]); } return 0; @@ -72,6 +104,7 @@ void *thread_func(void *arg) return NULL; } + CPU_SET_AFFINITY(); while (1) { pthread_mutex_lock(&pool->task_mutex); @@ -125,6 +158,8 @@ void *admin_thread(void *arg) ThreadPool *pool = (ThreadPool *)arg; struct timeval cur_time; long time_sec = 0; + + CPU_SET_AFFINITY(); while (!pool->destroying) { sleep(DEFAULT_TIME_SEC); gettimeofday(&cur_time, NULL); @@ -141,6 +176,72 @@ void *admin_thread(void *arg) return NULL; } +static void *deal_packet_thread(void *arg) +{ + int ret; + int offset = 0; + int buf_len; + struct serial_port_file *serial_port = (struct serial_port_file *)arg; + + CPU_SET_AFFINITY(); + while (!g_pool.destroying) { + if (!serial_port || !serial_port->rd_buf || serial_port->sock <= 0) { + tloge("serial_port ptr or rd_buf or fd is invalid\n"); + goto end; + } + + ret = read(serial_port->sock, serial_port->rd_buf + serial_port->offset, BUF_LEN_MAX_RD - serial_port->offset); + if (ret < 0) { + tloge("read domain socket failed, err: %s\n", strerror(errno)); + if (errno == ECONNRESET || errno == EBADF) { + goto end; + } + continue; + } + // when vm destroy, has many zero read + if (ret == 0) { + continue; + } + buf_len = ret + serial_port->offset; + while (1) { + void *packet = NULL; + packet = get_packet_item(serial_port->rd_buf, buf_len, &offset); + if (packet == NULL) { + break; + } + + vm_trace_data *data = (vm_trace_data *)packet; + data->serial_port_ptr = (uint64_t)serial_port; + data->vmid = serial_port->index; + thread_pool_submit(&g_pool, thread_entry, (void *)((uint64_t)packet)); + } + serial_port->offset = offset; + } + +end: + tlogd("reader thread %d exit\n", serial_port->index); + return NULL; +} + +int create_reader_thread(struct serial_port_file *serial_port, int i) +{ + int ret; + char name[20] = {0}; + if ((ret = pthread_create(&g_pool.reader_threads[i], NULL, deal_packet_thread, serial_port))) { + tloge("create reader thread failed\n"); + return ret; + } + sprintf(name, "reader_%d", i); + if ((ret = pthread_setname_np(g_pool.reader_threads[i], name))) { + tloge("set thread name failed\n"); + return ret; + } + if ((ret = pthread_detach(g_pool.reader_threads[i]))) { + tloge("thread detach failed\n"); + } + return ret; +} + /* Submit the task to the thread pool. */ void thread_pool_submit(ThreadPool *pool, void *(*task_func)(void *), void *arg) { diff --git a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h index 0603d01..8cf5c29 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h +++ b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h @@ -19,10 +19,12 @@ #include #include #include +#include "serial_port.h" #define THREAD_POOL_SIZE 196 #define TASK_QUEUE_SIZE 32 #define DEFAULT_TIME_SEC 30 +#define CPU_SET_NUM 8 /* task structure */ typedef struct { @@ -34,6 +36,7 @@ typedef struct { typedef struct { pthread_t admin_tid; pthread_t threads[THREAD_POOL_SIZE]; // Thread array + pthread_t reader_threads[SERIAL_PORT_NUM]; unsigned int session_ids[THREAD_POOL_SIZE]; // Session ID of the ongoing command bool kill_flag[THREAD_POOL_SIZE]; void *task_args[THREAD_POOL_SIZE]; @@ -60,6 +63,7 @@ int thread_pool_init(ThreadPool *pool); void thread_pool_destroy(ThreadPool *pool); void *thread_func(void *arg); void *admin_thread(void *arg); +int create_reader_thread(struct serial_port_file *serial_port, int i); void thread_pool_submit(ThreadPool *pool, void *(*task_func)(void *), void *arg); void replenish_thread_pool(ThreadPool *pool, pthread_t thd); void set_thread_session_id(ThreadPool *pool, pthread_t thd, unsigned int id); diff --git a/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.c b/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.c index 2155fcf..62fa837 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.c @@ -774,43 +774,7 @@ END: return NULL; } -void proc_event(struct serial_port_file *serial_port) -{ - int ret; - int offset = 0; - int buf_len; - int fd; - - if (!serial_port || !serial_port->rd_buf || serial_port->sock <= 0) { - tloge("serial_port ptr or rd_buf is NULL!\n"); - return; - } - fd = serial_port->sock; - ret = read(fd, serial_port->rd_buf + serial_port->offset, BUF_LEN_MAX_RD - serial_port->offset); - if (ret < 0) { - tloge("read domain socket failed \n"); - return; - } - if (ret == 0) - return; - buf_len = ret + serial_port->offset; - while (1) { - void *packet = NULL; - packet = get_packet_item(serial_port->rd_buf, buf_len, &offset); - if (packet == NULL) - break; - - vm_trace_data *data = (vm_trace_data *)packet; - data->serial_port_ptr = (uint64_t)serial_port; - data->vmid = serial_port->index; - thread_pool_submit(&g_pool, thread_entry, (void *)((uint64_t)packet)); - } - serial_port->offset = offset; -} - int main() { - int ret = 0; - int i; serial_port_list_init(); if (thread_pool_init(&g_pool)) goto END2; @@ -819,34 +783,6 @@ int main() { while (1) { check_stat_serial_port(); - ret = safepoll(g_pollfd, SERIAL_PORT_NUM, 20*1000); - if (ret <= 0) { - tlogv("pollfd no event or failed, ret = %d\n", ret); - continue; - } - - tlogv("poll receive event %d\n", ret); - for (i = 0; i < SERIAL_PORT_NUM; i++) { - if (g_pollfd[i].revents == 0) { - continue; - } - - tlogv("vm %d, event %x, fd %d\n", i, g_pollfd[i].revents, g_pollfd[i].fd); - if (g_pollfd[i].revents & POLLIN) { - proc_event(g_serial_array[i]); - } - - if (g_pollfd[i].revents & POLLERR || - g_pollfd[i].revents & POLLNVAL) { - tloge("vm %d got error event\n", i); - continue; - } - - if (g_pollfd[i].revents & POLLHUP) { - tloge("vm %d got POLLHUP event\n", i); - release_vm_file(g_serial_array[i], i); - } - } } END1: diff --git a/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.h b/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.h index f8eaf49..cf753f8 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.h +++ b/trustzone-awared-vm/Host/vtzb_proxy/vtzb_proxy.h @@ -76,4 +76,6 @@ typedef union { }share; }struct_page_block; +void *thread_entry(void *args); + #endif /* __VTZB_PROXY_H__ */ \ No newline at end of file diff --git a/trustzone-awared-vm/VM/vtzdriver/vtzf.c b/trustzone-awared-vm/VM/vtzdriver/vtzf.c index 8c79e50..9c9f597 100644 --- a/trustzone-awared-vm/VM/vtzdriver/vtzf.c +++ b/trustzone-awared-vm/VM/vtzdriver/vtzf.c @@ -2016,6 +2016,10 @@ int public_ioctl(const struct file *file, unsigned int cmd, tloge("copy from user failed\n"); return -EFAULT; } + // no need to load kunpeng_sec_drv.sec in vm + if (ioctlArg.sec_file_info.secfile_type == LOAD_DYNAMIC_DRV) { + return 0; + } ret = tc_ns_load_secfile(file->private_data, &ioctlArg); if (copy_to_user(argp, &ioctlArg, sizeof(ioctlArg)) != 0 && ret == 0) ret = -EFAULT; -- Gitee From 41cf81b510c1addc3b25c6317e57ac9082991389 Mon Sep 17 00:00:00 2001 From: chenzheng Date: Thu, 26 Dec 2024 07:29:27 +0000 Subject: [PATCH 4/4] bugfix: agent unregister args should be array --- trustzone-awared-vm/Host/vtzb_proxy/agent.c | 6 ++- .../Host/vtzb_proxy/thread_pool.c | 37 ++++++++++++------- .../Host/vtzb_proxy/thread_pool.h | 4 +- trustzone-awared-vm/Host/vtzb_proxy/vm.c | 9 +---- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/trustzone-awared-vm/Host/vtzb_proxy/agent.c b/trustzone-awared-vm/Host/vtzb_proxy/agent.c index 5f67c75..9e9eb22 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/agent.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/agent.c @@ -23,6 +23,7 @@ extern ThreadPool g_pool; void do_free_agent(struct_agent_args *agent_args) { int ret = -1; + unsigned long buf[2] = {0}; if (agent_args == NULL) { tloge("agent args is null\n"); @@ -31,14 +32,15 @@ void do_free_agent(struct_agent_args *agent_args) tlogv("free agent fd %u\n", agent_args->dev_fd); ListRemoveEntry(&(agent_args->node)); - ret = ioctl(agent_args->dev_fd, TC_NS_CLIENT_IOCTL_UNREGISTER_AGENT, agent_args->args.id); + buf[0] = agent_args->args.id; + ret = ioctl(agent_args->dev_fd, TC_NS_CLIENT_IOCTL_UNREGISTER_AGENT, buf); if (ret) { tloge("ioctl failed\n"); } close(agent_args->dev_fd); agent_args->dev_fd = -1; - if (agent_args->thd!= 0) { + if (agent_args->thd != 0) { thread_pool_submit(&g_pool, Kill_useless_thread, (void *)(agent_args->thd)); } pthread_spin_destroy(&agent_args->spinlock); diff --git a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c index c818bdb..e897e78 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.c @@ -48,7 +48,7 @@ do { \ /* Initialize the thread pool. */ int thread_pool_init(ThreadPool *pool) { - char name[20] = {0}; + char name[THREAD_NAME_LEN] = {0}; pool->task_cnt = 0; pool->busy_cnt = 0; pool->front = pool->rear = 0; @@ -78,22 +78,41 @@ int thread_pool_init(ThreadPool *pool) } /* Recreate a new thread to fill the gap in the thread pool after killing a thread. */ -void replenish_thread_pool(ThreadPool *pool, pthread_t thd) +void replenish_thread_pool(ThreadPool *pool, pthread_t thd, char *name) { for (int i = 0; i < THREAD_POOL_SIZE; i++) { if (pthread_equal(pool->threads[i], thd)) { g_thd_args[i].index = i; g_thd_args[i].pool = pool; pthread_create(&pool->threads[i], NULL, thread_func, &g_thd_args[i]); + pthread_setname_np(pool->threads[i], name); pthread_detach(pool->threads[i]); pool->kill_flag[i] = false; - tlogv("old id %lu, new id %lu\n", thd, pool->threads[i]); + tlogv("thread %s : old id %lu, new id %lu\n", name, thd, pool->threads[i]); return; } } tloge("can't found the killed thread %lu\n", thd); } +void restart_pool_thread(ThreadPool *pool, pthread_t tid) +{ + char name[THREAD_NAME_LEN] = {0}; + int result; + + pthread_getname_np(tid, name, THREAD_NAME_LEN); + tlogv("try to kill thread %s: %lu\n", name, tid); + result = pthread_kill(tid, SIGUSR1); + if (result != 0) { + tloge("try to kill thread failed, ret %d\n", result); + return; + } + pthread_mutex_lock(&pool->busy_mutex); + pool->busy_cnt--; + pthread_mutex_unlock(&pool->busy_mutex); + replenish_thread_pool(pool, tid, name); +} + /* Thread function */ void *thread_func(void *arg) { @@ -192,10 +211,10 @@ static void *deal_packet_thread(void *arg) ret = read(serial_port->sock, serial_port->rd_buf + serial_port->offset, BUF_LEN_MAX_RD - serial_port->offset); if (ret < 0) { - tloge("read domain socket failed, err: %s\n", strerror(errno)); if (errno == ECONNRESET || errno == EBADF) { goto end; } + tloge("read domain socket failed, err: %s\n", strerror(errno)); continue; } // when vm destroy, has many zero read @@ -226,7 +245,7 @@ end: int create_reader_thread(struct serial_port_file *serial_port, int i) { int ret; - char name[20] = {0}; + char name[THREAD_NAME_LEN] = {0}; if ((ret = pthread_create(&g_pool.reader_threads[i], NULL, deal_packet_thread, serial_port))) { tloge("create reader thread failed\n"); return ret; @@ -285,14 +304,6 @@ void thread_pool_destroy(ThreadPool *pool) pthread_cond_destroy(&pool->queue_not_empty); } -bool check_if_thd_exist(pthread_t thd) -{ - int kill_rc = pthread_kill(thd, 0); - if(kill_rc != 0) - return false; - return true; -} - void set_thread_session_id(ThreadPool *pool, pthread_t thd, unsigned int id) { pthread_mutex_lock(&pool->session_mutex); diff --git a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h index 8cf5c29..5a5b880 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h +++ b/trustzone-awared-vm/Host/vtzb_proxy/thread_pool.h @@ -25,6 +25,7 @@ #define TASK_QUEUE_SIZE 32 #define DEFAULT_TIME_SEC 30 #define CPU_SET_NUM 8 +#define THREAD_NAME_LEN 32 /* task structure */ typedef struct { @@ -65,7 +66,8 @@ void *thread_func(void *arg); void *admin_thread(void *arg); int create_reader_thread(struct serial_port_file *serial_port, int i); void thread_pool_submit(ThreadPool *pool, void *(*task_func)(void *), void *arg); -void replenish_thread_pool(ThreadPool *pool, pthread_t thd); +void replenish_thread_pool(ThreadPool *pool, pthread_t thd, char *name); +void restart_pool_thread(ThreadPool *pool, pthread_t tid); void set_thread_session_id(ThreadPool *pool, pthread_t thd, unsigned int id); unsigned int get_thread_session_id(ThreadPool *pool, pthread_t thd, unsigned int session_id); #endif diff --git a/trustzone-awared-vm/Host/vtzb_proxy/vm.c b/trustzone-awared-vm/Host/vtzb_proxy/vm.c index 1fca1c3..333f1bb 100644 --- a/trustzone-awared-vm/Host/vtzb_proxy/vm.c +++ b/trustzone-awared-vm/Host/vtzb_proxy/vm.c @@ -35,14 +35,7 @@ void add_session_list(int ptzfd, struct vm_file *vm_fp, TC_NS_ClientContext *cli void *Kill_useless_thread(void *args) { pthread_t tid = (pthread_t)args; - tlogv("try to kill thread %lu\n", tid); - int result = pthread_kill(tid, SIGUSR1); - if (result == 0) { - pthread_mutex_lock(&g_pool.busy_mutex); - g_pool.busy_cnt--; - pthread_mutex_unlock(&g_pool.busy_mutex); - replenish_thread_pool(&g_pool, tid); - } + restart_pool_thread(&g_pool, tid); return NULL; } -- Gitee