From f3e54de7d296a874c9dddfda4779d644e9a265bb Mon Sep 17 00:00:00 2001 From: Yabin Li Date: Fri, 19 Jul 2024 15:45:27 +0800 Subject: [PATCH 1/4] hw/vfio/hct: sharing CCP resources between host and virtual machines. ccp support been used by guest and host at same time Signed-off-by: liyabin Signed-off-by: yangdepei --- hw/vfio/hct.c | 112 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 76 insertions(+), 36 deletions(-) diff --git a/hw/vfio/hct.c b/hw/vfio/hct.c index 2f40bf5c939..dd541f7f8b3 100644 --- a/hw/vfio/hct.c +++ b/hw/vfio/hct.c @@ -43,7 +43,7 @@ #define HCT_SHARE_DEV "/dev/hct_share" -#define HCT_VERSION_STRING "0.5" +#define HCT_VERSION_STRING "0.6" #define DEF_VERSION_STRING "0.1" #define VERSION_SIZE 16 @@ -64,6 +64,8 @@ #define HCT_PASID_BAR_IDX 4 #define PASID_OFFSET 40 +#define HCT_PASID_MEM_GID_OFFSET 1024 +#define HCT_PASID_MEM_MDEV_OFFSET 2048 static volatile struct hct_data { int init; @@ -94,31 +96,91 @@ struct hct_dev_ctrl { unsigned char rsvd[3]; union { unsigned char version[VERSION_SIZE]; + unsigned int id; + unsigned int pasid; struct { unsigned long vaddr; unsigned long iova; unsigned long size; }; - unsigned int id; }; }; + +static int hct_get_sysfs_value(const char *path, int *val) +{ + FILE *fp = NULL; + char buf[CCP_INDEX_BYTES]; + unsigned long v; + + fp = fopen(path, "r"); + if (!fp) { + error_report("fail to open %s, errno %d.\n", path, errno); + return -EINVAL; + } + + if (fgets(buf, sizeof(buf), fp) == NULL) { + fclose(fp); + return -EINVAL; + } + + if (1 != sscanf(buf, "%lu", &v)) { + fclose(fp); + return -EINVAL; + } + + *val = (int)v; + + fclose(fp); + return 0; +} + +/* + * the memory layout of pasid_memory is as follows: + * offset -- 0 1024 2048 4096 + * a page -- |pasid(8B) --- |gid(8B) --- |mdev_idx(8B) --- | + */ static int pasid_get_and_init(HCTDevState *state) { + void *base = (void *)hct_data.pasid_memory; struct hct_dev_ctrl ctrl; + unsigned long *gid = NULL; + unsigned long *mdev_idx = NULL; + char path[PATH_MAX]; + int index; int ret; ctrl.op = HCT_SHARE_OP_GET_PASID; - ctrl.id = -1; ret = ioctl(hct_data.hct_fd, HCT_SHARE_OP, &ctrl); if (ret < 0) { ret = -errno; - error_report("GET_PASID fail: %d", -errno); + error_report("get pasid fail, errno: %d.", errno); goto out; } - *hct_data.pasid_memory = ctrl.id; - hct_data.pasid = ctrl.id; + hct_data.pasid = (unsigned long)ctrl.pasid; + *(unsigned long *)base = (unsigned long)ctrl.pasid; + + ctrl.op = HCT_SHARE_OP_GET_ID; + ret = ioctl(hct_data.hct_fd, HCT_SHARE_OP, &ctrl); + if (ret < 0) { + ret = -errno; + error_report("get gid fail, errno: %d", errno); + goto out; + } + + gid = (unsigned long *)((unsigned long)base + HCT_PASID_MEM_GID_OFFSET); + *(unsigned long *)gid = (unsigned long)ctrl.id; + + snprintf(path, PATH_MAX, "%s/vendor/idx", state->vdev.sysfsdev); + if (hct_get_sysfs_value(path, &index)) { + ret = -EINVAL; + error_report("get %s sysfs value fail.\n", path); + goto out; + } + + mdev_idx = (unsigned long *)((unsigned long)base + HCT_PASID_MEM_MDEV_OFFSET); + *(unsigned long *)mdev_idx = (unsigned long)index; out: return ret; @@ -236,41 +298,19 @@ static int hct_check_duplicated_index(int index) static int hct_get_ccp_index(HCTDevState *state) { char path[PATH_MAX]; - char buf[CCP_INDEX_BYTES]; - int fd; - int ret; - int ccp_index; + int index; snprintf(path, PATH_MAX, "%s/vendor/id", state->vdev.sysfsdev); - fd = qemu_open_old(path, O_RDONLY); - if (fd < 0) { - error_report("open %s fail\n", path); - return -errno; - } - - memset(buf, 0, sizeof(buf)); - ret = read(fd, buf, sizeof(buf)); - if (ret < 0) { - ret = -errno; - error_report("read %s fail\n", path); - goto out; - } - - if (1 != sscanf(buf, "%d", &ccp_index)) { - ret = -errno; - error_report("format addr %s fail\n", buf); - goto out; + if (hct_get_sysfs_value(path, &index)) { + error_report("get %s sysfs value fail.\n", path); + return -1; } - if (!hct_check_duplicated_index(ccp_index)) { - state->sdev.shared_memory_offset = ccp_index; - } else { - ret = -1; - } + if (hct_check_duplicated_index(index)) + return -1; -out: - qemu_close(fd); - return ret; + state->sdev.shared_memory_offset = index; + return 0; } static int hct_api_version_check(void) -- Gitee From 36fd43fdce4749d77c68f0e453ad44be854773b8 Mon Sep 17 00:00:00 2001 From: Yabin Li Date: Wed, 31 Jul 2024 14:27:58 +0800 Subject: [PATCH 2/4] hw/vfio/hct: compatible with hct.ko modules in versions 0.5 and earlier. Signed-off-by: liyabin Signed-off-by: yangdepei --- hw/vfio/hct.c | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/hw/vfio/hct.c b/hw/vfio/hct.c index dd541f7f8b3..83e15e18d34 100644 --- a/hw/vfio/hct.c +++ b/hw/vfio/hct.c @@ -29,6 +29,7 @@ #include "hw/qdev-properties.h" #define MAX_CCP_CNT 48 +#define DEF_CCP_CNT_MAX 16 #define PAGE_SIZE 4096 #define HCT_SHARED_MEMORY_SIZE (PAGE_SIZE * MAX_CCP_CNT) #define CCP_INDEX_BYTES 4 @@ -45,6 +46,8 @@ #define HCT_VERSION_STRING "0.6" #define DEF_VERSION_STRING "0.1" +#define HCT_VERSION_STR_02 "0.2" +#define HCT_VERSION_STR_05 "0.5" #define VERSION_SIZE 16 #define HCT_SHARE_IOC_TYPE 'C' @@ -71,6 +74,7 @@ static volatile struct hct_data { int init; int hct_fd; unsigned long pasid; + unsigned long hct_shared_size; uint8_t *pasid_memory; uint8_t *hct_shared_memory; uint8_t ccp_index[MAX_CCP_CNT]; @@ -322,17 +326,22 @@ static int hct_api_version_check(void) memcpy(ctrl.version, DEF_VERSION_STRING, sizeof(DEF_VERSION_STRING)); ret = ioctl(hct_data.hct_fd, HCT_SHARE_OP, &ctrl); if (ret < 0) { - error_report("ret %d, errno %d: fail to get hct.ko version.\n", ret, - errno); + error_report("ret %d, errno %d: fail to get hct.ko version.\n", ret, errno); return -1; - } else if (memcmp(ctrl.version, HCT_VERSION_STRING, - sizeof(HCT_VERSION_STRING)) < 0) { - error_report("The hct.ko version is %s, please upgrade to version %s " - "or higher.\n", - ctrl.version, HCT_VERSION_STRING); + } else if (memcmp(ctrl.version, HCT_VERSION_STR_02, sizeof(HCT_VERSION_STR_02)) < 0) { + error_report("The hct.ko version is %s, please upgrade to version %s or higher.\n", + ctrl.version, HCT_VERSION_STR_02); return -1; + } else if (memcmp(ctrl.version, HCT_VERSION_STRING, sizeof(HCT_VERSION_STRING)) != 0) { + info_report("The hct.ko version is %s, please upgrade to version %s.\n", + ctrl.version, HCT_VERSION_STRING); } + if (memcmp(ctrl.version, HCT_VERSION_STR_05, sizeof(HCT_VERSION_STR_05)) < 0) + hct_data.hct_shared_size = PAGE_SIZE * DEF_CCP_CNT_MAX; + else + hct_data.hct_shared_size = HCT_SHARED_MEMORY_SIZE; + return 0; } @@ -340,9 +349,9 @@ static int hct_shared_memory_init(void) { int ret = 0; - hct_data.hct_shared_memory = - mmap(NULL, HCT_SHARED_MEMORY_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, - hct_data.hct_fd, 0); + hct_data.hct_shared_memory = mmap(NULL, hct_data.hct_shared_size, + PROT_READ | PROT_WRITE, MAP_SHARED, + hct_data.hct_fd, 0); if (hct_data.hct_shared_memory == MAP_FAILED) { ret = -errno; error_report("map hct shared memory fail\n"); @@ -442,7 +451,7 @@ static void hct_data_uninit(HCTDevState *state) } if (hct_data.hct_shared_memory) { - munmap((void *)hct_data.hct_shared_memory, HCT_SHARED_MEMORY_SIZE); + munmap((void *)hct_data.hct_shared_memory, hct_data.hct_shared_size); hct_data.hct_shared_memory = NULL; } @@ -495,7 +504,7 @@ unmap_pasid_memory_exit: munmap(hct_data.pasid_memory, PAGE_SIZE); unmap_shared_memory_exit: - munmap((void *)hct_data.hct_shared_memory, HCT_SHARED_MEMORY_SIZE); + munmap((void *)hct_data.hct_shared_memory, hct_data.hct_shared_size); out: return ret; -- Gitee From 3a6ed806369fa3805ae3af6649fdb59d19ec15ee Mon Sep 17 00:00:00 2001 From: yangdepei Date: Tue, 27 Aug 2024 13:58:35 +0800 Subject: [PATCH 3/4] hw/vfio/hct: fix ccp index error caused by uninitialized buf Signed-off-by: liyabin Signed-off-by: yangdepei --- hw/vfio/hct.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/vfio/hct.c b/hw/vfio/hct.c index 83e15e18d34..f6724a38fb5 100644 --- a/hw/vfio/hct.c +++ b/hw/vfio/hct.c @@ -114,7 +114,7 @@ struct hct_dev_ctrl { static int hct_get_sysfs_value(const char *path, int *val) { FILE *fp = NULL; - char buf[CCP_INDEX_BYTES]; + char buf[CCP_INDEX_BYTES] = {0}; unsigned long v; fp = fopen(path, "r"); @@ -301,7 +301,7 @@ static int hct_check_duplicated_index(int index) static int hct_get_ccp_index(HCTDevState *state) { - char path[PATH_MAX]; + char path[PATH_MAX] = {0}; int index; snprintf(path, PATH_MAX, "%s/vendor/id", state->vdev.sysfsdev); -- Gitee From 20efc073a228011b8719257e0d9303bfbf678b4a Mon Sep 17 00:00:00 2001 From: Yabin Li Date: Wed, 4 Sep 2024 20:55:42 +0800 Subject: [PATCH 4/4] hw/vfio/hct: remove the operation of reading the value of mdev index in qemu. Signed-off-by: liyabin Signed-off-by: yangdepei --- hw/vfio/hct.c | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/hw/vfio/hct.c b/hw/vfio/hct.c index f6724a38fb5..953d7b03a85 100644 --- a/hw/vfio/hct.c +++ b/hw/vfio/hct.c @@ -44,10 +44,10 @@ #define HCT_SHARE_DEV "/dev/hct_share" -#define HCT_VERSION_STRING "0.6" #define DEF_VERSION_STRING "0.1" #define HCT_VERSION_STR_02 "0.2" #define HCT_VERSION_STR_05 "0.5" +#define HCT_VERSION_STR_06 "0.6" #define VERSION_SIZE 16 #define HCT_SHARE_IOC_TYPE 'C' @@ -68,7 +68,6 @@ #define PASID_OFFSET 40 #define HCT_PASID_MEM_GID_OFFSET 1024 -#define HCT_PASID_MEM_MDEV_OFFSET 2048 static volatile struct hct_data { int init; @@ -77,6 +76,7 @@ static volatile struct hct_data { unsigned long hct_shared_size; uint8_t *pasid_memory; uint8_t *hct_shared_memory; + uint8_t hct_version[VERSION_SIZE]; uint8_t ccp_index[MAX_CCP_CNT]; uint8_t ccp_cnt; } hct_data; @@ -110,6 +110,11 @@ struct hct_dev_ctrl { }; }; +enum MDEV_USED_TYPE { + MDEV_USED_FOR_HOST, + MDEV_USED_FOR_VM, + MDEV_USED_UNDEF +}; static int hct_get_sysfs_value(const char *path, int *val) { @@ -141,17 +146,14 @@ static int hct_get_sysfs_value(const char *path, int *val) /* * the memory layout of pasid_memory is as follows: - * offset -- 0 1024 2048 4096 - * a page -- |pasid(8B) --- |gid(8B) --- |mdev_idx(8B) --- | + * offset -- 0 1024 4096 + * a page -- |pasid(8B) --- |gid(8B) --- | */ static int pasid_get_and_init(HCTDevState *state) { void *base = (void *)hct_data.pasid_memory; struct hct_dev_ctrl ctrl; unsigned long *gid = NULL; - unsigned long *mdev_idx = NULL; - char path[PATH_MAX]; - int index; int ret; ctrl.op = HCT_SHARE_OP_GET_PASID; @@ -176,16 +178,6 @@ static int pasid_get_and_init(HCTDevState *state) gid = (unsigned long *)((unsigned long)base + HCT_PASID_MEM_GID_OFFSET); *(unsigned long *)gid = (unsigned long)ctrl.id; - snprintf(path, PATH_MAX, "%s/vendor/idx", state->vdev.sysfsdev); - if (hct_get_sysfs_value(path, &index)) { - ret = -EINVAL; - error_report("get %s sysfs value fail.\n", path); - goto out; - } - - mdev_idx = (unsigned long *)((unsigned long)base + HCT_PASID_MEM_MDEV_OFFSET); - *(unsigned long *)mdev_idx = (unsigned long)index; - out: return ret; } @@ -302,7 +294,20 @@ static int hct_check_duplicated_index(int index) static int hct_get_ccp_index(HCTDevState *state) { char path[PATH_MAX] = {0}; - int index; + int mdev_used, index; + + if (memcmp((void *)hct_data.hct_version, HCT_VERSION_STR_06, + sizeof(HCT_VERSION_STR_06)) >= 0) { + snprintf(path, PATH_MAX, "%s/vendor/use", state->vdev.sysfsdev); + if (hct_get_sysfs_value(path, &mdev_used)) { + error_report("get %s sysfs value fail.\n", path); + return -1; + } else if (mdev_used != MDEV_USED_FOR_VM) { + error_report("The value of file node(%s) is %d, should be MDEV_USED_FOR_VM(%d), pls check.\n", + path, mdev_used, MDEV_USED_FOR_VM); + return -1; + } + } snprintf(path, PATH_MAX, "%s/vendor/id", state->vdev.sysfsdev); if (hct_get_sysfs_value(path, &index)) { @@ -332,11 +337,9 @@ static int hct_api_version_check(void) error_report("The hct.ko version is %s, please upgrade to version %s or higher.\n", ctrl.version, HCT_VERSION_STR_02); return -1; - } else if (memcmp(ctrl.version, HCT_VERSION_STRING, sizeof(HCT_VERSION_STRING)) != 0) { - info_report("The hct.ko version is %s, please upgrade to version %s.\n", - ctrl.version, HCT_VERSION_STRING); } + memcpy((void *)hct_data.hct_version, (void *)ctrl.version, sizeof(hct_data.hct_version)); if (memcmp(ctrl.version, HCT_VERSION_STR_05, sizeof(HCT_VERSION_STR_05)) < 0) hct_data.hct_shared_size = PAGE_SIZE * DEF_CCP_CNT_MAX; else -- Gitee