From b33b7c39cbb8a5583e926efe943c1242e2edab21 Mon Sep 17 00:00:00 2001 From: Yang Zhong Date: Mon, 1 Nov 2021 12:20:05 -0400 Subject: [PATCH 01/31] numa: Enable numa for SGX EPC sections commit 1105812382e1126d86dddc16b3700f8c79dc93d1 upstream. The basic SGX did not enable numa for SGX EPC sections, which result in all EPC sections located in numa node 0. This patch enable SGX numa function in the guest and the EPC section can work with RAM as one numa node. The Guest kernel related log: [ 0.009981] ACPI: SRAT: Node 0 PXM 0 [mem 0x180000000-0x183ffffff] [ 0.009982] ACPI: SRAT: Node 1 PXM 1 [mem 0x184000000-0x185bfffff] The SRAT table can normally show SGX EPC sections menory info in different numa nodes. The SGX EPC numa related command: ...... -m 4G,maxmem=20G \ -smp sockets=2,cores=2 \ -cpu host,+sgx-provisionkey \ -object memory-backend-ram,size=2G,host-nodes=0,policy=bind,id=node0 \ -object memory-backend-epc,id=mem0,size=64M,prealloc=on,host-nodes=0,policy=bind \ -numa node,nodeid=0,cpus=0-1,memdev=node0 \ -object memory-backend-ram,size=2G,host-nodes=1,policy=bind,id=node1 \ -object memory-backend-epc,id=mem1,size=28M,prealloc=on,host-nodes=1,policy=bind \ -numa node,nodeid=1,cpus=2-3,memdev=node1 \ -M sgx-epc.0.memdev=mem0,sgx-epc.0.node=0,sgx-epc.1.memdev=mem1,sgx-epc.1.node=1 \ ...... Intel-SIG: commit 1105812382e1 ("numa: Enable numa for SGX EPC sections"). Backport SGX NUMA support for qemu v6.2.0 Signed-off-by: Yang Zhong Message-Id: <20211101162009.62161-2-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- hw/core/numa.c | 5 ++--- hw/i386/acpi-build.c | 2 ++ hw/i386/sgx-epc.c | 3 +++ hw/i386/sgx-stub.c | 4 ++++ hw/i386/sgx.c | 44 +++++++++++++++++++++++++++++++++++++++ include/hw/i386/sgx-epc.h | 3 +++ monitor/hmp-cmds.c | 1 + qapi/machine.json | 10 ++++++++- qemu-options.hx | 4 ++-- 9 files changed, 70 insertions(+), 6 deletions(-) diff --git a/hw/core/numa.c b/hw/core/numa.c index e6050b22739..1aa05dcf425 100644 --- a/hw/core/numa.c +++ b/hw/core/numa.c @@ -784,9 +784,8 @@ static void numa_stat_memory_devices(NumaNodeMem node_mem[]) break; case MEMORY_DEVICE_INFO_KIND_SGX_EPC: se = value->u.sgx_epc.data; - /* TODO: once we support numa, assign to right node */ - node_mem[0].node_mem += se->size; - node_mem[0].node_plugged_mem += se->size; + node_mem[se->node].node_mem += se->size; + node_mem[se->node].node_plugged_mem = 0; break; default: g_assert_not_reached(); diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index a99c6e4fe3f..8383b83ee36 100644 --- a/hw/i386/acpi-build.c +++ b/hw/i386/acpi-build.c @@ -2068,6 +2068,8 @@ build_srat(GArray *table_data, BIOSLinker *linker, MachineState *machine) nvdimm_build_srat(table_data); } + sgx_epc_build_srat(table_data); + /* * TODO: this part is not in ACPI spec and current linux kernel boots fine * without these entries. But I recall there were issues the last time I diff --git a/hw/i386/sgx-epc.c b/hw/i386/sgx-epc.c index e508827e787..96b2940d75e 100644 --- a/hw/i386/sgx-epc.c +++ b/hw/i386/sgx-epc.c @@ -21,6 +21,7 @@ static Property sgx_epc_properties[] = { DEFINE_PROP_UINT64(SGX_EPC_ADDR_PROP, SGXEPCDevice, addr, 0), + DEFINE_PROP_UINT32(SGX_EPC_NUMA_NODE_PROP, SGXEPCDevice, node, 0), DEFINE_PROP_LINK(SGX_EPC_MEMDEV_PROP, SGXEPCDevice, hostmem, TYPE_MEMORY_BACKEND_EPC, HostMemoryBackendEpc *), DEFINE_PROP_END_OF_LIST(), @@ -139,6 +140,8 @@ static void sgx_epc_md_fill_device_info(const MemoryDeviceState *md, se->memaddr = epc->addr; se->size = object_property_get_uint(OBJECT(epc), SGX_EPC_SIZE_PROP, NULL); + se->node = object_property_get_uint(OBJECT(epc), SGX_EPC_NUMA_NODE_PROP, + NULL); se->memdev = object_get_canonical_path(OBJECT(epc->hostmem)); info->u.sgx_epc.data = se; diff --git a/hw/i386/sgx-stub.c b/hw/i386/sgx-stub.c index c9b379e6651..26833eb233c 100644 --- a/hw/i386/sgx-stub.c +++ b/hw/i386/sgx-stub.c @@ -6,6 +6,10 @@ #include "qapi/error.h" #include "qapi/qapi-commands-misc-target.h" +void sgx_epc_build_srat(GArray *table_data) +{ +} + SGXInfo *qmp_query_sgx(Error **errp) { error_setg(errp, "SGX support is not compiled in"); diff --git a/hw/i386/sgx.c b/hw/i386/sgx.c index 8fef3dd8fad..d04299904a2 100644 --- a/hw/i386/sgx.c +++ b/hw/i386/sgx.c @@ -23,6 +23,7 @@ #include "sysemu/hw_accel.h" #include "sysemu/reset.h" #include +#include "hw/acpi/aml-build.h" #define SGX_MAX_EPC_SECTIONS 8 #define SGX_CPUID_EPC_INVALID 0x0 @@ -36,6 +37,46 @@ #define RETRY_NUM 2 +static int sgx_epc_device_list(Object *obj, void *opaque) +{ + GSList **list = opaque; + + if (object_dynamic_cast(obj, TYPE_SGX_EPC)) { + *list = g_slist_append(*list, DEVICE(obj)); + } + + object_child_foreach(obj, sgx_epc_device_list, opaque); + return 0; +} + +static GSList *sgx_epc_get_device_list(void) +{ + GSList *list = NULL; + + object_child_foreach(qdev_get_machine(), sgx_epc_device_list, &list); + return list; +} + +void sgx_epc_build_srat(GArray *table_data) +{ + GSList *device_list = sgx_epc_get_device_list(); + + for (; device_list; device_list = device_list->next) { + DeviceState *dev = device_list->data; + Object *obj = OBJECT(dev); + uint64_t addr, size; + int node; + + node = object_property_get_uint(obj, SGX_EPC_NUMA_NODE_PROP, + &error_abort); + addr = object_property_get_uint(obj, SGX_EPC_ADDR_PROP, &error_abort); + size = object_property_get_uint(obj, SGX_EPC_SIZE_PROP, &error_abort); + + build_srat_memory(table_data, addr, size, node, MEM_AFFINITY_ENABLED); + } + g_slist_free(device_list); +} + static uint64_t sgx_calc_section_metric(uint64_t low, uint64_t high) { return (low & MAKE_64BIT_MASK(12, 20)) + @@ -226,6 +267,9 @@ void pc_machine_init_sgx_epc(PCMachineState *pcms) /* set the memdev link with memory backend */ object_property_parse(obj, SGX_EPC_MEMDEV_PROP, list->value->memdev, &error_fatal); + /* set the numa node property for sgx epc object */ + object_property_set_uint(obj, SGX_EPC_NUMA_NODE_PROP, list->value->node, + &error_fatal); object_property_set_bool(obj, "realized", true, &error_fatal); object_unref(obj); } diff --git a/include/hw/i386/sgx-epc.h b/include/hw/i386/sgx-epc.h index a6a65be854f..581fac389a6 100644 --- a/include/hw/i386/sgx-epc.h +++ b/include/hw/i386/sgx-epc.h @@ -25,6 +25,7 @@ #define SGX_EPC_ADDR_PROP "addr" #define SGX_EPC_SIZE_PROP "size" #define SGX_EPC_MEMDEV_PROP "memdev" +#define SGX_EPC_NUMA_NODE_PROP "node" /** * SGXEPCDevice: @@ -38,6 +39,7 @@ typedef struct SGXEPCDevice { /* public */ uint64_t addr; + uint32_t node; HostMemoryBackendEpc *hostmem; } SGXEPCDevice; @@ -56,6 +58,7 @@ typedef struct SGXEPCState { } SGXEPCState; bool sgx_epc_get_section(int section_nr, uint64_t *addr, uint64_t *size); +void sgx_epc_build_srat(GArray *table_data); static inline uint64_t sgx_epc_above_4g_end(SGXEPCState *sgx_epc) { diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c index 9c91bf93e94..2669156b284 100644 --- a/monitor/hmp-cmds.c +++ b/monitor/hmp-cmds.c @@ -1810,6 +1810,7 @@ void hmp_info_memory_devices(Monitor *mon, const QDict *qdict) se->id ? se->id : ""); monitor_printf(mon, " memaddr: 0x%" PRIx64 "\n", se->memaddr); monitor_printf(mon, " size: %" PRIu64 "\n", se->size); + monitor_printf(mon, " node: %" PRId64 "\n", se->node); monitor_printf(mon, " memdev: %s\n", se->memdev); break; default: diff --git a/qapi/machine.json b/qapi/machine.json index 067e3f53787..16e771affcf 100644 --- a/qapi/machine.json +++ b/qapi/machine.json @@ -1207,12 +1207,15 @@ # # @memdev: memory backend linked with device # +# @node: the numa node +# # Since: 6.2 ## { 'struct': 'SgxEPCDeviceInfo', 'data': { '*id': 'str', 'memaddr': 'size', 'size': 'size', + 'node': 'int', 'memdev': 'str' } } @@ -1285,10 +1288,15 @@ # # @memdev: memory backend linked with device # +# @node: the numa node +# # Since: 6.2 ## { 'struct': 'SgxEPC', - 'data': { 'memdev': 'str' } } + 'data': { 'memdev': 'str', + 'node': 'int' + } +} ## # @SgxEPCProperties: diff --git a/qemu-options.hx b/qemu-options.hx index ae2c6dbbfc0..489b58e1511 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -127,11 +127,11 @@ SRST ERST DEF("M", HAS_ARG, QEMU_OPTION_M, - " sgx-epc.0.memdev=memid\n", + " sgx-epc.0.memdev=memid,sgx-epc.0.node=numaid\n", QEMU_ARCH_ALL) SRST -``sgx-epc.0.memdev=@var{memid}`` +``sgx-epc.0.memdev=@var{memid},sgx-epc.0.node=@var{numaid}`` Define an SGX EPC section. ERST -- Gitee From e6ac4cf2a115a341dd3851f1ef18937004cf9bf6 Mon Sep 17 00:00:00 2001 From: Yang Zhong Date: Mon, 1 Nov 2021 12:20:07 -0400 Subject: [PATCH 02/31] numa: Support SGX numa in the monitor and Libvirt interfaces commit 4755927ae12547c2e7cb22c5fa1b39038c6c11b1 upstream. Add the SGXEPCSection list into SGXInfo to show the multiple SGX EPC sections detailed info, not the total size like before. This patch can enable numa support for 'info sgx' command and QMP interfaces. The new interfaces show each EPC section info in one numa node. Libvirt can use QMP interface to get the detailed host SGX EPC capabilities to decide how to allocate host EPC sections to guest. (qemu) info sgx SGX support: enabled SGX1 support: enabled SGX2 support: enabled FLC support: enabled NUMA node #0: size=67108864 NUMA node #1: size=29360128 The QMP interface show: (QEMU) query-sgx {"return": {"sgx": true, "sgx2": true, "sgx1": true, "sections": \ [{"node": 0, "size": 67108864}, {"node": 1, "size": 29360128}], "flc": true}} (QEMU) query-sgx-capabilities {"return": {"sgx": true, "sgx2": true, "sgx1": true, "sections": \ [{"node": 0, "size": 17070817280}, {"node": 1, "size": 17079205888}], "flc": true}} Intel-SIG: commit 4755927ae125 ("numa: Support SGX numa in the monitor and Libvirt interfaces"). Backport SGX NUMA support for qemu v6.2.0 Signed-off-by: Yang Zhong Message-Id: <20211101162009.62161-4-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- hw/i386/sgx.c | 51 +++++++++++++++++++++++++++++++++++-------- qapi/misc-target.json | 19 ++++++++++++++-- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/hw/i386/sgx.c b/hw/i386/sgx.c index d04299904a2..5de5dd08936 100644 --- a/hw/i386/sgx.c +++ b/hw/i386/sgx.c @@ -83,11 +83,13 @@ static uint64_t sgx_calc_section_metric(uint64_t low, uint64_t high) ((high & MAKE_64BIT_MASK(0, 20)) << 32); } -static uint64_t sgx_calc_host_epc_section_size(void) +static SGXEPCSectionList *sgx_calc_host_epc_sections(void) { + SGXEPCSectionList *head = NULL, **tail = &head; + SGXEPCSection *section; uint32_t i, type; uint32_t eax, ebx, ecx, edx; - uint64_t size = 0; + uint32_t j = 0; for (i = 0; i < SGX_MAX_EPC_SECTIONS; i++) { host_cpuid(0x12, i + 2, &eax, &ebx, &ecx, &edx); @@ -101,10 +103,13 @@ static uint64_t sgx_calc_host_epc_section_size(void) break; } - size += sgx_calc_section_metric(ecx, edx); + section = g_new0(SGXEPCSection, 1); + section->node = j++; + section->size = sgx_calc_section_metric(ecx, edx); + QAPI_LIST_APPEND(tail, section); } - return size; + return head; } static void sgx_epc_reset(void *opaque) @@ -168,13 +173,35 @@ SGXInfo *qmp_query_sgx_capabilities(Error **errp) info->sgx1 = eax & (1U << 0) ? true : false; info->sgx2 = eax & (1U << 1) ? true : false; - info->section_size = sgx_calc_host_epc_section_size(); + info->sections = sgx_calc_host_epc_sections(); close(fd); return info; } +static SGXEPCSectionList *sgx_get_epc_sections_list(void) +{ + GSList *device_list = sgx_epc_get_device_list(); + SGXEPCSectionList *head = NULL, **tail = &head; + SGXEPCSection *section; + + for (; device_list; device_list = device_list->next) { + DeviceState *dev = device_list->data; + Object *obj = OBJECT(dev); + + section = g_new0(SGXEPCSection, 1); + section->node = object_property_get_uint(obj, SGX_EPC_NUMA_NODE_PROP, + &error_abort); + section->size = object_property_get_uint(obj, SGX_EPC_SIZE_PROP, + &error_abort); + QAPI_LIST_APPEND(tail, section); + } + g_slist_free(device_list); + + return head; +} + SGXInfo *qmp_query_sgx(Error **errp) { SGXInfo *info = NULL; @@ -193,14 +220,13 @@ SGXInfo *qmp_query_sgx(Error **errp) return NULL; } - SGXEPCState *sgx_epc = &pcms->sgx_epc; info = g_new0(SGXInfo, 1); info->sgx = true; info->sgx1 = true; info->sgx2 = true; info->flc = true; - info->section_size = sgx_epc->size; + info->sections = sgx_get_epc_sections_list(); return info; } @@ -208,6 +234,7 @@ SGXInfo *qmp_query_sgx(Error **errp) void hmp_info_sgx(Monitor *mon, const QDict *qdict) { Error *err = NULL; + SGXEPCSectionList *section_list, *section; g_autoptr(SGXInfo) info = qmp_query_sgx(&err); if (err) { @@ -222,8 +249,14 @@ void hmp_info_sgx(Monitor *mon, const QDict *qdict) info->sgx2 ? "enabled" : "disabled"); monitor_printf(mon, "FLC support: %s\n", info->flc ? "enabled" : "disabled"); - monitor_printf(mon, "size: %" PRIu64 "\n", - info->section_size); + + section_list = info->sections; + for (section = section_list; section; section = section->next) { + monitor_printf(mon, "NUMA node #%" PRId64 ": ", + section->value->node); + monitor_printf(mon, "size=%" PRIu64 "\n", + section->value->size); + } } bool sgx_epc_get_section(int section_nr, uint64_t *addr, uint64_t *size) diff --git a/qapi/misc-target.json b/qapi/misc-target.json index 5aa2b95b7d4..1022aa0184c 100644 --- a/qapi/misc-target.json +++ b/qapi/misc-target.json @@ -337,6 +337,21 @@ 'if': 'TARGET_ARM' } +## +# @SGXEPCSection: +# +# Information about intel SGX EPC section info +# +# @node: the numa node +# +# @size: the size of epc section +# +# Since: 6.2 +## +{ 'struct': 'SGXEPCSection', + 'data': { 'node': 'int', + 'size': 'uint64'}} + ## # @SGXInfo: # @@ -350,7 +365,7 @@ # # @flc: true if FLC is supported # -# @section-size: The EPC section size for guest +# @sections: The EPC sections info for guest # # Since: 6.2 ## @@ -359,7 +374,7 @@ 'sgx1': 'bool', 'sgx2': 'bool', 'flc': 'bool', - 'section-size': 'uint64'}, + 'sections': ['SGXEPCSection']}, 'if': 'TARGET_I386' } ## -- Gitee From eb688ed58b6e6c5b36eb701e28bb1977ca8be35a Mon Sep 17 00:00:00 2001 From: Yang Zhong Date: Mon, 1 Nov 2021 12:20:08 -0400 Subject: [PATCH 03/31] doc: Add the SGX numa description commit d1889b36098c79e2e6ac90faf3d0dc5ec0057677 upstream. Add the SGX numa reference command and how to check if SGX numa is support or not with multiple EPC sections. Intel-SIG: commit d1889b36098c ("doc: Add the SGX numa description"). Backport SGX NUMA support for qemu v6.2.0 Signed-off-by: Yang Zhong Message-Id: <20211101162009.62161-5-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- docs/system/i386/sgx.rst | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/system/i386/sgx.rst b/docs/system/i386/sgx.rst index f8fade5ac2d..0f0a73f7587 100644 --- a/docs/system/i386/sgx.rst +++ b/docs/system/i386/sgx.rst @@ -141,8 +141,7 @@ To launch a SGX guest: |qemu_system_x86| \\ -cpu host,+sgx-provisionkey \\ -object memory-backend-epc,id=mem1,size=64M,prealloc=on \\ - -object memory-backend-epc,id=mem2,size=28M \\ - -M sgx-epc.0.memdev=mem1,sgx-epc.1.memdev=mem2 + -M sgx-epc.0.memdev=mem1,sgx-epc.0.node=0 Utilizing SGX in the guest requires a kernel/OS with SGX support. The support can be determined in guest by:: @@ -152,8 +151,32 @@ The support can be determined in guest by:: and SGX epc info by:: $ dmesg | grep sgx - [ 1.242142] sgx: EPC section 0x180000000-0x181bfffff - [ 1.242319] sgx: EPC section 0x181c00000-0x1837fffff + [ 0.182807] sgx: EPC section 0x140000000-0x143ffffff + [ 0.183695] sgx: [Firmware Bug]: Unable to map EPC section to online node. Fallback to the NUMA node 0. + +To launch a SGX numa guest: + +.. parsed-literal:: + + |qemu_system_x86| \\ + -cpu host,+sgx-provisionkey \\ + -object memory-backend-ram,size=2G,host-nodes=0,policy=bind,id=node0 \\ + -object memory-backend-epc,id=mem0,size=64M,prealloc=on,host-nodes=0,policy=bind \\ + -numa node,nodeid=0,cpus=0-1,memdev=node0 \\ + -object memory-backend-ram,size=2G,host-nodes=1,policy=bind,id=node1 \\ + -object memory-backend-epc,id=mem1,size=28M,prealloc=on,host-nodes=1,policy=bind \\ + -numa node,nodeid=1,cpus=2-3,memdev=node1 \\ + -M sgx-epc.0.memdev=mem0,sgx-epc.0.node=0,sgx-epc.1.memdev=mem1,sgx-epc.1.node=1 + +and SGX epc numa info by:: + + $ dmesg | grep sgx + [ 0.369937] sgx: EPC section 0x180000000-0x183ffffff + [ 0.370259] sgx: EPC section 0x184000000-0x185bfffff + + $ dmesg | grep SRAT + [ 0.009981] ACPI: SRAT: Node 0 PXM 0 [mem 0x180000000-0x183ffffff] + [ 0.009982] ACPI: SRAT: Node 1 PXM 1 [mem 0x184000000-0x185bfffff] References ---------- -- Gitee From 65ef64f4e9dbbf88fc05743ba6c9253ba0fe5bdd Mon Sep 17 00:00:00 2001 From: Yang Zhong Date: Thu, 20 Jan 2022 17:31:04 -0500 Subject: [PATCH 04/31] qapi: Cleanup SGX related comments and restore @section-size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit a66bd91f030827742778a9e0da19fe55716b4a60 upstream. The SGX NUMA patches were merged into Qemu 7.0 release, we need clarify detailed version history information and also change some related comments, which make SGX related comments clearer. The QMP command schema promises backwards compatibility as standard. We temporarily restore "@section-size", which can avoid incompatible API breakage. The "@section-size" will be deprecated in 7.2 version. Intel-SIG: commit a66bd91f0308 ("qapi: Cleanup SGX related comments and restore @section-size"). Backport SGX NUMA support for qemu v6.2.0 Suggested-by: Daniel P. Berrangé Signed-off-by: Yang Zhong Reviewed-by: Daniel P. Berrangé Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20220120223104.437161-1-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- docs/about/deprecated.rst | 13 +++++++++++++ hw/i386/sgx.c | 11 +++++++++-- qapi/machine.json | 4 ++-- qapi/misc-target.json | 22 +++++++++++++++++----- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/docs/about/deprecated.rst b/docs/about/deprecated.rst index ff7488cb63b..33925edf450 100644 --- a/docs/about/deprecated.rst +++ b/docs/about/deprecated.rst @@ -270,6 +270,19 @@ accepted incorrect commands will return an error. Users should make sure that all arguments passed to ``device_add`` are consistent with the documented property types. +``query-sgx`` return value member ``section-size`` (since 7.0) +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +Member ``section-size`` in return value elements with meta-type ``uint64`` is +deprecated. Use ``sections`` instead. + + +``query-sgx-capabilities`` return value member ``section-size`` (since 7.0) +''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +Member ``section-size`` in return value elements with meta-type ``uint64`` is +deprecated. Use ``sections`` instead. + System accelerators ------------------- diff --git a/hw/i386/sgx.c b/hw/i386/sgx.c index 5de5dd08936..a2b318dd938 100644 --- a/hw/i386/sgx.c +++ b/hw/i386/sgx.c @@ -83,7 +83,7 @@ static uint64_t sgx_calc_section_metric(uint64_t low, uint64_t high) ((high & MAKE_64BIT_MASK(0, 20)) << 32); } -static SGXEPCSectionList *sgx_calc_host_epc_sections(void) +static SGXEPCSectionList *sgx_calc_host_epc_sections(uint64_t *size) { SGXEPCSectionList *head = NULL, **tail = &head; SGXEPCSection *section; @@ -106,6 +106,7 @@ static SGXEPCSectionList *sgx_calc_host_epc_sections(void) section = g_new0(SGXEPCSection, 1); section->node = j++; section->size = sgx_calc_section_metric(ecx, edx); + *size += section->size; QAPI_LIST_APPEND(tail, section); } @@ -156,6 +157,7 @@ SGXInfo *qmp_query_sgx_capabilities(Error **errp) { SGXInfo *info = NULL; uint32_t eax, ebx, ecx, edx; + uint64_t size = 0; int fd = qemu_open_old("/dev/sgx_vepc", O_RDWR); if (fd < 0) { @@ -173,7 +175,8 @@ SGXInfo *qmp_query_sgx_capabilities(Error **errp) info->sgx1 = eax & (1U << 0) ? true : false; info->sgx2 = eax & (1U << 1) ? true : false; - info->sections = sgx_calc_host_epc_sections(); + info->sections = sgx_calc_host_epc_sections(&size); + info->section_size = size; close(fd); @@ -220,12 +223,14 @@ SGXInfo *qmp_query_sgx(Error **errp) return NULL; } + SGXEPCState *sgx_epc = &pcms->sgx_epc; info = g_new0(SGXInfo, 1); info->sgx = true; info->sgx1 = true; info->sgx2 = true; info->flc = true; + info->section_size = sgx_epc->size; info->sections = sgx_get_epc_sections_list(); return info; @@ -249,6 +254,8 @@ void hmp_info_sgx(Monitor *mon, const QDict *qdict) info->sgx2 ? "enabled" : "disabled"); monitor_printf(mon, "FLC support: %s\n", info->flc ? "enabled" : "disabled"); + monitor_printf(mon, "size: %" PRIu64 "\n", + info->section_size); section_list = info->sections; for (section = section_list; section; section = section->next) { diff --git a/qapi/machine.json b/qapi/machine.json index 16e771affcf..a9f33d0f27d 100644 --- a/qapi/machine.json +++ b/qapi/machine.json @@ -1207,7 +1207,7 @@ # # @memdev: memory backend linked with device # -# @node: the numa node +# @node: the numa node (Since: 7.0) # # Since: 6.2 ## @@ -1288,7 +1288,7 @@ # # @memdev: memory backend linked with device # -# @node: the numa node +# @node: the numa node (Since: 7.0) # # Since: 6.2 ## diff --git a/qapi/misc-target.json b/qapi/misc-target.json index 1022aa0184c..4bc45d24741 100644 --- a/qapi/misc-target.json +++ b/qapi/misc-target.json @@ -344,9 +344,9 @@ # # @node: the numa node # -# @size: the size of epc section +# @size: the size of EPC section # -# Since: 6.2 +# Since: 7.0 ## { 'struct': 'SGXEPCSection', 'data': { 'node': 'int', @@ -365,7 +365,13 @@ # # @flc: true if FLC is supported # -# @sections: The EPC sections info for guest +# @section-size: The EPC section size for guest +# Redundant with @sections. Just for backward compatibility. +# +# @sections: The EPC sections info for guest (Since: 7.0) +# +# Features: +# @deprecated: Member @section-size is deprecated. Use @sections instead. # # Since: 6.2 ## @@ -374,6 +380,8 @@ 'sgx1': 'bool', 'sgx2': 'bool', 'flc': 'bool', + 'section-size': { 'type': 'uint64', + 'features': [ 'deprecated' ] }, 'sections': ['SGXEPCSection']}, 'if': 'TARGET_I386' } @@ -390,7 +398,9 @@ # # -> { "execute": "query-sgx" } # <- { "return": { "sgx": true, "sgx1" : true, "sgx2" : true, -# "flc": true, "section-size" : 0 } } +# "flc": true, "section-size" : 96468992, +# "sections": [{"node": 0, "size": 67108864}, +# {"node": 1, "size": 29360128}]} } # ## { 'command': 'query-sgx', 'returns': 'SGXInfo', 'if': 'TARGET_I386' } @@ -408,7 +418,9 @@ # # -> { "execute": "query-sgx-capabilities" } # <- { "return": { "sgx": true, "sgx1" : true, "sgx2" : true, -# "flc": true, "section-size" : 0 } } +# "flc": true, "section-size" : 96468992, +# "section" : [{"node": 0, "size": 67108864}, +# {"node": 1, "size": 29360128}]} } # ## { 'command': 'query-sgx-capabilities', 'returns': 'SGXInfo', 'if': 'TARGET_I386' } -- Gitee From 9c978b33f4e0a6d912a50057cae492edc7f189d3 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 22 Feb 2022 17:58:11 +0100 Subject: [PATCH 05/31] linux-headers: include missing changes from 5.17 commit 1ea5208febcc068449b63282d72bb719ab67a466 upstream. linux-headers: include missing changes from 5.17 Intel-SIG: commit 1ea5208febcc ("linux-headers: include missing changes from 5.17"). Backport AMX support for qemu v6.2.0 Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- linux-headers/asm-x86/kvm.h | 3 +++ linux-headers/linux/kvm.h | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h index a6c327f8ad9..2ab4f1818ae 100644 --- a/linux-headers/asm-x86/kvm.h +++ b/linux-headers/asm-x86/kvm.h @@ -437,6 +437,9 @@ struct kvm_sync_regs { #define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE 0x00000001 +/* attributes for system fd (group 0) */ +#define KVM_X86_XCOMP_GUEST_SUPP 0 + struct kvm_vmx_nested_state_data { __u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE]; __u8 shadow_vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE]; diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h index bcaf66cc4d2..f8fca4d916d 100644 --- a/linux-headers/linux/kvm.h +++ b/linux-headers/linux/kvm.h @@ -1112,6 +1112,10 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_BINARY_STATS_FD 203 #define KVM_CAP_EXIT_ON_EMULATION_FAILURE 204 #define KVM_CAP_ARM_MTE 205 +#define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206 +#define KVM_CAP_VM_GPA_BITS 207 +#define KVM_CAP_XSAVE2 208 +#define KVM_CAP_SYS_ATTRIBUTES 209 #ifdef KVM_CAP_IRQ_ROUTING @@ -2004,4 +2008,7 @@ struct kvm_stats_desc { #define KVM_GET_STATS_FD _IO(KVMIO, 0xce) +/* Available with KVM_CAP_XSAVE2 */ +#define KVM_GET_XSAVE2 _IOR(KVMIO, 0xcf, struct kvm_xsave) + #endif /* __LINUX_KVM_H */ -- Gitee From a929e0f59e38cba7427fc8768e1d3406110f4666 Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Wed, 16 Feb 2022 22:04:27 -0800 Subject: [PATCH 06/31] x86: Fix the 64-byte boundary enumeration for extended state commit 131266b7565bd437127bd231563572696bb27235 upstream. The extended state subleaves (EAX=0Dh, ECX=n, n>1).ECX[1] indicate whether the extended state component locates on the next 64-byte boundary following the preceding state component when the compacted format of an XSAVE area is used. Right now, they are all zero because no supported component needed the bit to be set, but the upcoming AMX feature will use it. Fix the subleaves value according to KVM's supported cpuid. Intel-SIG: commit 131266b7565b ("x86: Fix the 64-byte boundary enumeration for extended state"). Backport AMX support for qemu v6.2.0 Signed-off-by: Jing Liu Signed-off-by: Yang Zhong Message-Id: <20220217060434.52460-2-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/cpu.c | 1 + target/i386/cpu.h | 6 ++++++ target/i386/kvm/kvm-cpu.c | 1 + 3 files changed, 8 insertions(+) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index aa9e6368004..37f06b0b1a8 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -5487,6 +5487,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count, const ExtSaveArea *esa = &x86_ext_save_areas[count]; *eax = esa->size; *ebx = esa->offset; + *ecx = esa->ecx & ESA_FEATURE_ALIGN64_MASK; } } break; diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 04f2b790c9f..db0c3daae7b 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -548,6 +548,11 @@ typedef enum X86Seg { #define XSTATE_Hi16_ZMM_MASK (1ULL << XSTATE_Hi16_ZMM_BIT) #define XSTATE_PKRU_MASK (1ULL << XSTATE_PKRU_BIT) +#define ESA_FEATURE_ALIGN64_BIT 1 + +#define ESA_FEATURE_ALIGN64_MASK (1U << ESA_FEATURE_ALIGN64_BIT) + + /* CPUID feature words */ typedef enum FeatureWord { FEAT_1_EDX, /* CPUID[1].EDX */ @@ -1354,6 +1359,7 @@ QEMU_BUILD_BUG_ON(sizeof(XSavePKRU) != 0x8); typedef struct ExtSaveArea { uint32_t feature, bits; uint32_t offset, size; + uint32_t ecx; } ExtSaveArea; #define XSAVE_STATE_AREA_COUNT (XSTATE_PKRU_BIT + 1) diff --git a/target/i386/kvm/kvm-cpu.c b/target/i386/kvm/kvm-cpu.c index d95028018e8..ce27d3b1dff 100644 --- a/target/i386/kvm/kvm-cpu.c +++ b/target/i386/kvm/kvm-cpu.c @@ -104,6 +104,7 @@ static void kvm_cpu_xsave_init(void) if (sz != 0) { assert(esa->size == sz); esa->offset = kvm_arch_get_supported_cpuid(s, 0xd, i, R_EBX); + esa->ecx = kvm_arch_get_supported_cpuid(s, 0xd, i, R_ECX); } } } -- Gitee From 6908190b7bc082973966ac12e15337ffd6ff75a6 Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Wed, 16 Feb 2022 22:04:28 -0800 Subject: [PATCH 07/31] x86: Add AMX XTILECFG and XTILEDATA components commit 1f16764f7d4515bfd5e4ae0aae814fa280a7d0c8 upstream. The AMX TILECFG register and the TMMx tile data registers are saved/restored via XSAVE, respectively in state component 17 (64 bytes) and state component 18 (8192 bytes). Add AMX feature bits to x86_ext_save_areas array to set up AMX components. Add structs that define the layout of AMX XSAVE areas and use QEMU_BUILD_BUG_ON to validate the structs sizes. Intel-SIG: commit 1f16764f7d45 ("x86: Add AMX XTILECFG and XTILEDATA components"). Backport AMX support for qemu v6.2.0 Signed-off-by: Jing Liu Signed-off-by: Yang Zhong Message-Id: <20220217060434.52460-3-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/cpu.c | 8 ++++++++ target/i386/cpu.h | 18 +++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 37f06b0b1a8..ea7e8f90811 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -1401,6 +1401,14 @@ ExtSaveArea x86_ext_save_areas[XSAVE_STATE_AREA_COUNT] = { [XSTATE_PKRU_BIT] = { .feature = FEAT_7_0_ECX, .bits = CPUID_7_0_ECX_PKU, .size = sizeof(XSavePKRU) }, + [XSTATE_XTILE_CFG_BIT] = { + .feature = FEAT_7_0_EDX, .bits = CPUID_7_0_EDX_AMX_TILE, + .size = sizeof(XSaveXTILECFG), + }, + [XSTATE_XTILE_DATA_BIT] = { + .feature = FEAT_7_0_EDX, .bits = CPUID_7_0_EDX_AMX_TILE, + .size = sizeof(XSaveXTILEDATA) + }, }; static uint32_t xsave_area_size(uint64_t mask) diff --git a/target/i386/cpu.h b/target/i386/cpu.h index db0c3daae7b..9e65c78e4cc 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -537,6 +537,8 @@ typedef enum X86Seg { #define XSTATE_ZMM_Hi256_BIT 6 #define XSTATE_Hi16_ZMM_BIT 7 #define XSTATE_PKRU_BIT 9 +#define XSTATE_XTILE_CFG_BIT 17 +#define XSTATE_XTILE_DATA_BIT 18 #define XSTATE_FP_MASK (1ULL << XSTATE_FP_BIT) #define XSTATE_SSE_MASK (1ULL << XSTATE_SSE_BIT) @@ -845,6 +847,8 @@ typedef uint64_t FeatureWordArray[FEATURE_WORDS]; #define CPUID_7_0_EDX_TSX_LDTRK (1U << 16) /* AVX512_FP16 instruction */ #define CPUID_7_0_EDX_AVX512_FP16 (1U << 23) +/* AMX tile (two-dimensional register) */ +#define CPUID_7_0_EDX_AMX_TILE (1U << 24) /* Speculation Control */ #define CPUID_7_0_EDX_SPEC_CTRL (1U << 26) /* Single Thread Indirect Branch Predictors */ @@ -1348,6 +1352,16 @@ typedef struct XSavePKRU { uint32_t padding; } XSavePKRU; +/* Ext. save area 17: AMX XTILECFG state */ +typedef struct XSaveXTILECFG { + uint8_t xtilecfg[64]; +} XSaveXTILECFG; + +/* Ext. save area 18: AMX XTILEDATA state */ +typedef struct XSaveXTILEDATA { + uint8_t xtiledata[8][1024]; +} XSaveXTILEDATA; + QEMU_BUILD_BUG_ON(sizeof(XSaveAVX) != 0x100); QEMU_BUILD_BUG_ON(sizeof(XSaveBNDREG) != 0x40); QEMU_BUILD_BUG_ON(sizeof(XSaveBNDCSR) != 0x40); @@ -1355,6 +1369,8 @@ QEMU_BUILD_BUG_ON(sizeof(XSaveOpmask) != 0x40); QEMU_BUILD_BUG_ON(sizeof(XSaveZMM_Hi256) != 0x200); QEMU_BUILD_BUG_ON(sizeof(XSaveHi16_ZMM) != 0x400); QEMU_BUILD_BUG_ON(sizeof(XSavePKRU) != 0x8); +QEMU_BUILD_BUG_ON(sizeof(XSaveXTILECFG) != 0x40); +QEMU_BUILD_BUG_ON(sizeof(XSaveXTILEDATA) != 0x2000); typedef struct ExtSaveArea { uint32_t feature, bits; @@ -1362,7 +1378,7 @@ typedef struct ExtSaveArea { uint32_t ecx; } ExtSaveArea; -#define XSAVE_STATE_AREA_COUNT (XSTATE_PKRU_BIT + 1) +#define XSAVE_STATE_AREA_COUNT (XSTATE_XTILE_DATA_BIT + 1) extern ExtSaveArea x86_ext_save_areas[XSAVE_STATE_AREA_COUNT]; -- Gitee From b8a4938f813e1f3e7c7d33ec3bfa51d1ce0260c9 Mon Sep 17 00:00:00 2001 From: Yang Zhong Date: Wed, 16 Feb 2022 22:04:29 -0800 Subject: [PATCH 08/31] x86: Grant AMX permission for guest commit 19db68ca68a78fa033a21d419036b6e416554564 upstream. Kernel allocates 4K xstate buffer by default. For XSAVE features which require large state component (e.g. AMX), Linux kernel dynamically expands the xstate buffer only after the process has acquired the necessary permissions. Those are called dynamically- enabled XSAVE features (or dynamic xfeatures). There are separate permissions for native tasks and guests. Qemu should request the guest permissions for dynamic xfeatures which will be exposed to the guest. This only needs to be done once before the first vcpu is created. KVM implemented one new ARCH_GET_XCOMP_SUPP system attribute API to get host side supported_xcr0 and Qemu can decide if it can request dynamically enabled XSAVE features permission. https://lore.kernel.org/all/20220126152210.3044876-1-pbonzini@redhat.com/ Intel-SIG: commit 19db68ca68a7 ("x86: Grant AMX permission for guest"). Backport AMX support for qemu v6.2.0 Suggested-by: Paolo Bonzini Signed-off-by: Yang Zhong Signed-off-by: Jing Liu Message-Id: <20220217060434.52460-4-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/cpu.c | 7 +++++ target/i386/cpu.h | 4 +++ target/i386/kvm/kvm-cpu.c | 12 ++++---- target/i386/kvm/kvm.c | 57 ++++++++++++++++++++++++++++++++++++++ target/i386/kvm/kvm_i386.h | 1 + 5 files changed, 75 insertions(+), 6 deletions(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index ea7e8f90811..1d0c0060775 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -6006,6 +6006,7 @@ static void x86_cpu_enable_xsave_components(X86CPU *cpu) CPUX86State *env = &cpu->env; int i; uint64_t mask; + static bool request_perm; if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE)) { env->features[FEAT_XSAVE_COMP_LO] = 0; @@ -6021,6 +6022,12 @@ static void x86_cpu_enable_xsave_components(X86CPU *cpu) } } + /* Only request permission for first vcpu */ + if (kvm_enabled() && !request_perm) { + kvm_request_xsave_components(cpu, mask); + request_perm = true; + } + env->features[FEAT_XSAVE_COMP_LO] = mask; env->features[FEAT_XSAVE_COMP_HI] = mask >> 32; } diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 9e65c78e4cc..589fe39d035 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -549,6 +549,10 @@ typedef enum X86Seg { #define XSTATE_ZMM_Hi256_MASK (1ULL << XSTATE_ZMM_Hi256_BIT) #define XSTATE_Hi16_ZMM_MASK (1ULL << XSTATE_Hi16_ZMM_BIT) #define XSTATE_PKRU_MASK (1ULL << XSTATE_PKRU_BIT) +#define XSTATE_XTILE_CFG_MASK (1ULL << XSTATE_XTILE_CFG_BIT) +#define XSTATE_XTILE_DATA_MASK (1ULL << XSTATE_XTILE_DATA_BIT) + +#define XSTATE_DYNAMIC_MASK (XSTATE_XTILE_DATA_MASK) #define ESA_FEATURE_ALIGN64_BIT 1 diff --git a/target/i386/kvm/kvm-cpu.c b/target/i386/kvm/kvm-cpu.c index ce27d3b1dff..a35a1bf9fe4 100644 --- a/target/i386/kvm/kvm-cpu.c +++ b/target/i386/kvm/kvm-cpu.c @@ -84,7 +84,7 @@ static void kvm_cpu_max_instance_init(X86CPU *cpu) static void kvm_cpu_xsave_init(void) { static bool first = true; - KVMState *s = kvm_state; + uint32_t eax, ebx, ecx, edx; int i; if (!first) { @@ -100,11 +100,11 @@ static void kvm_cpu_xsave_init(void) ExtSaveArea *esa = &x86_ext_save_areas[i]; if (esa->size) { - int sz = kvm_arch_get_supported_cpuid(s, 0xd, i, R_EAX); - if (sz != 0) { - assert(esa->size == sz); - esa->offset = kvm_arch_get_supported_cpuid(s, 0xd, i, R_EBX); - esa->ecx = kvm_arch_get_supported_cpuid(s, 0xd, i, R_ECX); + host_cpuid(0xd, i, &eax, &ebx, &ecx, &edx); + if (eax != 0) { + assert(esa->size == eax); + esa->offset = ebx; + esa->ecx = ecx; } } } diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 5a698bde19a..e7f57d05a2c 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -17,6 +17,7 @@ #include "qapi/error.h" #include #include +#include #include #include "standard-headers/asm-x86/kvm_para.h" @@ -347,6 +348,7 @@ uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function, struct kvm_cpuid2 *cpuid; uint32_t ret = 0; uint32_t cpuid_1_edx; + uint64_t bitmask; cpuid = get_supported_cpuid(s); @@ -404,6 +406,25 @@ uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function, if (!has_msr_arch_capabs) { ret &= ~CPUID_7_0_EDX_ARCH_CAPABILITIES; } + } else if (function == 0xd && index == 0 && + (reg == R_EAX || reg == R_EDX)) { + struct kvm_device_attr attr = { + .group = 0, + .attr = KVM_X86_XCOMP_GUEST_SUPP, + .addr = (unsigned long) &bitmask + }; + + bool sys_attr = kvm_check_extension(s, KVM_CAP_SYS_ATTRIBUTES); + if (!sys_attr) { + warn_report("cannot get sys attribute capabilities %d", sys_attr); + } + + int rc = kvm_ioctl(s, KVM_GET_DEVICE_ATTR, &attr); + if (rc == -1 && (errno == ENXIO || errno == EINVAL)) { + warn_report("KVM_GET_DEVICE_ATTR(0, KVM_X86_XCOMP_GUEST_SUPP) " + "error: %d", rc); + } + ret = (reg == R_EAX) ? bitmask : bitmask >> 32; } else if (function == 0x80000001 && reg == R_ECX) { /* * It's safe to enable TOPOEXT even if it's not returned by @@ -5050,3 +5071,39 @@ bool kvm_arch_cpu_check_are_resettable(void) { return !sev_es_enabled(); } + +#define ARCH_REQ_XCOMP_GUEST_PERM 0x1025 + +void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask) +{ + KVMState *s = kvm_state; + uint64_t supported; + + mask &= XSTATE_DYNAMIC_MASK; + if (!mask) { + return; + } + /* + * Just ignore bits that are not in CPUID[EAX=0xD,ECX=0]. + * ARCH_REQ_XCOMP_GUEST_PERM would fail, and QEMU has warned + * about them already because they are not supported features. + */ + supported = kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EAX); + supported |= (uint64_t)kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EDX) << 32; + mask &= supported; + + while (mask) { + int bit = ctz64(mask); + int rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_GUEST_PERM, bit); + if (rc) { + /* + * Older kernel version (<5.17) do not support + * ARCH_REQ_XCOMP_GUEST_PERM, but also do not return + * any dynamic feature from kvm_arch_get_supported_cpuid. + */ + warn_report("prctl(ARCH_REQ_XCOMP_GUEST_PERM) failure " + "for feature bit %d", bit); + } + mask &= ~BIT_ULL(bit); + } +} diff --git a/target/i386/kvm/kvm_i386.h b/target/i386/kvm/kvm_i386.h index a978509d507..4124912c202 100644 --- a/target/i386/kvm/kvm_i386.h +++ b/target/i386/kvm/kvm_i386.h @@ -52,5 +52,6 @@ bool kvm_hyperv_expand_features(X86CPU *cpu, Error **errp); uint64_t kvm_swizzle_msi_ext_dest_id(uint64_t address); bool kvm_enable_sgx_provisioning(KVMState *s); +void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask); #endif -- Gitee From 8454161858acf24faef04e48ec4546b9c819bafd Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Wed, 16 Feb 2022 22:04:30 -0800 Subject: [PATCH 09/31] x86: Add XFD faulting bit for state components commit 0f17f6b30f3b051f0f96ccc98c9f7f395713699f upstream. Intel introduces XFD faulting mechanism for extended XSAVE features to dynamically enable the features in runtime. If CPUID (EAX=0Dh, ECX=n, n>1).ECX[2] is set as 1, it indicates support for XFD faulting of this state component. Intel-SIG: commit 0f17f6b30f3b ("x86: Add XFD faulting bit for state components"). Backport AMX support for qemu v6.2.0 Signed-off-by: Jing Liu Signed-off-by: Yang Zhong Message-Id: <20220217060434.52460-5-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/cpu.c | 3 ++- target/i386/cpu.h | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 1d0c0060775..464d2811c6b 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -5495,7 +5495,8 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count, const ExtSaveArea *esa = &x86_ext_save_areas[count]; *eax = esa->size; *ebx = esa->offset; - *ecx = esa->ecx & ESA_FEATURE_ALIGN64_MASK; + *ecx = esa->ecx & + (ESA_FEATURE_ALIGN64_MASK | ESA_FEATURE_XFD_MASK); } } break; diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 589fe39d035..78aac950cfb 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -555,8 +555,10 @@ typedef enum X86Seg { #define XSTATE_DYNAMIC_MASK (XSTATE_XTILE_DATA_MASK) #define ESA_FEATURE_ALIGN64_BIT 1 +#define ESA_FEATURE_XFD_BIT 2 #define ESA_FEATURE_ALIGN64_MASK (1U << ESA_FEATURE_ALIGN64_BIT) +#define ESA_FEATURE_XFD_MASK (1U << ESA_FEATURE_XFD_BIT) /* CPUID feature words */ -- Gitee From 59cd1f0bfcc93de768f2f6bd6967e65ce33e9c4f Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Wed, 16 Feb 2022 22:04:31 -0800 Subject: [PATCH 10/31] x86: Add AMX CPUIDs enumeration commit f21a48171cf3fa39532fc8553fd82e81b88b6474 upstream. Add AMX primary feature bits XFD and AMX_TILE to enumerate the CPU's AMX capability. Meanwhile, add AMX TILE and TMUL CPUID leaf and subleaves which exist when AMX TILE is present to provide the maximum capability of TILE and TMUL. Intel-SIG: commit f21a48171cf3 ("x86: Add AMX CPUIDs enumeration"). Backport AMX support for qemu v6.2.0 Signed-off-by: Jing Liu Signed-off-by: Yang Zhong Message-Id: <20220217060434.52460-6-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/cpu.c | 55 ++++++++++++++++++++++++++++++++++++++++--- target/i386/kvm/kvm.c | 4 +++- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 464d2811c6b..4ce2f12d0c4 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -574,6 +574,18 @@ static CPUCacheInfo legacy_l3_cache = { #define INTEL_PT_CYCLE_BITMAP 0x1fff /* Support 0,2^(0~11) */ #define INTEL_PT_PSB_BITMAP (0x003f << 16) /* Support 2K,4K,8K,16K,32K,64K */ +/* CPUID Leaf 0x1D constants: */ +#define INTEL_AMX_TILE_MAX_SUBLEAF 0x1 +#define INTEL_AMX_TOTAL_TILE_BYTES 0x2000 +#define INTEL_AMX_BYTES_PER_TILE 0x400 +#define INTEL_AMX_BYTES_PER_ROW 0x40 +#define INTEL_AMX_TILE_MAX_NAMES 0x8 +#define INTEL_AMX_TILE_MAX_ROWS 0x10 + +/* CPUID Leaf 0x1E constants: */ +#define INTEL_AMX_TMUL_MAX_K 0x10 +#define INTEL_AMX_TMUL_MAX_N 0x40 + void x86_cpu_vendor_words2str(char *dst, uint32_t vendor1, uint32_t vendor2, uint32_t vendor3) { @@ -843,8 +855,8 @@ FeatureWordInfo feature_word_info[FEATURE_WORDS] = { "avx512-vp2intersect", NULL, "md-clear", NULL, NULL, NULL, "serialize", NULL, "tsx-ldtrk", NULL, NULL /* pconfig */, NULL, - NULL, NULL, NULL, "avx512-fp16", - NULL, NULL, "spec-ctrl", "stibp", + NULL, NULL, "amx-bf16", "avx512-fp16", + "amx-tile", "amx-int8", "spec-ctrl", "stibp", NULL, "arch-capabilities", "core-capability", "ssbd", }, .cpuid = { @@ -909,7 +921,7 @@ FeatureWordInfo feature_word_info[FEATURE_WORDS] = { .type = CPUID_FEATURE_WORD, .feat_names = { "xsaveopt", "xsavec", "xgetbv1", "xsaves", - NULL, NULL, NULL, NULL, + "xfd", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, @@ -5585,6 +5597,43 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count, } break; } + case 0x1D: { + /* AMX TILE */ + *eax = 0; + *ebx = 0; + *ecx = 0; + *edx = 0; + if (!(env->features[FEAT_7_0_EDX] & CPUID_7_0_EDX_AMX_TILE)) { + break; + } + + if (count == 0) { + /* Highest numbered palette subleaf */ + *eax = INTEL_AMX_TILE_MAX_SUBLEAF; + } else if (count == 1) { + *eax = INTEL_AMX_TOTAL_TILE_BYTES | + (INTEL_AMX_BYTES_PER_TILE << 16); + *ebx = INTEL_AMX_BYTES_PER_ROW | (INTEL_AMX_TILE_MAX_NAMES << 16); + *ecx = INTEL_AMX_TILE_MAX_ROWS; + } + break; + } + case 0x1E: { + /* AMX TMUL */ + *eax = 0; + *ebx = 0; + *ecx = 0; + *edx = 0; + if (!(env->features[FEAT_7_0_EDX] & CPUID_7_0_EDX_AMX_TILE)) { + break; + } + + if (count == 0) { + /* Highest numbered palette subleaf */ + *ebx = INTEL_AMX_TMUL_MAX_K | (INTEL_AMX_TMUL_MAX_N << 8); + } + break; + } case 0x40000000: /* * CPUID code in kvm_arch_init_vcpu() ignores stuff diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index e7f57d05a2c..60ccdec5e80 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -1779,7 +1779,9 @@ int kvm_arch_init_vcpu(CPUState *cs) c = &cpuid_data.entries[cpuid_i++]; } break; - case 0x14: { + case 0x14: + case 0x1d: + case 0x1e: { uint32_t times; c->function = i; -- Gitee From 54e433638e3412dcfd66adb25e70cb169c37b9be Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Wed, 16 Feb 2022 22:04:32 -0800 Subject: [PATCH 11/31] x86: add support for KVM_CAP_XSAVE2 and AMX state migration commit e56dd3c70abb31893c61ac834109fa7a38841330 upstream. When dynamic xfeatures (e.g. AMX) are used by the guest, the xsave area would be larger than 4KB. KVM_GET_XSAVE2 and KVM_SET_XSAVE under KVM_CAP_XSAVE2 works with a xsave buffer larger than 4KB. Always use the new ioctls under KVM_CAP_XSAVE2 when KVM supports it. Intel-SIG: commit e56dd3c70abb ("x86: add support for KVM_CAP_XSAVE2 and AMX state migration"). Backport AMX support for qemu v6.2.0 Signed-off-by: Jing Liu Signed-off-by: Zeng Guang Signed-off-by: Wei Wang Signed-off-by: Yang Zhong Message-Id: <20220217060434.52460-7-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/cpu.h | 4 ++++ target/i386/kvm/kvm.c | 42 ++++++++++++++++++++++++-------------- target/i386/xsave_helper.c | 28 +++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 78aac950cfb..25a378a5adf 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -1522,6 +1522,10 @@ typedef struct CPUX86State { uint64_t opmask_regs[NB_OPMASK_REGS]; YMMReg zmmh_regs[CPU_NB_REGS]; ZMMReg hi16_zmm_regs[CPU_NB_REGS]; +#ifdef TARGET_X86_64 + uint8_t xtilecfg[64]; + uint8_t xtiledata[8192]; +#endif /* sysenter registers */ uint32_t sysenter_cs; diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 60ccdec5e80..b0b22dcf7c9 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -123,6 +123,7 @@ static uint32_t num_architectural_pmu_gp_counters; static uint32_t num_architectural_pmu_fixed_counters; static int has_xsave; +static int has_xsave2; static int has_xcrs; static int has_pit_state2; static int has_exception_payload; @@ -1585,6 +1586,26 @@ static Error *invtsc_mig_blocker; #define KVM_MAX_CPUID_ENTRIES 100 +static void kvm_init_xsave(CPUX86State *env) +{ + if (has_xsave2) { + env->xsave_buf_len = QEMU_ALIGN_UP(has_xsave2, 4096); + } else if (has_xsave) { + env->xsave_buf_len = sizeof(struct kvm_xsave); + } else { + return; + } + + env->xsave_buf = qemu_memalign(4096, env->xsave_buf_len); + memset(env->xsave_buf, 0, env->xsave_buf_len); + /* + * The allocated storage must be large enough for all of the + * possible XSAVE state components. + */ + assert(kvm_arch_get_supported_cpuid(kvm_state, 0xd, 0, R_ECX) <= + env->xsave_buf_len); +} + int kvm_arch_init_vcpu(CPUState *cs) { struct { @@ -1614,6 +1635,8 @@ int kvm_arch_init_vcpu(CPUState *cs) cpuid_i = 0; + has_xsave2 = kvm_check_extension(cs->kvm_state, KVM_CAP_XSAVE2); + r = kvm_arch_set_tsc_khz(cs); if (r < 0) { return r; @@ -2003,19 +2026,7 @@ int kvm_arch_init_vcpu(CPUState *cs) if (r) { goto fail; } - - if (has_xsave) { - env->xsave_buf_len = sizeof(struct kvm_xsave); - env->xsave_buf = qemu_memalign(4096, env->xsave_buf_len); - memset(env->xsave_buf, 0, env->xsave_buf_len); - - /* - * The allocated storage must be large enough for all of the - * possible XSAVE state components. - */ - assert(kvm_arch_get_supported_cpuid(kvm_state, 0xd, 0, R_ECX) - <= env->xsave_buf_len); - } + kvm_init_xsave(env); max_nested_state_len = kvm_max_nested_state_length(); if (max_nested_state_len > 0) { @@ -3263,13 +3274,14 @@ static int kvm_get_xsave(X86CPU *cpu) { CPUX86State *env = &cpu->env; void *xsave = env->xsave_buf; - int ret; + int type, ret; if (!has_xsave) { return kvm_get_fpu(cpu); } - ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_XSAVE, xsave); + type = has_xsave2 ? KVM_GET_XSAVE2 : KVM_GET_XSAVE; + ret = kvm_vcpu_ioctl(CPU(cpu), type, xsave); if (ret < 0) { return ret; } diff --git a/target/i386/xsave_helper.c b/target/i386/xsave_helper.c index ac61a963440..996e9f3bfef 100644 --- a/target/i386/xsave_helper.c +++ b/target/i386/xsave_helper.c @@ -126,6 +126,20 @@ void x86_cpu_xsave_all_areas(X86CPU *cpu, void *buf, uint32_t buflen) memcpy(pkru, &env->pkru, sizeof(env->pkru)); } + + e = &x86_ext_save_areas[XSTATE_XTILE_CFG_BIT]; + if (e->size && e->offset) { + XSaveXTILECFG *tilecfg = buf + e->offset; + + memcpy(tilecfg, &env->xtilecfg, sizeof(env->xtilecfg)); + } + + e = &x86_ext_save_areas[XSTATE_XTILE_DATA_BIT]; + if (e->size && e->offset && buflen >= e->size + e->offset) { + XSaveXTILEDATA *tiledata = buf + e->offset; + + memcpy(tiledata, &env->xtiledata, sizeof(env->xtiledata)); + } #endif } @@ -247,5 +261,19 @@ void x86_cpu_xrstor_all_areas(X86CPU *cpu, const void *buf, uint32_t buflen) pkru = buf + e->offset; memcpy(&env->pkru, pkru, sizeof(env->pkru)); } + + e = &x86_ext_save_areas[XSTATE_XTILE_CFG_BIT]; + if (e->size && e->offset) { + const XSaveXTILECFG *tilecfg = buf + e->offset; + + memcpy(&env->xtilecfg, tilecfg, sizeof(env->xtilecfg)); + } + + e = &x86_ext_save_areas[XSTATE_XTILE_DATA_BIT]; + if (e->size && e->offset && buflen >= e->size + e->offset) { + const XSaveXTILEDATA *tiledata = buf + e->offset; + + memcpy(&env->xtiledata, tiledata, sizeof(env->xtiledata)); + } #endif } -- Gitee From ef4db965b60a25476046febdc99b3b8c75bdf96b Mon Sep 17 00:00:00 2001 From: Zeng Guang Date: Wed, 16 Feb 2022 22:04:33 -0800 Subject: [PATCH 12/31] x86: Support XFD and AMX xsave data migration commit cdec2b753b487d9e8aab028231c35d87789ea083 upstream. XFD(eXtended Feature Disable) allows to enable a feature on xsave state while preventing specific user threads from using the feature. Support save and restore XFD MSRs if CPUID.D.1.EAX[4] enumerate to be valid. Likewise migrate the MSRs and related xsave state necessarily. Intel-SIG: commit cdec2b753b48 ("x86: Support XFD and AMX xsave data migration"). Backport AMX support for qemu v6.2.0 Signed-off-by: Zeng Guang Signed-off-by: Wei Wang Signed-off-by: Yang Zhong Message-Id: <20220217060434.52460-8-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/cpu.h | 9 +++++++++ target/i386/kvm/kvm.c | 18 +++++++++++++++++ target/i386/machine.c | 46 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 25a378a5adf..481b325de69 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -505,6 +505,9 @@ typedef enum X86Seg { #define MSR_VM_HSAVE_PA 0xc0010117 +#define MSR_IA32_XFD 0x000001c4 +#define MSR_IA32_XFD_ERR 0x000001c5 + #define MSR_IA32_BNDCFGS 0x00000d90 #define MSR_IA32_XSS 0x00000da0 #define MSR_IA32_UMWAIT_CONTROL 0xe1 @@ -870,6 +873,8 @@ typedef uint64_t FeatureWordArray[FEATURE_WORDS]; #define CPUID_7_1_EAX_AVX_VNNI (1U << 4) /* AVX512 BFloat16 Instruction */ #define CPUID_7_1_EAX_AVX512_BF16 (1U << 5) +/* XFD Extend Feature Disabled */ +#define CPUID_D_1_EAX_XFD (1U << 4) /* Packets which contain IP payload have LIP values */ #define CPUID_14_0_ECX_LIP (1U << 31) @@ -1611,6 +1616,10 @@ typedef struct CPUX86State { uint64_t msr_rtit_cr3_match; uint64_t msr_rtit_addrs[MAX_RTIT_ADDRS]; + /* Per-VCPU XFD MSRs */ + uint64_t msr_xfd; + uint64_t msr_xfd_err; + /* exception/interrupt handling */ int error_code; int exception_is_int; diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index b0b22dcf7c9..49fca5ea888 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -3219,6 +3219,13 @@ static int kvm_put_msrs(X86CPU *cpu, int level) env->msr_ia32_sgxlepubkeyhash[3]); } + if (env->features[FEAT_XSAVE] & CPUID_D_1_EAX_XFD) { + kvm_msr_entry_add(cpu, MSR_IA32_XFD, + env->msr_xfd); + kvm_msr_entry_add(cpu, MSR_IA32_XFD_ERR, + env->msr_xfd_err); + } + /* Note: MSR_IA32_FEATURE_CONTROL is written separately, see * kvm_put_msr_feature_control. */ } @@ -3570,6 +3577,11 @@ static int kvm_get_msrs(X86CPU *cpu) kvm_msr_entry_add(cpu, MSR_IA32_SGXLEPUBKEYHASH3, 0); } + if (env->features[FEAT_XSAVE] & CPUID_D_1_EAX_XFD) { + kvm_msr_entry_add(cpu, MSR_IA32_XFD, 0); + kvm_msr_entry_add(cpu, MSR_IA32_XFD_ERR, 0); + } + ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, cpu->kvm_msr_buf); if (ret < 0) { return ret; @@ -3866,6 +3878,12 @@ static int kvm_get_msrs(X86CPU *cpu) env->msr_ia32_sgxlepubkeyhash[index - MSR_IA32_SGXLEPUBKEYHASH0] = msrs[i].data; break; + case MSR_IA32_XFD: + env->msr_xfd = msrs[i].data; + break; + case MSR_IA32_XFD_ERR: + env->msr_xfd_err = msrs[i].data; + break; } } diff --git a/target/i386/machine.c b/target/i386/machine.c index 83c2b91529b..3977e9d8f8d 100644 --- a/target/i386/machine.c +++ b/target/i386/machine.c @@ -1455,6 +1455,48 @@ static const VMStateDescription vmstate_msr_intel_sgx = { } }; +static bool xfd_msrs_needed(void *opaque) +{ + X86CPU *cpu = opaque; + CPUX86State *env = &cpu->env; + + return !!(env->features[FEAT_XSAVE] & CPUID_D_1_EAX_XFD); +} + +static const VMStateDescription vmstate_msr_xfd = { + .name = "cpu/msr_xfd", + .version_id = 1, + .minimum_version_id = 1, + .needed = xfd_msrs_needed, + .fields = (VMStateField[]) { + VMSTATE_UINT64(env.msr_xfd, X86CPU), + VMSTATE_UINT64(env.msr_xfd_err, X86CPU), + VMSTATE_END_OF_LIST() + } +}; + +#ifdef TARGET_X86_64 +static bool amx_xtile_needed(void *opaque) +{ + X86CPU *cpu = opaque; + CPUX86State *env = &cpu->env; + + return !!(env->features[FEAT_7_0_EDX] & CPUID_7_0_EDX_AMX_TILE); +} + +static const VMStateDescription vmstate_amx_xtile = { + .name = "cpu/intel_amx_xtile", + .version_id = 1, + .minimum_version_id = 1, + .needed = amx_xtile_needed, + .fields = (VMStateField[]) { + VMSTATE_UINT8_ARRAY(env.xtilecfg, X86CPU, 64), + VMSTATE_UINT8_ARRAY(env.xtiledata, X86CPU, 8192), + VMSTATE_END_OF_LIST() + } +}; +#endif + const VMStateDescription vmstate_x86_cpu = { .name = "cpu", .version_id = 12, @@ -1593,6 +1635,10 @@ const VMStateDescription vmstate_x86_cpu = { #endif &vmstate_msr_tsx_ctrl, &vmstate_msr_intel_sgx, + &vmstate_msr_xfd, +#ifdef TARGET_X86_64 + &vmstate_amx_xtile, +#endif NULL } }; -- Gitee From e53dc47a92033f7209e3c7641614c4e68adc5a47 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 18 Mar 2022 16:23:47 +0100 Subject: [PATCH 13/31] target/i386: kvm: do not access uninitialized variable on older kernels commit 3ec5ad40081b14af28496198b4d08dbe13386790 upstream. KVM support for AMX includes a new system attribute, KVM_X86_XCOMP_GUEST_SUPP. Commit 19db68ca68 ("x86: Grant AMX permission for guest", 2022-03-15) however did not fully consider the behavior on older kernels. First, it warns too aggressively. Second, it invokes the KVM_GET_DEVICE_ATTR ioctl unconditionally and then uses the "bitmask" variable, which remains uninitialized if the ioctl fails. Third, kvm_ioctl returns -errno rather than -1 on errors. While at it, explain why the ioctl is needed and KVM_GET_SUPPORTED_CPUID is not enough. Intel-SIG: commit 3ec5ad40081b ("target/i386: kvm: do not access uninitialized variable on older kernels"). Backport AMX support for qemu v6.2.0 Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/kvm/kvm.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 49fca5ea888..20e418463d6 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -409,6 +409,12 @@ uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function, } } else if (function == 0xd && index == 0 && (reg == R_EAX || reg == R_EDX)) { + /* + * The value returned by KVM_GET_SUPPORTED_CPUID does not include + * features that still have to be enabled with the arch_prctl + * system call. QEMU needs the full value, which is retrieved + * with KVM_GET_DEVICE_ATTR. + */ struct kvm_device_attr attr = { .group = 0, .attr = KVM_X86_XCOMP_GUEST_SUPP, @@ -417,13 +423,16 @@ uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function, bool sys_attr = kvm_check_extension(s, KVM_CAP_SYS_ATTRIBUTES); if (!sys_attr) { - warn_report("cannot get sys attribute capabilities %d", sys_attr); + return ret; } int rc = kvm_ioctl(s, KVM_GET_DEVICE_ATTR, &attr); - if (rc == -1 && (errno == ENXIO || errno == EINVAL)) { - warn_report("KVM_GET_DEVICE_ATTR(0, KVM_X86_XCOMP_GUEST_SUPP) " - "error: %d", rc); + if (rc < 0) { + if (rc != -ENXIO) { + warn_report("KVM_GET_DEVICE_ATTR(0, KVM_X86_XCOMP_GUEST_SUPP) " + "error: %d", rc); + } + return ret; } ret = (reg == R_EAX) ? bitmask : bitmask >> 32; } else if (function == 0x80000001 && reg == R_ECX) { -- Gitee From 544c092b9231a38a8866178fbd86a2232f6ac86c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 23 Mar 2022 12:33:25 +0100 Subject: [PATCH 14/31] KVM: x86: workaround invalid CPUID[0xD,9] info on some AMD processors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 58f7db26f21c690cf9a669c314cfd7371506084a upstream. Some AMD processors expose the PKRU extended save state even if they do not have the related PKU feature in CPUID. Worse, when they do they report a size of 64, whereas the expected size of the PKRU extended save state is 8, therefore the esa->size == eax assertion does not hold. The state is already ignored by KVM_GET_SUPPORTED_CPUID because it was not enabled in the host XCR0. However, QEMU kvm_cpu_xsave_init() runs before QEMU invokes arch_prctl() to enable dynamically-enabled save states such as XTILEDATA, and KVM_GET_SUPPORTED_CPUID hides save states that have yet to be enabled. Therefore, kvm_cpu_xsave_init() needs to consult the host CPUID instead of KVM_GET_SUPPORTED_CPUID, and dies with an assertion failure. When setting up the ExtSaveArea array to match the host, ignore features that KVM does not report as supported. This will cause QEMU to skip the incorrect CPUID leaf instead of tripping the assertion. Intel-SIG: commit 58f7db26f21c ("KVM: x86: workaround invalid CPUID[0xD,9] info on some AMD processors"). Backport AMX support for qemu v6.2.0 Closes: https://gitlab.com/qemu-project/qemu/-/issues/916 Reported-by: Daniel P. Berrangé Analyzed-by: Yang Zhong Reported-by: Peter Krempa Tested-by: Daniel P. Berrangé Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/cpu.c | 4 ++-- target/i386/cpu.h | 2 ++ target/i386/kvm/kvm-cpu.c | 19 ++++++++++++------- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 4ce2f12d0c4..3cb518a3e05 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -4972,8 +4972,8 @@ CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp) return cpu_list; } -static uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, - bool migratable_only) +uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, + bool migratable_only) { FeatureWordInfo *wi = &feature_word_info[w]; uint64_t r = 0; diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 481b325de69..968a0164443 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -604,6 +604,8 @@ typedef enum FeatureWord { } FeatureWord; typedef uint64_t FeatureWordArray[FEATURE_WORDS]; +uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, + bool migratable_only); /* cpuid_features bits */ #define CPUID_FP87 (1U << 0) diff --git a/target/i386/kvm/kvm-cpu.c b/target/i386/kvm/kvm-cpu.c index a35a1bf9fe4..5eb955ce9a6 100644 --- a/target/i386/kvm/kvm-cpu.c +++ b/target/i386/kvm/kvm-cpu.c @@ -99,13 +99,18 @@ static void kvm_cpu_xsave_init(void) for (i = XSTATE_SSE_BIT + 1; i < XSAVE_STATE_AREA_COUNT; i++) { ExtSaveArea *esa = &x86_ext_save_areas[i]; - if (esa->size) { - host_cpuid(0xd, i, &eax, &ebx, &ecx, &edx); - if (eax != 0) { - assert(esa->size == eax); - esa->offset = ebx; - esa->ecx = ecx; - } + if (!esa->size) { + continue; + } + if ((x86_cpu_get_supported_feature_word(esa->feature, false) & esa->bits) + != esa->bits) { + continue; + } + host_cpuid(0xd, i, &eax, &ebx, &ecx, &edx); + if (eax != 0) { + assert(esa->size == eax); + esa->offset = ebx; + esa->ecx = ecx; } } } -- Gitee From fd80669fbd1053f6d5677b364908436a6d6c38ba Mon Sep 17 00:00:00 2001 From: Jason Zeng Date: Wed, 22 Feb 2023 13:59:37 +0800 Subject: [PATCH 15/31] Update linux headers to v6.0-rc4 commit d525f73f9186a5bc641b8caf0b2c9bb94e5aa963 upstream. commit 7e18e42e4b280c85b76967a9106a13ca61c16179 Intel-SIG: d525f73f9186 ("Update linux headers to v6.0-rc4"). Backport notify-vm-exit for qemu v6.2.0 Signed-off-by: Chenyi Qiang Reviewed-by: Cornelia Huck Message-Id: <20220915091035.3897-3-chenyi.qiang@intel.com> Signed-off-by: Thomas Huth [ jason: only include missing header changes for notify-vm-exit ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- linux-headers/asm-x86/kvm.h | 6 +++++- linux-headers/linux/kvm.h | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h index 2ab4f1818ae..46e730b62f5 100644 --- a/linux-headers/asm-x86/kvm.h +++ b/linux-headers/asm-x86/kvm.h @@ -324,6 +324,7 @@ struct kvm_reinject_control { #define KVM_VCPUEVENT_VALID_SHADOW 0x00000004 #define KVM_VCPUEVENT_VALID_SMM 0x00000008 #define KVM_VCPUEVENT_VALID_PAYLOAD 0x00000010 +#define KVM_VCPUEVENT_VALID_TRIPLE_FAULT 0x00000020 /* Interrupt shadow states */ #define KVM_X86_SHADOW_INT_MOV_SS 0x01 @@ -358,7 +359,10 @@ struct kvm_vcpu_events { __u8 smm_inside_nmi; __u8 latched_init; } smi; - __u8 reserved[27]; + struct { + __u8 pending; + } triple_fault; + __u8 reserved[26]; __u8 exception_has_payload; __u64 exception_payload; }; diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h index f8fca4d916d..f3d2c40598c 100644 --- a/linux-headers/linux/kvm.h +++ b/linux-headers/linux/kvm.h @@ -269,6 +269,7 @@ struct kvm_xen_exit { #define KVM_EXIT_AP_RESET_HOLD 32 #define KVM_EXIT_X86_BUS_LOCK 33 #define KVM_EXIT_XEN 34 +#define KVM_EXIT_NOTIFY 37 /* For KVM_EXIT_INTERNAL_ERROR */ /* Emulate instruction failed. */ @@ -469,6 +470,11 @@ struct kvm_run { } msr; /* KVM_EXIT_XEN */ struct kvm_xen_exit xen; + /* KVM_EXIT_NOTIFY */ + struct { +#define KVM_NOTIFY_CONTEXT_INVALID (1 << 0) + __u32 flags; + } notify; /* Fix the size of the union. */ char padding[256]; }; @@ -1116,6 +1122,8 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_VM_GPA_BITS 207 #define KVM_CAP_XSAVE2 208 #define KVM_CAP_SYS_ATTRIBUTES 209 +#define KVM_CAP_X86_TRIPLE_FAULT_EVENT 218 +#define KVM_CAP_X86_NOTIFY_VMEXIT 219 #ifdef KVM_CAP_IRQ_ROUTING @@ -2011,4 +2019,8 @@ struct kvm_stats_desc { /* Available with KVM_CAP_XSAVE2 */ #define KVM_GET_XSAVE2 _IOR(KVMIO, 0xcf, struct kvm_xsave) +/* Available with KVM_CAP_X86_NOTIFY_VMEXIT */ +#define KVM_X86_NOTIFY_VMEXIT_ENABLED (1ULL << 0) +#define KVM_X86_NOTIFY_VMEXIT_USER (1ULL << 1) + #endif /* __LINUX_KVM_H */ -- Gitee From 4824a265f03574ff726fc8de78b93ee7a2fa56ff Mon Sep 17 00:00:00 2001 From: Chenyi Qiang Date: Thu, 29 Sep 2022 15:20:11 +0800 Subject: [PATCH 16/31] i386: kvm: extend kvm_{get, put}_vcpu_events to support pending triple fault commit 12f89a39cf3c5760cba82ce68929d748961f62df upstream. For the direct triple faults, i.e. hardware detected and KVM morphed to VM-Exit, KVM will never lose them. But for triple faults sythesized by KVM, e.g. the RSM path, if KVM exits to userspace before the request is serviced, userspace could migrate the VM and lose the triple fault. A new flag KVM_VCPUEVENT_VALID_TRIPLE_FAULT is defined to signal that the event.triple_fault_pending field contains a valid state if the KVM_CAP_X86_TRIPLE_FAULT_EVENT capability is enabled. Intel-SIG: commit 12f89a39cf3c ("i386: kvm: extend kvm_{get, put}_vcpu_events to support pending triple fault"). Backport notify-vm-exit for qemu v6.2.0 Acked-by: Peter Xu Signed-off-by: Chenyi Qiang Message-Id: <20220929072014.20705-2-chenyi.qiang@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/cpu.c | 1 + target/i386/cpu.h | 1 + target/i386/kvm/kvm.c | 20 ++++++++++++++++++++ target/i386/machine.c | 20 ++++++++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 3cb518a3e05..431c72837de 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -5976,6 +5976,7 @@ static void x86_cpu_reset(DeviceState *dev) env->exception_has_payload = false; env->exception_payload = 0; env->nmi_injected = false; + env->triple_fault_pending = false; #if !defined(CONFIG_USER_ONLY) /* We hard-wire the BSP to the first CPU. */ apic_designate_bsp(cpu->apic_state, s->cpu_index == 0); diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 968a0164443..0b709da122c 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -1697,6 +1697,7 @@ typedef struct CPUX86State { uint8_t has_error_code; uint8_t exception_has_payload; uint64_t exception_payload; + uint8_t triple_fault_pending; uint32_t ins_len; uint32_t sipi_vector; bool tsc_valid; diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 20e418463d6..b2006dbdce2 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -127,6 +127,7 @@ static int has_xsave2; static int has_xcrs; static int has_pit_state2; static int has_exception_payload; +static int has_triple_fault_event; static bool has_msr_mcg_ext_ctl; @@ -2380,6 +2381,16 @@ int kvm_arch_init(MachineState *ms, KVMState *s) } } + has_triple_fault_event = kvm_check_extension(s, KVM_CAP_X86_TRIPLE_FAULT_EVENT); + if (has_triple_fault_event) { + ret = kvm_vm_enable_cap(s, KVM_CAP_X86_TRIPLE_FAULT_EVENT, 0, true); + if (ret < 0) { + error_report("kvm: Failed to enable triple fault event cap: %s", + strerror(-ret)); + return ret; + } + } + ret = kvm_get_supported_msrs(s); if (ret < 0) { return ret; @@ -4004,6 +4015,11 @@ static int kvm_put_vcpu_events(X86CPU *cpu, int level) } } + if (has_triple_fault_event) { + events.flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT; + events.triple_fault.pending = env->triple_fault_pending; + } + return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events); } @@ -4073,6 +4089,10 @@ static int kvm_get_vcpu_events(X86CPU *cpu) } } + if (events.flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) { + env->triple_fault_pending = events.triple_fault.pending; + } + env->sipi_vector = events.sipi_vector; return 0; diff --git a/target/i386/machine.c b/target/i386/machine.c index 3977e9d8f8d..41cf5c00534 100644 --- a/target/i386/machine.c +++ b/target/i386/machine.c @@ -1497,6 +1497,25 @@ static const VMStateDescription vmstate_amx_xtile = { }; #endif +static bool triple_fault_needed(void *opaque) +{ + X86CPU *cpu = opaque; + CPUX86State *env = &cpu->env; + + return env->triple_fault_pending; +} + +static const VMStateDescription vmstate_triple_fault = { + .name = "cpu/triple_fault", + .version_id = 1, + .minimum_version_id = 1, + .needed = triple_fault_needed, + .fields = (VMStateField[]) { + VMSTATE_UINT8(env.triple_fault_pending, X86CPU), + VMSTATE_END_OF_LIST() + } +}; + const VMStateDescription vmstate_x86_cpu = { .name = "cpu", .version_id = 12, @@ -1639,6 +1658,7 @@ const VMStateDescription vmstate_x86_cpu = { #ifdef TARGET_X86_64 &vmstate_amx_xtile, #endif + &vmstate_triple_fault, NULL } }; -- Gitee From 4850a2248a1036f63d0dc711862a7300b008a2db Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 29 Sep 2022 15:20:12 +0800 Subject: [PATCH 17/31] kvm: allow target-specific accelerator properties commit 3dba0a335cf5c53146b606be6ddfab4df81c464e upstream. Several hypervisor capabilities in KVM are target-specific. When exposed to QEMU users as accelerator properties (i.e. -accel kvm,prop=value), they should not be available for all targets. Add a hook for targets to add their own properties to -accel kvm, for now no such property is defined. Intel-SIG: commit 3dba0a335cf5 ("kvm: allow target-specific accelerator properties"). Backport notify-vm-exit for qemu v6.2.0 Signed-off-by: Paolo Bonzini Message-Id: <20220929072014.20705-3-chenyi.qiang@intel.com> Signed-off-by: Paolo Bonzini [ jason: remove changes in target/riscv/kvm.c since riscv kvm is not supported in qemu-6.2.0 and linux 5.10 ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- accel/kvm/kvm-all.c | 2 ++ include/sysemu/kvm.h | 2 ++ target/arm/kvm.c | 4 ++++ target/i386/kvm/kvm.c | 4 ++++ target/mips/kvm.c | 4 ++++ target/ppc/kvm.c | 4 ++++ target/s390x/kvm/kvm.c | 4 ++++ 7 files changed, 24 insertions(+) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index eecd8031cf6..9d1be509a26 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -3651,6 +3651,8 @@ static void kvm_accel_class_init(ObjectClass *oc, void *data) NULL, NULL); object_class_property_set_description(oc, "dirty-ring-size", "Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)"); + + kvm_arch_accel_class_init(oc); } static const TypeInfo kvm_accel_type = { diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h index 7b22aeb6ae1..d7ee0b6245c 100644 --- a/include/sysemu/kvm.h +++ b/include/sysemu/kvm.h @@ -333,6 +333,8 @@ bool kvm_device_supported(int vmfd, uint64_t type); extern const KVMCapabilityInfo kvm_arch_required_capabilities[]; +void kvm_arch_accel_class_init(ObjectClass *oc); + void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run); MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run); diff --git a/target/arm/kvm.c b/target/arm/kvm.c index bbf1ce7ba3b..0ca90a58a95 100644 --- a/target/arm/kvm.c +++ b/target/arm/kvm.c @@ -1057,3 +1057,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index b2006dbdce2..96b31f5735c 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -5168,3 +5168,7 @@ void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask) mask &= ~BIT_ULL(bit); } } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/mips/kvm.c b/target/mips/kvm.c index 086debd9f01..f80ac72dd18 100644 --- a/target/mips/kvm.c +++ b/target/mips/kvm.c @@ -1295,3 +1295,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/ppc/kvm.c b/target/ppc/kvm.c index dc93b99189e..da2dcea1e25 100644 --- a/target/ppc/kvm.c +++ b/target/ppc/kvm.c @@ -2959,3 +2959,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/s390x/kvm/kvm.c b/target/s390x/kvm/kvm.c index 5b1fdb55c47..671d0f179c7 100644 --- a/target/s390x/kvm/kvm.c +++ b/target/s390x/kvm/kvm.c @@ -2562,3 +2562,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} -- Gitee From 07d3e449422b64e7a766d41e7f952bc4c335041d Mon Sep 17 00:00:00 2001 From: Chenyi Qiang Date: Thu, 29 Sep 2022 15:20:13 +0800 Subject: [PATCH 18/31] kvm: expose struct KVMState commit 5f8a6bce1f1080058ed29d716cae81ea805142ae upstream. Expose struct KVMState out of kvm-all.c so that the field of struct KVMState can be accessed when defining target-specific accelerator properties. Intel-SIG: commit 5f8a6bce1f10 ("kvm: expose struct KVMState"). Backport notify-vm-exit for qemu v6.2.0 Signed-off-by: Chenyi Qiang Message-Id: <20220929072014.20705-4-chenyi.qiang@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- accel/kvm/kvm-all.c | 74 -------------------------------------- include/sysemu/kvm_int.h | 76 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 74 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 9d1be509a26..32aad7402fe 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -71,86 +71,12 @@ do { } while (0) #endif -#define KVM_MSI_HASHTAB_SIZE 256 - struct KVMParkedVcpu { unsigned long vcpu_id; int kvm_fd; QLIST_ENTRY(KVMParkedVcpu) node; }; -enum KVMDirtyRingReaperState { - KVM_DIRTY_RING_REAPER_NONE = 0, - /* The reaper is sleeping */ - KVM_DIRTY_RING_REAPER_WAIT, - /* The reaper is reaping for dirty pages */ - KVM_DIRTY_RING_REAPER_REAPING, -}; - -/* - * KVM reaper instance, responsible for collecting the KVM dirty bits - * via the dirty ring. - */ -struct KVMDirtyRingReaper { - /* The reaper thread */ - QemuThread reaper_thr; - volatile uint64_t reaper_iteration; /* iteration number of reaper thr */ - volatile enum KVMDirtyRingReaperState reaper_state; /* reap thr state */ -}; - -struct KVMState -{ - AccelState parent_obj; - - int nr_slots; - int fd; - int vmfd; - int coalesced_mmio; - int coalesced_pio; - struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; - bool coalesced_flush_in_progress; - int vcpu_events; - int robust_singlestep; - int debugregs; -#ifdef KVM_CAP_SET_GUEST_DEBUG - QTAILQ_HEAD(, kvm_sw_breakpoint) kvm_sw_breakpoints; -#endif - int max_nested_state_len; - int many_ioeventfds; - int intx_set_mask; - int kvm_shadow_mem; - bool kernel_irqchip_allowed; - bool kernel_irqchip_required; - OnOffAuto kernel_irqchip_split; - bool sync_mmu; - uint64_t manual_dirty_log_protect; - /* The man page (and posix) say ioctl numbers are signed int, but - * they're not. Linux, glibc and *BSD all treat ioctl numbers as - * unsigned, and treating them as signed here can break things */ - unsigned irq_set_ioctl; - unsigned int sigmask_len; - GHashTable *gsimap; -#ifdef KVM_CAP_IRQ_ROUTING - struct kvm_irq_routing *irq_routes; - int nr_allocated_irq_routes; - unsigned long *used_gsi_bitmap; - unsigned int gsi_count; - QTAILQ_HEAD(, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE]; -#endif - KVMMemoryListener memory_listener; - QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus; - - /* For "info mtree -f" to tell if an MR is registered in KVM */ - int nr_as; - struct KVMAs { - KVMMemoryListener *ml; - AddressSpace *as; - } *as; - uint64_t kvm_dirty_ring_bytes; /* Size of the per-vcpu dirty ring */ - uint32_t kvm_dirty_ring_size; /* Number of dirty GFNs per ring */ - struct KVMDirtyRingReaper reaper; -}; - KVMState *kvm_state; bool kvm_kernel_irqchip; bool kvm_split_irqchip; diff --git a/include/sysemu/kvm_int.h b/include/sysemu/kvm_int.h index 1f5487d9b74..3b4adcdc101 100644 --- a/include/sysemu/kvm_int.h +++ b/include/sysemu/kvm_int.h @@ -10,6 +10,7 @@ #define QEMU_KVM_INT_H #include "exec/memory.h" +#include "qapi/qapi-types-common.h" #include "qemu/accel.h" #include "sysemu/kvm.h" @@ -36,6 +37,81 @@ typedef struct KVMMemoryListener { int as_id; } KVMMemoryListener; +#define KVM_MSI_HASHTAB_SIZE 256 + +enum KVMDirtyRingReaperState { + KVM_DIRTY_RING_REAPER_NONE = 0, + /* The reaper is sleeping */ + KVM_DIRTY_RING_REAPER_WAIT, + /* The reaper is reaping for dirty pages */ + KVM_DIRTY_RING_REAPER_REAPING, +}; + +/* + * KVM reaper instance, responsible for collecting the KVM dirty bits + * via the dirty ring. + */ +struct KVMDirtyRingReaper { + /* The reaper thread */ + QemuThread reaper_thr; + volatile uint64_t reaper_iteration; /* iteration number of reaper thr */ + volatile enum KVMDirtyRingReaperState reaper_state; /* reap thr state */ +}; +struct KVMState +{ + AccelState parent_obj; + + int nr_slots; + int fd; + int vmfd; + int coalesced_mmio; + int coalesced_pio; + struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; + bool coalesced_flush_in_progress; + int vcpu_events; + int robust_singlestep; + int debugregs; +#ifdef KVM_CAP_SET_GUEST_DEBUG + QTAILQ_HEAD(, kvm_sw_breakpoint) kvm_sw_breakpoints; +#endif + int max_nested_state_len; + int many_ioeventfds; + int intx_set_mask; + int kvm_shadow_mem; + bool kernel_irqchip_allowed; + bool kernel_irqchip_required; + OnOffAuto kernel_irqchip_split; + bool sync_mmu; + uint64_t manual_dirty_log_protect; + /* The man page (and posix) say ioctl numbers are signed int, but + * they're not. Linux, glibc and *BSD all treat ioctl numbers as + * unsigned, and treating them as signed here can break things */ + unsigned irq_set_ioctl; + unsigned int sigmask_len; + GHashTable *gsimap; +#ifdef KVM_CAP_IRQ_ROUTING + struct kvm_irq_routing *irq_routes; + int nr_allocated_irq_routes; + unsigned long *used_gsi_bitmap; + unsigned int gsi_count; + QTAILQ_HEAD(, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE]; +#endif + KVMMemoryListener memory_listener; + QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus; + + /* For "info mtree -f" to tell if an MR is registered in KVM */ + int nr_as; + struct KVMAs { + KVMMemoryListener *ml; + AddressSpace *as; + } *as; + uint64_t kvm_dirty_ring_bytes; /* Size of the per-vcpu dirty ring */ + uint32_t kvm_dirty_ring_size; /* Number of dirty GFNs per ring */ + struct KVMDirtyRingReaper reaper; + NotifyVmexitOption notify_vmexit; + uint32_t notify_window; +}; + void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, AddressSpace *as, int as_id, const char *name); -- Gitee From a035732e1078e973d1a98a55fdf7cfd763c625f8 Mon Sep 17 00:00:00 2001 From: Chenyi Qiang Date: Thu, 29 Sep 2022 15:20:14 +0800 Subject: [PATCH 19/31] i386: add notify VM exit support commit e2e69f6bb907a70ac518230c54e98e7abcb0c911 upstream. There are cases that malicious virtual machine can cause CPU stuck (due to event windows don't open up), e.g., infinite loop in microcode when nested #AC (CVE-2015-5307). No event window means no event (NMI, SMI and IRQ) can be delivered. It leads the CPU to be unavailable to host or other VMs. Notify VM exit is introduced to mitigate such kind of attacks, which will generate a VM exit if no event window occurs in VM non-root mode for a specified amount of time (notify window). A new KVM capability KVM_CAP_X86_NOTIFY_VMEXIT is exposed to user space so that the user can query the capability and set the expected notify window when creating VMs. The format of the argument when enabling this capability is as follows: Bit 63:32 - notify window specified in qemu command Bit 31:0 - some flags (e.g. KVM_X86_NOTIFY_VMEXIT_ENABLED is set to enable the feature.) Users can configure the feature by a new (x86 only) accel property: qemu -accel kvm,notify-vmexit=run|internal-error|disable,notify-window=n The default option of notify-vmexit is run, which will enable the capability and do nothing if the exit happens. The internal-error option raises a KVM internal error if it happens. The disable option does not enable the capability. The default value of notify-window is 0. It is valid only when notify-vmexit is not disabled. The valid range of notify-window is non-negative. It is even safe to set it to zero since there's an internal hardware threshold to be added to ensure no false positive. Because a notify VM exit may happen with VM_CONTEXT_INVALID set in exit qualification (no cases are anticipated that would set this bit), which means VM context is corrupted. It would be reflected in the flags of KVM_EXIT_NOTIFY exit. If KVM_NOTIFY_CONTEXT_INVALID bit is set, raise a KVM internal error unconditionally. Intel-SIG: commit e2e69f6bb907 ("i386: add notify VM exit support"). Backport notify-vm-exit for qemu v6.2.0 Acked-by: Peter Xu Signed-off-by: Chenyi Qiang Message-Id: <20220929072014.20705-5-chenyi.qiang@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit message ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- accel/kvm/kvm-all.c | 2 + qapi/run-state.json | 17 ++++++++ qemu-options.hx | 11 +++++ target/i386/kvm/kvm.c | 98 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 32aad7402fe..cb3f1327307 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -3550,6 +3550,8 @@ static void kvm_accel_instance_init(Object *obj) s->kernel_irqchip_split = ON_OFF_AUTO_AUTO; /* KVM dirty ring is by default off */ s->kvm_dirty_ring_size = 0; + s->notify_vmexit = NOTIFY_VMEXIT_OPTION_RUN; + s->notify_window = 0; } static void kvm_accel_class_init(ObjectClass *oc, void *data) diff --git a/qapi/run-state.json b/qapi/run-state.json index 43d66d700fc..08c38b2c67b 100644 --- a/qapi/run-state.json +++ b/qapi/run-state.json @@ -638,3 +638,20 @@ { 'struct': 'MemoryFailureFlags', 'data': { 'action-required': 'bool', 'recursive': 'bool'} } + +## +# @NotifyVmexitOption: +# +# An enumeration of the options specified when enabling notify VM exit +# +# @run: enable the feature, do nothing and continue if the notify VM exit happens. +# +# @internal-error: enable the feature, raise a internal error if the notify +# VM exit happens. +# +# @disable: disable the feature. +# +# Since: 7.2 +## +{ 'enum': 'NotifyVmexitOption', + 'data': [ 'run', 'internal-error', 'disable' ] } \ No newline at end of file diff --git a/qemu-options.hx b/qemu-options.hx index 489b58e1511..ac61c051627 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -152,6 +152,7 @@ DEF("accel", HAS_ARG, QEMU_OPTION_accel, " split-wx=on|off (enable TCG split w^x mapping)\n" " tb-size=n (TCG translation block cache size)\n" " dirty-ring-size=n (KVM dirty ring GFN count, default 0)\n" + " notify-vmexit=run|internal-error|disable,notify-window=n (enable notify VM exit and set notify window, x86 only)\n" " thread=single|multi (enable multi-threaded TCG)\n", QEMU_ARCH_ALL) SRST ``-accel name[,prop=value[,...]]`` @@ -203,6 +204,16 @@ SRST is disabled (dirty-ring-size=0). When enabled, KVM will instead record dirty pages in a bitmap. + ``notify-vmexit=run|internal-error|disable,notify-window=n`` + Enables or disables notify VM exit support on x86 host and specify + the corresponding notify window to trigger the VM exit if enabled. + ``run`` option enables the feature. It does nothing and continue + if the exit happens. ``internal-error`` option enables the feature. + It raises a internal error. ``disable`` option doesn't enable the feature. + This feature can mitigate the CPU stuck issue due to event windows don't + open up for a specified of time (i.e. notify-window). + Default: notify-vmexit=run,notify-window=0. + ERST DEF("smp", HAS_ARG, QEMU_OPTION_smp, diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 96b31f5735c..697db955a2d 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -15,6 +15,7 @@ #include "qemu/osdep.h" #include "qapi/qapi-events-run-state.h" #include "qapi/error.h" +#include "qapi/visitor.h" #include #include #include @@ -2496,6 +2497,21 @@ int kvm_arch_init(MachineState *ms, KVMState *s) } } + if (s->notify_vmexit != NOTIFY_VMEXIT_OPTION_DISABLE && + kvm_check_extension(s, KVM_CAP_X86_NOTIFY_VMEXIT)) { + uint64_t notify_window_flags = + ((uint64_t)s->notify_window << 32) | + KVM_X86_NOTIFY_VMEXIT_ENABLED | + KVM_X86_NOTIFY_VMEXIT_USER; + ret = kvm_vm_enable_cap(s, KVM_CAP_X86_NOTIFY_VMEXIT, 0, + notify_window_flags); + if (ret < 0) { + error_report("kvm: Failed to enable notify vmexit cap: %s", + strerror(-ret)); + return ret; + } + } + return 0; } @@ -4839,6 +4855,9 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) X86CPU *cpu = X86_CPU(cs); uint64_t code; int ret; + bool ctx_invalid; + char str[256]; + KVMState *state; switch (run->exit_reason) { case KVM_EXIT_HLT: @@ -4894,6 +4913,21 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) /* already handled in kvm_arch_post_run */ ret = 0; break; + case KVM_EXIT_NOTIFY: + ctx_invalid = !!(run->notify.flags & KVM_NOTIFY_CONTEXT_INVALID); + state = KVM_STATE(current_accel()); + sprintf(str, "Encounter a notify exit with %svalid context in" + " guest. There can be possible misbehaves in guest." + " Please have a look.", ctx_invalid ? "in" : ""); + if (ctx_invalid || + state->notify_vmexit == NOTIFY_VMEXIT_OPTION_INTERNAL_ERROR) { + warn_report("KVM internal error: %s", str); + ret = -1; + } else { + warn_report_once("KVM: %s", str); + ret = 0; + } + break; default: fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason); ret = -1; @@ -5169,6 +5203,70 @@ void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask) } } +static int kvm_arch_get_notify_vmexit(Object *obj, Error **errp) +{ + KVMState *s = KVM_STATE(obj); + return s->notify_vmexit; +} + +static void kvm_arch_set_notify_vmexit(Object *obj, int value, Error **errp) +{ + KVMState *s = KVM_STATE(obj); + + if (s->fd != -1) { + error_setg(errp, "Cannot set properties after the accelerator has been initialized"); + return; + } + + s->notify_vmexit = value; +} + +static void kvm_arch_get_notify_window(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + uint32_t value = s->notify_window; + + visit_type_uint32(v, name, &value, errp); +} + +static void kvm_arch_set_notify_window(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + Error *error = NULL; + uint32_t value; + + if (s->fd != -1) { + error_setg(errp, "Cannot set properties after the accelerator has been initialized"); + return; + } + + visit_type_uint32(v, name, &value, &error); + if (error) { + error_propagate(errp, error); + return; + } + + s->notify_window = value; +} + void kvm_arch_accel_class_init(ObjectClass *oc) { + object_class_property_add_enum(oc, "notify-vmexit", "NotifyVMexitOption", + &NotifyVmexitOption_lookup, + kvm_arch_get_notify_vmexit, + kvm_arch_set_notify_vmexit); + object_class_property_set_description(oc, "notify-vmexit", + "Enable notify VM exit"); + + object_class_property_add(oc, "notify-window", "uint32", + kvm_arch_get_notify_window, + kvm_arch_set_notify_window, + NULL, NULL); + object_class_property_set_description(oc, "notify-window", + "Clock cycles without an event window " + "after which a notification VM exit occurs"); } -- Gitee From 6897c726d38f57b94b371d0be29b529a832fd6c1 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Mon, 14 Mar 2022 14:25:41 +0000 Subject: [PATCH 20/31] target/i386: Fix sanity check on max APIC ID / X2APIC enablement commit dc89f32d92bba795b0665f075b78d8881cf67ab3 upstream. The check on x86ms->apic_id_limit in pc_machine_done() had two problems. Firstly, we need KVM to support the X2APIC API in order to allow IRQ delivery to APICs >= 255. So we need to call/check kvm_enable_x2apic(), which was done elsewhere in *some* cases but not all. Secondly, microvm needs the same check. So move it from pc_machine_done() to x86_cpus_init() where it will work for both. The check in kvm_cpu_instance_init() is now redundant and can be dropped. Intel-SIG: commit dc89f32d92bb ("target/i386: Fix sanity check on max APIC ID / X2APIC enablement"). Backport bugfix of max APIC ID / X2APIC for qemu-6.2.0 Signed-off-by: David Woodhouse Acked-by: Claudio Fontana Message-Id: <20220314142544.150555-1-dwmw2@infradead.org> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin [ jason: amend commit log ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- hw/i386/pc.c | 8 -------- hw/i386/x86.c | 16 ++++++++++++++++ target/i386/kvm/kvm-cpu.c | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/hw/i386/pc.c b/hw/i386/pc.c index a2ef40ecbc2..9959f93216a 100644 --- a/hw/i386/pc.c +++ b/hw/i386/pc.c @@ -736,14 +736,6 @@ void pc_machine_done(Notifier *notifier, void *data) /* update FW_CFG_NB_CPUS to account for -device added CPUs */ fw_cfg_modify_i16(x86ms->fw_cfg, FW_CFG_NB_CPUS, x86ms->boot_cpus); } - - - if (x86ms->apic_id_limit > 255 && !xen_enabled() && - !kvm_irqchip_in_kernel()) { - error_report("current -smp configuration requires kernel " - "irqchip support."); - exit(EXIT_FAILURE); - } } void pc_guest_info_init(PCMachineState *pcms) diff --git a/hw/i386/x86.c b/hw/i386/x86.c index b84840a1bb9..f64639b8731 100644 --- a/hw/i386/x86.c +++ b/hw/i386/x86.c @@ -39,6 +39,7 @@ #include "sysemu/replay.h" #include "sysemu/sysemu.h" #include "sysemu/cpu-timers.h" +#include "sysemu/xen.h" #include "trace.h" #include "hw/i386/x86.h" @@ -136,6 +137,21 @@ void x86_cpus_init(X86MachineState *x86ms, int default_cpu_version) */ x86ms->apic_id_limit = x86_cpu_apic_id_from_index(x86ms, ms->smp.max_cpus - 1) + 1; + + /* + * Can we support APIC ID 255 or higher? + * + * Under Xen: yes. + * With userspace emulated lapic: no + * With KVM's in-kernel lapic: only if X2APIC API is enabled. + */ + if (x86ms->apic_id_limit > 255 && !xen_enabled() && + (!kvm_irqchip_in_kernel() || !kvm_enable_x2apic())) { + error_report("current -smp configuration requires kernel " + "irqchip and X2APIC API support."); + exit(EXIT_FAILURE); + } + possible_cpus = mc->possible_cpu_arch_ids(ms); for (i = 0; i < ms->smp.cpus; i++) { x86_cpu_new(x86ms, possible_cpus->cpus[i].arch_id, &error_fatal); diff --git a/target/i386/kvm/kvm-cpu.c b/target/i386/kvm/kvm-cpu.c index 5eb955ce9a6..7237378a7d4 100644 --- a/target/i386/kvm/kvm-cpu.c +++ b/target/i386/kvm/kvm-cpu.c @@ -171,7 +171,7 @@ static void kvm_cpu_instance_init(CPUState *cs) /* only applies to builtin_x86_defs cpus */ if (!kvm_irqchip_in_kernel()) { x86_cpu_change_kvm_default("x2apic", "off"); - } else if (kvm_irqchip_is_split() && kvm_enable_x2apic()) { + } else if (kvm_irqchip_is_split()) { x86_cpu_change_kvm_default("kvm-msi-ext-dest-id", "on"); } -- Gitee From 265778c806a35c37345519ba0cf1ce11a5259c5a Mon Sep 17 00:00:00 2001 From: Zeng Guang Date: Thu, 25 Aug 2022 10:52:46 +0800 Subject: [PATCH 21/31] target/i386: Set maximum APIC ID to KVM prior to vCPU creation commit 19e2a9fb9da067acba95b3be83588bda5a3f6a99 upstream. Specify maximum possible APIC ID assigned for current VM session to KVM prior to the creation of vCPUs. By this setting, KVM can set up VM-scoped data structure indexed by the APIC ID, e.g. Posted-Interrupt Descriptor pointer table to support Intel IPI virtualization, with the most optimal memory footprint. It can be achieved by calling KVM_ENABLE_CAP for KVM_CAP_MAX_VCPU_ID capability once KVM has enabled it. Ignoring the return error if KVM doesn't support this capability yet. Intel-SIG: commit 19e2a9fb9da0 ("target/i386: Set maximum APIC ID to KVM prior to vCPU creation"). Backport setting maximum APIC ID for qemu-6.2.0 Signed-off-by: Zeng Guang Acked-by: Peter Xu Acked-by: Michael S. Tsirkin Message-Id: <20220825025246.26618-1-guang.zeng@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit log ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- hw/i386/x86.c | 4 ++++ target/i386/kvm/kvm-stub.c | 5 +++++ target/i386/kvm/kvm.c | 5 +++++ target/i386/kvm/kvm_i386.h | 2 ++ 4 files changed, 16 insertions(+) diff --git a/hw/i386/x86.c b/hw/i386/x86.c index f64639b8731..a3258d78fac 100644 --- a/hw/i386/x86.c +++ b/hw/i386/x86.c @@ -152,6 +152,10 @@ void x86_cpus_init(X86MachineState *x86ms, int default_cpu_version) exit(EXIT_FAILURE); } + if (kvm_enabled()) { + kvm_set_max_apic_id(x86ms->apic_id_limit); + } + possible_cpus = mc->possible_cpu_arch_ids(ms); for (i = 0; i < ms->smp.cpus; i++) { x86_cpu_new(x86ms, possible_cpus->cpus[i].arch_id, &error_fatal); diff --git a/target/i386/kvm/kvm-stub.c b/target/i386/kvm/kvm-stub.c index f6e7e4466e1..e052f1c7b0e 100644 --- a/target/i386/kvm/kvm-stub.c +++ b/target/i386/kvm/kvm-stub.c @@ -44,3 +44,8 @@ bool kvm_hyperv_expand_features(X86CPU *cpu, Error **errp) { abort(); } + +void kvm_set_max_apic_id(uint32_t max_apic_id) +{ + return; +} diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 697db955a2d..4288a72abf4 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -5270,3 +5270,8 @@ void kvm_arch_accel_class_init(ObjectClass *oc) "Clock cycles without an event window " "after which a notification VM exit occurs"); } + +void kvm_set_max_apic_id(uint32_t max_apic_id) +{ + kvm_vm_enable_cap(kvm_state, KVM_CAP_MAX_VCPU_ID, 0, max_apic_id); +} diff --git a/target/i386/kvm/kvm_i386.h b/target/i386/kvm/kvm_i386.h index 4124912c202..58590138e5e 100644 --- a/target/i386/kvm/kvm_i386.h +++ b/target/i386/kvm/kvm_i386.h @@ -54,4 +54,6 @@ uint64_t kvm_swizzle_msi_ext_dest_id(uint64_t address); bool kvm_enable_sgx_provisioning(KVMState *s); void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask); +void kvm_set_max_apic_id(uint32_t max_apic_id); + #endif -- Gitee From 896d1fae937df626638f58050d59f137aba9cee9 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 9 Nov 2022 15:48:34 +1300 Subject: [PATCH 22/31] target/i386: Add SGX aex-notify and EDECCSSA support commit d45f24fe7525d8a8aaa4ca6d9d214dc41819caa5 upstream. The new SGX Asynchronous Exit (AEX) notification mechanism (AEX-notify) allows one enclave to receive a notification in the ERESUME after the enclave exit due to an AEX. EDECCSSA is a new SGX user leaf function (ENCLU[EDECCSSA]) to facilitate the AEX notification handling. Whether the hardware supports to create enclave with AEX-notify support is enumerated via CPUID.(EAX=0x12,ECX=0x1):EAX[10]. The new EDECCSSA user leaf function is enumerated via CPUID.(EAX=0x12,ECX=0x0):EAX[11]. Add support to allow to expose the new SGX AEX-notify feature and the new EDECCSSA user leaf function to KVM guest. Intel-SIG: commit d45f24fe7525 ("target/i386: Add SGX aex-notify and EDECCSSA support"). Backport SGX aex_notify and EDECCSSA support Link: https://lore.kernel.org/lkml/166760360549.4906.809756297092548496.tip-bot2@tip-bot2/ Link: https://lore.kernel.org/lkml/166760360934.4906.2427175408052308969.tip-bot2@tip-bot2/ Reviewed-by: Yang Zhong Signed-off-by: Kai Huang Message-Id: <20221109024834.172705-1-kai.huang@intel.com> Signed-off-by: Paolo Bonzini [ jason: amend commit log ] Signed-off-by: Jason Zeng Signed-off-by: Aubrey Li --- target/i386/cpu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 431c72837de..4f2db58d9e4 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -1204,7 +1204,7 @@ FeatureWordInfo feature_word_info[FEATURE_WORDS] = { .feat_names = { "sgx1", "sgx2", NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, + NULL, NULL, NULL, "sgx-edeccssa", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, @@ -1244,7 +1244,7 @@ FeatureWordInfo feature_word_info[FEATURE_WORDS] = { .feat_names = { NULL, "sgx-debug", "sgx-mode64", NULL, "sgx-provisionkey", "sgx-tokenkey", NULL, "sgx-kss", - NULL, NULL, NULL, NULL, + NULL, NULL, "sgx-aex-notify", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, -- Gitee From cb0308f8f2d9e86fa9fc8c37a6a37504d7d78acd Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 27 Feb 2023 10:57:09 +0100 Subject: [PATCH 23/31] target/i386: add FSRM to TCG commit c0728d4e3d23356691e4182eac54c67e1ca26618 upstream. Fast short REP MOVS can be added to TCG, since a trivial translation of string operation is a good option for short lengths. Intel-SIG: commit c0728d4e3d23 target/i386: add FSRM to TCG Backport to add new CPU model SapphireRapids and new fast string op leaves. Reviewed-by: Xiaoyao Li Signed-off-by: Paolo Bonzini [ Aichun Shi: amend commit log ] Signed-off-by: Aichun Shi Signed-off-by: Aubrey Li --- target/i386/cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 4f2db58d9e4..0d7c6c05343 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -660,7 +660,7 @@ void x86_cpu_vendor_words2str(char *dst, uint32_t vendor1, #define TCG_7_0_ECX_FEATURES (CPUID_7_0_ECX_PKU | \ /* CPUID_7_0_ECX_OSPKE is dynamic */ \ CPUID_7_0_ECX_LA57 | CPUID_7_0_ECX_PKS) -#define TCG_7_0_EDX_FEATURES 0 +#define TCG_7_0_EDX_FEATURES CPUID_7_0_EDX_FSRM #define TCG_7_1_EAX_FEATURES 0 #define TCG_APM_FEATURES 0 #define TCG_6_EAX_FEATURES CPUID_6_EAX_ARAT -- Gitee From 5406894e87b23e488d47c46eca5804f16f5939a5 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 27 Feb 2023 10:55:46 +0100 Subject: [PATCH 24/31] target/i386: add FZRM, FSRS, FSRC commit 58794f644e43ef8e60ed05395c58099311c1fcd1 upstream. These are three more markers for string operation optimizations. They can all be added to TCG, whose string operations are more or less as fast as they can be for short lengths. Intel-SIG: commit 58794f644e43 target/i386: add FZRM, FSRS, FSRC Backport to add new CPU model SapphireRapids and new fast string op leaves. Reviewed-by: Xiaoyao Li Signed-off-by: Paolo Bonzini [ Aichun Shi: amend commit log ] Signed-off-by: Aichun Shi Signed-off-by: Aubrey Li --- target/i386/cpu.c | 7 ++++--- target/i386/cpu.h | 7 +++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 0d7c6c05343..bea0115dfc9 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -661,7 +661,8 @@ void x86_cpu_vendor_words2str(char *dst, uint32_t vendor1, /* CPUID_7_0_ECX_OSPKE is dynamic */ \ CPUID_7_0_ECX_LA57 | CPUID_7_0_ECX_PKS) #define TCG_7_0_EDX_FEATURES CPUID_7_0_EDX_FSRM -#define TCG_7_1_EAX_FEATURES 0 +#define TCG_7_1_EAX_FEATURES (CPUID_7_1_EAX_FZRM | CPUID_7_1_EAX_FSRS | \ + CPUID_7_1_EAX_FSRC) #define TCG_APM_FEATURES 0 #define TCG_6_EAX_FEATURES CPUID_6_EAX_ARAT #define TCG_XSAVE_FEATURES (CPUID_XSAVE_XSAVEOPT | CPUID_XSAVE_XGETBV1) @@ -871,8 +872,8 @@ FeatureWordInfo feature_word_info[FEATURE_WORDS] = { .feat_names = { NULL, NULL, NULL, NULL, "avx-vnni", "avx512-bf16", NULL, NULL, - NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, + NULL, NULL, "fzrm", "fsrs", + "fsrc", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 0b709da122c..c211fd54fec 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -875,6 +875,13 @@ uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, #define CPUID_7_1_EAX_AVX_VNNI (1U << 4) /* AVX512 BFloat16 Instruction */ #define CPUID_7_1_EAX_AVX512_BF16 (1U << 5) +/* Fast Zero REP MOVS */ +#define CPUID_7_1_EAX_FZRM (1U << 10) +/* Fast Short REP STOS */ +#define CPUID_7_1_EAX_FSRS (1U << 11) +/* Fast Short REP CMPS/SCAS */ +#define CPUID_7_1_EAX_FSRC (1U << 12) + /* XFD Extend Feature Disabled */ #define CPUID_D_1_EAX_XFD (1U << 4) -- Gitee From be64ca9d64a4565ae3ec20ad883c5c86eb31a3c4 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 27 Feb 2023 10:41:46 +0100 Subject: [PATCH 25/31] target/i386: KVM: allow fast string operations if host supports them commit 3023c9b4d1092eb27a523c08d9e78cbaec67b59b upstream. These are just a flag that documents the performance characteristic of an instruction; it needs no hypervisor support. So include them even if KVM does not show them. In particular, FZRM/FSRS/FSRC have only been added very recently, but they are available on Sapphire Rapids processors. Intel-SIG: commit 3023c9b4d109 target/i386: KVM: allow fast string operations if host supports them Backport to add new CPU model SapphireRapids and new fast string op leaves. Reviewed-by: Xiaoyao Li Signed-off-by: Paolo Bonzini [ Aichun Shi: amend commit log ] Signed-off-by: Aichun Shi Signed-off-by: Aubrey Li --- target/i386/kvm/kvm.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 4288a72abf4..e0c38dec605 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -350,7 +350,7 @@ uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function, { struct kvm_cpuid2 *cpuid; uint32_t ret = 0; - uint32_t cpuid_1_edx; + uint32_t cpuid_1_edx, unused; uint64_t bitmask; cpuid = get_supported_cpuid(s); @@ -397,10 +397,20 @@ uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function, } else if (function == 6 && reg == R_EAX) { ret |= CPUID_6_EAX_ARAT; /* safe to allow because of emulated APIC */ } else if (function == 7 && index == 0 && reg == R_EBX) { + /* Not new instructions, just an optimization. */ + uint32_t ebx; + host_cpuid(7, 0, &unused, &ebx, &unused, &unused); + ret |= ebx & CPUID_7_0_EBX_ERMS; + if (host_tsx_broken()) { ret &= ~(CPUID_7_0_EBX_RTM | CPUID_7_0_EBX_HLE); } } else if (function == 7 && index == 0 && reg == R_EDX) { + /* Not new instructions, just an optimization. */ + uint32_t edx; + host_cpuid(7, 0, &unused, &unused, &unused, &edx); + ret |= edx & CPUID_7_0_EDX_FSRM; + /* * Linux v4.17-v4.20 incorrectly return ARCH_CAPABILITIES on SVM hosts. * We can detect the bug by checking if MSR_IA32_ARCH_CAPABILITIES is @@ -409,6 +419,11 @@ uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function, if (!has_msr_arch_capabs) { ret &= ~CPUID_7_0_EDX_ARCH_CAPABILITIES; } + } else if (function == 7 && index == 1 && reg == R_EAX) { + /* Not new instructions, just an optimization. */ + uint32_t eax; + host_cpuid(7, 1, &eax, &unused, &unused, &unused); + ret |= eax & (CPUID_7_1_EAX_FZRM | CPUID_7_1_EAX_FSRS | CPUID_7_1_EAX_FSRC); } else if (function == 0xd && index == 0 && (reg == R_EAX || reg == R_EDX)) { /* -- Gitee From 6810e9f4cc91bc48bc1f66c9d12533de6c0965b5 Mon Sep 17 00:00:00 2001 From: "Wang, Lei" Date: Thu, 11 Aug 2022 22:57:51 -0700 Subject: [PATCH 26/31] i386: Add new CPU model SapphireRapids commit 7eb061b06e97af9a8da7f31b839d78997ae737fc upstream. The new CPU model mostly inherits features from Icelake-Server, while adding new features: - AMX (Advance Matrix eXtensions) - Bus Lock Debug Exception and new instructions: - AVX VNNI (Vector Neural Network Instruction): - VPDPBUS: Multiply and Add Unsigned and Signed Bytes - VPDPBUSDS: Multiply and Add Unsigned and Signed Bytes with Saturation - VPDPWSSD: Multiply and Add Signed Word Integers - VPDPWSSDS: Multiply and Add Signed Integers with Saturation - FP16: Replicates existing AVX512 computational SP (FP32) instructions using FP16 instead of FP32 for ~2X performance gain - SERIALIZE: Provide software with a simple way to force the processor to complete all modifications, faster, allowed in all privilege levels and not causing an unconditional VM exit - TSX Suspend Load Address Tracking: Allows programmers to choose which memory accesses do not need to be tracked in the TSX read set - AVX512_BF16: Vector Neural Network Instructions supporting BFLOAT16 inputs and conversion instructions from IEEE single precision - fast zero-length MOVSB (KVM doesn't support yet) - fast short STOSB (KVM doesn't support yet) - fast short CMPSB, SCASB (KVM doesn't support yet) Features that may be added in future versions: - CET (virtualization support hasn't been merged) Intel-SIG: commit 7eb061b06e97 i386: Add new CPU model SapphireRapids Backport to add new CPU model SapphireRapids and new fast string op leaves. Signed-off-by: Wang, Lei Reviewed-by: Robert Hoo Message-Id: <20220812055751.14553-1-lei4.wang@intel.com> Reviewed-by: Xiaoyao Li Signed-off-by: Paolo Bonzini [ Aichun Shi: amend commit log ] Signed-off-by: Aichun Shi Signed-off-by: Aubrey Li --- target/i386/cpu.c | 133 +++++++++++++++++++++++++++++++++++++++++++++- target/i386/cpu.h | 4 ++ 2 files changed, 135 insertions(+), 2 deletions(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index bea0115dfc9..8fa43b3ee75 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -3529,6 +3529,135 @@ static const X86CPUDefinition builtin_x86_defs[] = { { /* end of list */ } } }, + { + .name = "SapphireRapids", + .level = 0x20, + .vendor = CPUID_VENDOR_INTEL, + .family = 6, + .model = 143, + .stepping = 4, + /* + * please keep the ascending order so that we can have a clear view of + * bit position of each feature. + */ + .features[FEAT_1_EDX] = + CPUID_FP87 | CPUID_VME | CPUID_DE | CPUID_PSE | CPUID_TSC | + CPUID_MSR | CPUID_PAE | CPUID_MCE | CPUID_CX8 | CPUID_APIC | + CPUID_SEP | CPUID_MTRR | CPUID_PGE | CPUID_MCA | CPUID_CMOV | + CPUID_PAT | CPUID_PSE36 | CPUID_CLFLUSH | CPUID_MMX | CPUID_FXSR | + CPUID_SSE | CPUID_SSE2, + .features[FEAT_1_ECX] = + CPUID_EXT_SSE3 | CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSSE3 | + CPUID_EXT_FMA | CPUID_EXT_CX16 | CPUID_EXT_PCID | CPUID_EXT_SSE41 | + CPUID_EXT_SSE42 | CPUID_EXT_X2APIC | CPUID_EXT_MOVBE | + CPUID_EXT_POPCNT | CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_AES | + CPUID_EXT_XSAVE | CPUID_EXT_AVX | CPUID_EXT_F16C | CPUID_EXT_RDRAND, + .features[FEAT_8000_0001_EDX] = + CPUID_EXT2_SYSCALL | CPUID_EXT2_NX | CPUID_EXT2_PDPE1GB | + CPUID_EXT2_RDTSCP | CPUID_EXT2_LM, + .features[FEAT_8000_0001_ECX] = + CPUID_EXT3_LAHF_LM | CPUID_EXT3_ABM | CPUID_EXT3_3DNOWPREFETCH, + .features[FEAT_8000_0008_EBX] = + CPUID_8000_0008_EBX_WBNOINVD, + .features[FEAT_7_0_EBX] = + CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 | CPUID_7_0_EBX_HLE | + CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP | CPUID_7_0_EBX_BMI2 | + CPUID_7_0_EBX_ERMS | CPUID_7_0_EBX_INVPCID | CPUID_7_0_EBX_RTM | + CPUID_7_0_EBX_AVX512F | CPUID_7_0_EBX_AVX512DQ | + CPUID_7_0_EBX_RDSEED | CPUID_7_0_EBX_ADX | CPUID_7_0_EBX_SMAP | + CPUID_7_0_EBX_AVX512IFMA | CPUID_7_0_EBX_CLFLUSHOPT | + CPUID_7_0_EBX_CLWB | CPUID_7_0_EBX_AVX512CD | CPUID_7_0_EBX_SHA_NI | + CPUID_7_0_EBX_AVX512BW | CPUID_7_0_EBX_AVX512VL, + .features[FEAT_7_0_ECX] = + CPUID_7_0_ECX_AVX512_VBMI | CPUID_7_0_ECX_UMIP | CPUID_7_0_ECX_PKU | + CPUID_7_0_ECX_AVX512_VBMI2 | CPUID_7_0_ECX_GFNI | + CPUID_7_0_ECX_VAES | CPUID_7_0_ECX_VPCLMULQDQ | + CPUID_7_0_ECX_AVX512VNNI | CPUID_7_0_ECX_AVX512BITALG | + CPUID_7_0_ECX_AVX512_VPOPCNTDQ | CPUID_7_0_ECX_LA57 | + CPUID_7_0_ECX_RDPID | CPUID_7_0_ECX_BUS_LOCK_DETECT, + .features[FEAT_7_0_EDX] = + CPUID_7_0_EDX_FSRM | CPUID_7_0_EDX_SERIALIZE | + CPUID_7_0_EDX_TSX_LDTRK | CPUID_7_0_EDX_AMX_BF16 | + CPUID_7_0_EDX_AVX512_FP16 | CPUID_7_0_EDX_AMX_TILE | + CPUID_7_0_EDX_AMX_INT8 | CPUID_7_0_EDX_SPEC_CTRL | + CPUID_7_0_EDX_ARCH_CAPABILITIES | CPUID_7_0_EDX_SPEC_CTRL_SSBD, + .features[FEAT_ARCH_CAPABILITIES] = + MSR_ARCH_CAP_RDCL_NO | MSR_ARCH_CAP_IBRS_ALL | + MSR_ARCH_CAP_SKIP_L1DFL_VMENTRY | MSR_ARCH_CAP_MDS_NO | + MSR_ARCH_CAP_PSCHANGE_MC_NO | MSR_ARCH_CAP_TAA_NO, + .features[FEAT_XSAVE] = + CPUID_XSAVE_XSAVEOPT | CPUID_XSAVE_XSAVEC | + CPUID_XSAVE_XGETBV1 | CPUID_XSAVE_XSAVES | CPUID_D_1_EAX_XFD, + .features[FEAT_6_EAX] = + CPUID_6_EAX_ARAT, + .features[FEAT_7_1_EAX] = + CPUID_7_1_EAX_AVX_VNNI | CPUID_7_1_EAX_AVX512_BF16 | + CPUID_7_1_EAX_FZRM | CPUID_7_1_EAX_FSRS | CPUID_7_1_EAX_FSRC, + .features[FEAT_VMX_BASIC] = + MSR_VMX_BASIC_INS_OUTS | MSR_VMX_BASIC_TRUE_CTLS, + .features[FEAT_VMX_ENTRY_CTLS] = + VMX_VM_ENTRY_LOAD_DEBUG_CONTROLS | VMX_VM_ENTRY_IA32E_MODE | + VMX_VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | + VMX_VM_ENTRY_LOAD_IA32_PAT | VMX_VM_ENTRY_LOAD_IA32_EFER, + .features[FEAT_VMX_EPT_VPID_CAPS] = + MSR_VMX_EPT_EXECONLY | + MSR_VMX_EPT_PAGE_WALK_LENGTH_4 | MSR_VMX_EPT_PAGE_WALK_LENGTH_5 | + MSR_VMX_EPT_WB | MSR_VMX_EPT_2MB | MSR_VMX_EPT_1GB | + MSR_VMX_EPT_INVEPT | MSR_VMX_EPT_AD_BITS | + MSR_VMX_EPT_INVEPT_SINGLE_CONTEXT | MSR_VMX_EPT_INVEPT_ALL_CONTEXT | + MSR_VMX_EPT_INVVPID | MSR_VMX_EPT_INVVPID_SINGLE_ADDR | + MSR_VMX_EPT_INVVPID_SINGLE_CONTEXT | + MSR_VMX_EPT_INVVPID_ALL_CONTEXT | + MSR_VMX_EPT_INVVPID_SINGLE_CONTEXT_NOGLOBALS, + .features[FEAT_VMX_EXIT_CTLS] = + VMX_VM_EXIT_SAVE_DEBUG_CONTROLS | + VMX_VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | + VMX_VM_EXIT_ACK_INTR_ON_EXIT | VMX_VM_EXIT_SAVE_IA32_PAT | + VMX_VM_EXIT_LOAD_IA32_PAT | VMX_VM_EXIT_SAVE_IA32_EFER | + VMX_VM_EXIT_LOAD_IA32_EFER | VMX_VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, + .features[FEAT_VMX_MISC] = + MSR_VMX_MISC_STORE_LMA | MSR_VMX_MISC_ACTIVITY_HLT | + MSR_VMX_MISC_VMWRITE_VMEXIT, + .features[FEAT_VMX_PINBASED_CTLS] = + VMX_PIN_BASED_EXT_INTR_MASK | VMX_PIN_BASED_NMI_EXITING | + VMX_PIN_BASED_VIRTUAL_NMIS | VMX_PIN_BASED_VMX_PREEMPTION_TIMER | + VMX_PIN_BASED_POSTED_INTR, + .features[FEAT_VMX_PROCBASED_CTLS] = + VMX_CPU_BASED_VIRTUAL_INTR_PENDING | + VMX_CPU_BASED_USE_TSC_OFFSETING | VMX_CPU_BASED_HLT_EXITING | + VMX_CPU_BASED_INVLPG_EXITING | VMX_CPU_BASED_MWAIT_EXITING | + VMX_CPU_BASED_RDPMC_EXITING | VMX_CPU_BASED_RDTSC_EXITING | + VMX_CPU_BASED_CR3_LOAD_EXITING | VMX_CPU_BASED_CR3_STORE_EXITING | + VMX_CPU_BASED_CR8_LOAD_EXITING | VMX_CPU_BASED_CR8_STORE_EXITING | + VMX_CPU_BASED_TPR_SHADOW | VMX_CPU_BASED_VIRTUAL_NMI_PENDING | + VMX_CPU_BASED_MOV_DR_EXITING | VMX_CPU_BASED_UNCOND_IO_EXITING | + VMX_CPU_BASED_USE_IO_BITMAPS | VMX_CPU_BASED_MONITOR_TRAP_FLAG | + VMX_CPU_BASED_USE_MSR_BITMAPS | VMX_CPU_BASED_MONITOR_EXITING | + VMX_CPU_BASED_PAUSE_EXITING | + VMX_CPU_BASED_ACTIVATE_SECONDARY_CONTROLS, + .features[FEAT_VMX_SECONDARY_CTLS] = + VMX_SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | + VMX_SECONDARY_EXEC_ENABLE_EPT | VMX_SECONDARY_EXEC_DESC | + VMX_SECONDARY_EXEC_RDTSCP | + VMX_SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | + VMX_SECONDARY_EXEC_ENABLE_VPID | VMX_SECONDARY_EXEC_WBINVD_EXITING | + VMX_SECONDARY_EXEC_UNRESTRICTED_GUEST | + VMX_SECONDARY_EXEC_APIC_REGISTER_VIRT | + VMX_SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | + VMX_SECONDARY_EXEC_RDRAND_EXITING | + VMX_SECONDARY_EXEC_ENABLE_INVPCID | + VMX_SECONDARY_EXEC_ENABLE_VMFUNC | VMX_SECONDARY_EXEC_SHADOW_VMCS | + VMX_SECONDARY_EXEC_RDSEED_EXITING | VMX_SECONDARY_EXEC_ENABLE_PML | + VMX_SECONDARY_EXEC_XSAVES, + .features[FEAT_VMX_VMFUNC] = + MSR_VMX_VMFUNC_EPT_SWITCHING, + .xlevel = 0x80000008, + .model_id = "Intel Xeon Processor (SapphireRapids)", + .versions = (X86CPUVersionDefinition[]) { + { .version = 1 }, + { /* end of list */ }, + }, + }, { .name = "Denverton", .level = 21, @@ -5599,7 +5728,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count, break; } case 0x1D: { - /* AMX TILE */ + /* AMX TILE, for now hardcoded for Sapphire Rapids*/ *eax = 0; *ebx = 0; *ecx = 0; @@ -5620,7 +5749,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count, break; } case 0x1E: { - /* AMX TMUL */ + /* AMX TMUL, for now hardcoded for Sapphire Rapids */ *eax = 0; *ebx = 0; *ecx = 0; diff --git a/target/i386/cpu.h b/target/i386/cpu.h index c211fd54fec..238d7f8656e 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -856,10 +856,14 @@ uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, #define CPUID_7_0_EDX_SERIALIZE (1U << 14) /* TSX Suspend Load Address Tracking instruction */ #define CPUID_7_0_EDX_TSX_LDTRK (1U << 16) +/* AMX_BF16 instruction */ +#define CPUID_7_0_EDX_AMX_BF16 (1U << 22) /* AVX512_FP16 instruction */ #define CPUID_7_0_EDX_AVX512_FP16 (1U << 23) /* AMX tile (two-dimensional register) */ #define CPUID_7_0_EDX_AMX_TILE (1U << 24) +/* AMX_INT8 instruction */ +#define CPUID_7_0_EDX_AMX_INT8 (1U << 25) /* Speculation Control */ #define CPUID_7_0_EDX_SPEC_CTRL (1U << 26) /* Single Thread Indirect Branch Predictors */ -- Gitee From 3a4afecb43fcd06246d45747065a7821996729a3 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 8 Nov 2021 13:38:58 +0100 Subject: [PATCH 27/31] configure, meson: move AVX tests to meson commit 622753d2fb501509ab03c241d476815f378d4ba5 upstream. For consistency with other tests, --enable-avx2 and --enable-avx512f fail to compile on x86 systems if cpuid.h is not available. Intel-SIG: commit 622753d2fb50 configure, meson: move AVX tests to meson Backport AVX512 support for xbzrle_encode_buffer. Reviewed-by: Richard Henderson Signed-off-by: Paolo Bonzini [ Aichun Shi: amend commit log ] Signed-off-by: Aichun Shi Signed-off-by: Aubrey Li --- configure | 103 ---------------------------------- meson.build | 50 ++++++++++++++++- meson_options.txt | 4 ++ scripts/meson-buildoptions.sh | 6 ++ 4 files changed, 58 insertions(+), 105 deletions(-) diff --git a/configure b/configure index 48c21775f3a..63f0ef926ce 100755 --- a/configure +++ b/configure @@ -330,8 +330,6 @@ qom_cast_debug="yes" trace_backends="log" trace_file="trace" opengl="$default_feature" -cpuid_h="no" -avx2_opt="$default_feature" guest_agent="$default_feature" guest_agent_with_vss="no" guest_agent_ntddscsi="no" @@ -1047,14 +1045,6 @@ for opt do ;; --disable-tools) want_tools="no" ;; - --disable-avx2) avx2_opt="no" - ;; - --enable-avx2) avx2_opt="yes" - ;; - --disable-avx512f) avx512f_opt="no" - ;; - --enable-avx512f) avx512f_opt="yes" - ;; --disable-virtio-blk-data-plane|--enable-virtio-blk-data-plane) echo "$0: $opt is obsolete, virtio-blk data-plane is always on" >&2 ;; @@ -1450,8 +1440,6 @@ cat << EOF tpm TPM support libssh ssh block device support numa libnuma support - avx2 AVX2 optimization support - avx512f AVX512F optimization support replication replication support opengl opengl support xfsctl xfsctl support @@ -2890,85 +2878,6 @@ else # "$safe_stack" = "" fi fi -######################################## -# check if cpuid.h is usable. - -cat > $TMPC << EOF -#include -int main(void) { - unsigned a, b, c, d; - int max = __get_cpuid_max(0, 0); - - if (max >= 1) { - __cpuid(1, a, b, c, d); - } - - if (max >= 7) { - __cpuid_count(7, 0, a, b, c, d); - } - - return 0; -} -EOF -if compile_prog "" "" ; then - cpuid_h=yes -fi - -########################################## -# avx2 optimization requirement check -# -# There is no point enabling this if cpuid.h is not usable, -# since we won't be able to select the new routines. - -if test "$cpuid_h" = "yes" && test "$avx2_opt" != "no"; then - cat > $TMPC << EOF -#pragma GCC push_options -#pragma GCC target("avx2") -#include -#include -static int bar(void *a) { - __m256i x = *(__m256i *)a; - return _mm256_testz_si256(x, x); -} -int main(int argc, char *argv[]) { return bar(argv[0]); } -EOF - if compile_object "-Werror" ; then - avx2_opt="yes" - else - avx2_opt="no" - fi -fi - -########################################## -# avx512f optimization requirement check -# -# There is no point enabling this if cpuid.h is not usable, -# since we won't be able to select the new routines. -# by default, it is turned off. -# if user explicitly want to enable it, check environment - -if test "$cpuid_h" = "yes" && test "$avx512f_opt" = "yes"; then - cat > $TMPC << EOF -#pragma GCC push_options -#pragma GCC target("avx512f") -#include -#include -static int bar(void *a) { - __m512i x = *(__m512i *)a; - return _mm512_test_epi64_mask(x, x); -} -int main(int argc, char *argv[]) -{ - return bar(argv[0]); -} -EOF - if ! compile_object "-Werror" ; then - avx512f_opt="no" - fi -else - avx512f_opt="no" -fi - ######################################## # check if __[u]int128_t is usable. @@ -3580,14 +3489,6 @@ if test "$opengl" = "yes" ; then echo "OPENGL_LIBS=$opengl_libs" >> $config_host_mak fi -if test "$avx2_opt" = "yes" ; then - echo "CONFIG_AVX2_OPT=y" >> $config_host_mak -fi - -if test "$avx512f_opt" = "yes" ; then - echo "CONFIG_AVX512F_OPT=y" >> $config_host_mak -fi - # XXX: suppress that if [ "$bsd" = "yes" ] ; then echo "CONFIG_BSD=y" >> $config_host_mak @@ -3620,10 +3521,6 @@ if test "$have_tsan" = "yes" && test "$have_tsan_iface_fiber" = "yes" ; then echo "CONFIG_TSAN=y" >> $config_host_mak fi -if test "$cpuid_h" = "yes" ; then - echo "CONFIG_CPUID_H=y" >> $config_host_mak -fi - if test "$int128" = "yes" ; then echo "CONFIG_INT128=y" >> $config_host_mak fi diff --git a/meson.build b/meson.build index 96de1a6ef94..36941a21627 100644 --- a/meson.build +++ b/meson.build @@ -1738,6 +1738,52 @@ config_host_data.set('CONFIG_GETAUXVAL', cc.links(gnu_source_prefix + ''' return getauxval(AT_HWCAP) == 0; }''')) +have_cpuid_h = cc.links(''' + #include + int main(void) { + unsigned a, b, c, d; + unsigned max = __get_cpuid_max(0, 0); + + if (max >= 1) { + __cpuid(1, a, b, c, d); + } + + if (max >= 7) { + __cpuid_count(7, 0, a, b, c, d); + } + + return 0; + }''') +config_host_data.set('CONFIG_CPUID_H', have_cpuid_h) + +config_host_data.set('CONFIG_AVX2_OPT', get_option('avx2') \ + .require(have_cpuid_h, error_message: 'cpuid.h not available, cannot enable AVX2') \ + .require(cc.links(''' + #pragma GCC push_options + #pragma GCC target("avx2") + #include + #include + static int bar(void *a) { + __m256i x = *(__m256i *)a; + return _mm256_testz_si256(x, x); + } + int main(int argc, char *argv[]) { return bar(argv[0]); } + '''), error_message: 'AVX2 not available').allowed()) + +config_host_data.set('CONFIG_AVX512F_OPT', get_option('avx512f') \ + .require(have_cpuid_h, error_message: 'cpuid.h not available, cannot enable AVX512F') \ + .require(cc.links(''' + #pragma GCC push_options + #pragma GCC target("avx512f") + #include + #include + static int bar(void *a) { + __m512i x = *(__m512i *)a; + return _mm512_test_epi64_mask(x, x); + } + int main(int argc, char *argv[]) { return bar(argv[0]); } + '''), error_message: 'AVX512F not available').allowed()) + config_host_data.set('CONFIG_AF_VSOCK', cc.compiles(gnu_source_prefix + ''' #include #include @@ -3256,8 +3302,8 @@ summary_info += {'membarrier': config_host.has_key('CONFIG_MEMBARRIER')} summary_info += {'debug stack usage': config_host.has_key('CONFIG_DEBUG_STACK_USAGE')} summary_info += {'mutex debugging': config_host.has_key('CONFIG_DEBUG_MUTEX')} summary_info += {'memory allocator': get_option('malloc')} -summary_info += {'avx2 optimization': config_host.has_key('CONFIG_AVX2_OPT')} -summary_info += {'avx512f optimization': config_host.has_key('CONFIG_AVX512F_OPT')} +summary_info += {'avx2 optimization': config_host_data.get('CONFIG_AVX2_OPT')} +summary_info += {'avx512f optimization': config_host_data.get('CONFIG_AVX512F_OPT')} summary_info += {'gprof enabled': config_host.has_key('CONFIG_GPROF')} summary_info += {'gcov': get_option('b_coverage')} summary_info += {'thread sanitizer': config_host.has_key('CONFIG_TSAN')} diff --git a/meson_options.txt b/meson_options.txt index e3923237322..e9cbe48cb90 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -66,6 +66,10 @@ option('cfi_debug', type: 'boolean', value: 'false', description: 'Verbose errors in case of CFI violation') option('multiprocess', type: 'feature', value: 'auto', description: 'Out of process device emulation support') +option('avx2', type: 'feature', value: 'auto', + description: 'AVX2 optimizations') +option('avx512f', type: 'feature', value: 'disabled', + description: 'AVX512F optimizations') option('attr', type : 'feature', value : 'auto', description: 'attr/xattr support') diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 7a17ff42182..b994bf16f0a 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -25,6 +25,8 @@ meson_options_help() { printf "%s\n" ' alsa ALSA sound support' printf "%s\n" ' attr attr/xattr support' printf "%s\n" ' auth-pam PAM access control' + printf "%s\n" ' avx2 AVX2 optimizations' + printf "%s\n" ' avx512f AVX512F optimizations' printf "%s\n" ' bpf eBPF support' printf "%s\n" ' brlapi brlapi character device driver' printf "%s\n" ' bzip2 bzip2 support for DMG images' @@ -107,6 +109,10 @@ _meson_option_parse() { --disable-attr) printf "%s" -Dattr=disabled ;; --enable-auth-pam) printf "%s" -Dauth_pam=enabled ;; --disable-auth-pam) printf "%s" -Dauth_pam=disabled ;; + --enable-avx2) printf "%s" -Davx2=enabled ;; + --disable-avx2) printf "%s" -Davx2=disabled ;; + --enable-avx512f) printf "%s" -Davx512f=enabled ;; + --disable-avx512f) printf "%s" -Davx512f=disabled ;; --enable-bpf) printf "%s" -Dbpf=enabled ;; --disable-bpf) printf "%s" -Dbpf=disabled ;; --enable-brlapi) printf "%s" -Dbrlapi=enabled ;; -- Gitee From 5cfb653a040f5549af71b984ed12ee437aea6d79 Mon Sep 17 00:00:00 2001 From: ling xu Date: Wed, 16 Nov 2022 23:29:22 +0800 Subject: [PATCH 28/31] AVX512 support for xbzrle_encode_buffer commit 04ffce137b6d85ab4e7687e54e4dffcef0a9ab99 upstream. This commit is the same with [PATCH v6 1/2], and provides avx512 support for xbzrle_encode_buffer function to accelerate xbzrle encoding speed. Runtime check of avx512 support and benchmark for this feature are added. Compared with C version of xbzrle_encode_buffer function, avx512 version can achieve 50%-70% performance improvement on benchmarking. In addition, if dirty data is randomly located in 4K page, the avx512 version can achieve almost 140% performance gain. Intel-SIG: commit 04ffce137b6d AVX512 support for xbzrle_encode_buffer Backport AVX512 support for xbzrle_encode_buffer. Signed-off-by: ling xu Co-authored-by: Zhou Zhao Co-authored-by: Jun Jin Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela [ Aichun Shi: amend commit log ] Signed-off-by: Aichun Shi Signed-off-by: Aubrey Li --- meson.build | 17 +++++ meson_options.txt | 2 + migration/ram.c | 34 +++++++++- migration/xbzrle.c | 124 ++++++++++++++++++++++++++++++++++ migration/xbzrle.h | 4 ++ scripts/meson-buildoptions.sh | 3 + 6 files changed, 181 insertions(+), 3 deletions(-) diff --git a/meson.build b/meson.build index 36941a21627..a68315ff736 100644 --- a/meson.build +++ b/meson.build @@ -1804,6 +1804,22 @@ config_host_data.set('CONFIG_AF_VSOCK', cc.compiles(gnu_source_prefix + ''' return -1; }''')) +config_host_data.set('CONFIG_AVX512BW_OPT', get_option('avx512bw') \ + .require(have_cpuid_h, error_message: 'cpuid.h not available, cannot enable AVX512BW') \ + .require(cc.links(''' + #pragma GCC push_options + #pragma GCC target("avx512bw") + #include + #include + static int bar(void *a) { + + __m512i *x = a; + __m512i res= _mm512_abs_epi8(*x); + return res[1]; + } + int main(int argc, char *argv[]) { return bar(argv[0]); } + '''), error_message: 'AVX512BW not available').allowed()) + ignored = ['CONFIG_QEMU_INTERP_PREFIX', # actually per-target 'HAVE_GDB_BIN'] arrays = ['CONFIG_BDRV_RW_WHITELIST', 'CONFIG_BDRV_RO_WHITELIST'] @@ -3303,6 +3319,7 @@ summary_info += {'debug stack usage': config_host.has_key('CONFIG_DEBUG_STACK_US summary_info += {'mutex debugging': config_host.has_key('CONFIG_DEBUG_MUTEX')} summary_info += {'memory allocator': get_option('malloc')} summary_info += {'avx2 optimization': config_host_data.get('CONFIG_AVX2_OPT')} +summary_info += {'avx512bw optimization': config_host_data.get('CONFIG_AVX512BW_OPT')} summary_info += {'avx512f optimization': config_host_data.get('CONFIG_AVX512F_OPT')} summary_info += {'gprof enabled': config_host.has_key('CONFIG_GPROF')} summary_info += {'gcov': get_option('b_coverage')} diff --git a/meson_options.txt b/meson_options.txt index e9cbe48cb90..ec9c3c0a05e 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -70,6 +70,8 @@ option('avx2', type: 'feature', value: 'auto', description: 'AVX2 optimizations') option('avx512f', type: 'feature', value: 'disabled', description: 'AVX512F optimizations') +option('avx512bw', type: 'feature', value: 'auto', + description: 'AVX512BW optimizations') option('attr', type : 'feature', value : 'auto', description: 'attr/xattr support') diff --git a/migration/ram.c b/migration/ram.c index 863035d2351..15b4e407232 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -86,6 +86,34 @@ static inline bool is_zero_range(uint8_t *p, uint64_t size) return buffer_is_zero(p, size); } +int (*xbzrle_encode_buffer_func)(uint8_t *, uint8_t *, int, + uint8_t *, int) = xbzrle_encode_buffer; +#if defined(CONFIG_AVX512BW_OPT) +#include "qemu/cpuid.h" +static void __attribute__((constructor)) init_cpu_flag(void) +{ + unsigned max = __get_cpuid_max(0, NULL); + int a, b, c, d; + if (max >= 1) { + __cpuid(1, a, b, c, d); + /* We must check that AVX is not just available, but usable. */ + if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) { + int bv; + __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0)); + __cpuid_count(7, 0, a, b, c, d); + /* 0xe6: + * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15 + * and ZMM16-ZMM31 state are enabled by OS) + * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS) + */ + if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) { + xbzrle_encode_buffer_func = xbzrle_encode_buffer_avx512; + } + } + } +} +#endif + XBZRLECacheStats xbzrle_counters; /* struct contains XBZRLE cache and a static page @@ -731,9 +759,9 @@ static int save_xbzrle_page(RAMState *rs, uint8_t **current_data, memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE); /* XBZRLE encoding (if there is no overflow) */ - encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf, - TARGET_PAGE_SIZE, XBZRLE.encoded_buf, - TARGET_PAGE_SIZE); + encoded_len = xbzrle_encode_buffer_func(prev_cached_page, XBZRLE.current_buf, + TARGET_PAGE_SIZE, XBZRLE.encoded_buf, + TARGET_PAGE_SIZE); /* * Update the cache contents, so that it corresponds to the data diff --git a/migration/xbzrle.c b/migration/xbzrle.c index 1ba482ded9c..05366e86c05 100644 --- a/migration/xbzrle.c +++ b/migration/xbzrle.c @@ -174,3 +174,127 @@ int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen) return d; } + +#if defined(CONFIG_AVX512BW_OPT) +#pragma GCC push_options +#pragma GCC target("avx512bw") +#include +int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, + uint8_t *dst, int dlen) +{ + uint32_t zrun_len = 0, nzrun_len = 0; + int d = 0, i = 0, num = 0; + uint8_t *nzrun_start = NULL; + /* add 1 to include residual part in main loop */ + uint32_t count512s = (slen >> 6) + 1; + /* countResidual is tail of data, i.e., countResidual = slen % 64 */ + uint32_t count_residual = slen & 0b111111; + bool never_same = true; + uint64_t mask_residual = 1; + mask_residual <<= count_residual; + mask_residual -= 1; + __m512i r = _mm512_set1_epi32(0); + + while (count512s) { + if (d + 2 > dlen) { + return -1; + } + + int bytes_to_check = 64; + uint64_t mask = 0xffffffffffffffff; + if (count512s == 1) { + bytes_to_check = count_residual; + mask = mask_residual; + } + __m512i old_data = _mm512_mask_loadu_epi8(r, + mask, old_buf + i); + __m512i new_data = _mm512_mask_loadu_epi8(r, + mask, new_buf + i); + uint64_t comp = _mm512_cmpeq_epi8_mask(old_data, new_data); + count512s--; + + bool is_same = (comp & 0x1); + while (bytes_to_check) { + if (is_same) { + if (nzrun_len) { + d += uleb128_encode_small(dst + d, nzrun_len); + if (d + nzrun_len > dlen) { + return -1; + } + nzrun_start = new_buf + i - nzrun_len; + memcpy(dst + d, nzrun_start, nzrun_len); + d += nzrun_len; + nzrun_len = 0; + } + /* 64 data at a time for speed */ + if (count512s && (comp == 0xffffffffffffffff)) { + i += 64; + zrun_len += 64; + break; + } + never_same = false; + num = __builtin_ctzll(~comp); + num = (num < bytes_to_check) ? num : bytes_to_check; + zrun_len += num; + bytes_to_check -= num; + comp >>= num; + i += num; + if (bytes_to_check) { + /* still has different data after same data */ + d += uleb128_encode_small(dst + d, zrun_len); + zrun_len = 0; + } else { + break; + } + } + if (never_same || zrun_len) { + /* + * never_same only acts if + * data begins with diff in first count512s + */ + d += uleb128_encode_small(dst + d, zrun_len); + zrun_len = 0; + never_same = false; + } + /* has diff, 64 data at a time for speed */ + if ((bytes_to_check == 64) && (comp == 0x0)) { + i += 64; + nzrun_len += 64; + break; + } + num = __builtin_ctzll(comp); + num = (num < bytes_to_check) ? num : bytes_to_check; + nzrun_len += num; + bytes_to_check -= num; + comp >>= num; + i += num; + if (bytes_to_check) { + /* mask like 111000 */ + d += uleb128_encode_small(dst + d, nzrun_len); + /* overflow */ + if (d + nzrun_len > dlen) { + return -1; + } + nzrun_start = new_buf + i - nzrun_len; + memcpy(dst + d, nzrun_start, nzrun_len); + d += nzrun_len; + nzrun_len = 0; + is_same = true; + } + } + } + + if (nzrun_len != 0) { + d += uleb128_encode_small(dst + d, nzrun_len); + /* overflow */ + if (d + nzrun_len > dlen) { + return -1; + } + nzrun_start = new_buf + i - nzrun_len; + memcpy(dst + d, nzrun_start, nzrun_len); + d += nzrun_len; + } + return d; +} +#pragma GCC pop_options +#endif diff --git a/migration/xbzrle.h b/migration/xbzrle.h index a0db507b9cd..6feb49160ad 100644 --- a/migration/xbzrle.h +++ b/migration/xbzrle.h @@ -18,4 +18,8 @@ int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen, uint8_t *dst, int dlen); int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen); +#if defined(CONFIG_AVX512BW_OPT) +int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, + uint8_t *dst, int dlen); +#endif #endif diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index b994bf16f0a..8c00cce4118 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -26,6 +26,7 @@ meson_options_help() { printf "%s\n" ' attr attr/xattr support' printf "%s\n" ' auth-pam PAM access control' printf "%s\n" ' avx2 AVX2 optimizations' + printf "%s\n" ' avx512bw AVX512BW optimizations' printf "%s\n" ' avx512f AVX512F optimizations' printf "%s\n" ' bpf eBPF support' printf "%s\n" ' brlapi brlapi character device driver' @@ -111,6 +112,8 @@ _meson_option_parse() { --disable-auth-pam) printf "%s" -Dauth_pam=disabled ;; --enable-avx2) printf "%s" -Davx2=enabled ;; --disable-avx2) printf "%s" -Davx2=disabled ;; + --enable-avx512bw) printf "%s" -Davx512bw=enabled ;; + --disable-avx512bw) printf "%s" -Davx512bw=disabled ;; --enable-avx512f) printf "%s" -Davx512f=enabled ;; --disable-avx512f) printf "%s" -Davx512f=disabled ;; --enable-bpf) printf "%s" -Dbpf=enabled ;; -- Gitee From 9c92755ffb1a52d7eaeb742e9817d2c07c557d07 Mon Sep 17 00:00:00 2001 From: ling xu Date: Wed, 16 Nov 2022 23:29:23 +0800 Subject: [PATCH 29/31] Update bench-code for addressing CI problem commit cc98c9fd5c17b8ab62ad91b183060d8f70b9d00d upstream. Unit test code is in test-xbzrle.c, and benchmark code is in xbzrle-bench.c for performance benchmarking. we have modified xbzrle-bench.c to address CI problem. Intel-SIG: commit cc98c9fd5c17 Update bench-code for addressing CI problem Backport AVX512 support for xbzrle_encode_buffer. Signed-off-by: ling xu Co-authored-by: Zhou Zhao Co-authored-by: Jun Jin Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela [ Aichun Shi: amend commit log ] Signed-off-by: Aichun Shi Signed-off-by: Aubrey Li --- tests/bench/meson.build | 6 + tests/bench/xbzrle-bench.c | 469 +++++++++++++++++++++++++++++++++++++ tests/unit/test-xbzrle.c | 39 ++- 3 files changed, 509 insertions(+), 5 deletions(-) create mode 100644 tests/bench/xbzrle-bench.c diff --git a/tests/bench/meson.build b/tests/bench/meson.build index 00b3c209dcb..54bc8938a8a 100644 --- a/tests/bench/meson.build +++ b/tests/bench/meson.build @@ -3,6 +3,12 @@ qht_bench = executable('qht-bench', sources: 'qht-bench.c', dependencies: [qemuutil]) +if have_system +xbzrle_bench = executable('xbzrle-bench', + sources: 'xbzrle-bench.c', + dependencies: [qemuutil,migration]) +endif + executable('atomic_add-bench', sources: files('atomic_add-bench.c'), dependencies: [qemuutil], diff --git a/tests/bench/xbzrle-bench.c b/tests/bench/xbzrle-bench.c new file mode 100644 index 00000000000..8848a3a32d7 --- /dev/null +++ b/tests/bench/xbzrle-bench.c @@ -0,0 +1,469 @@ +/* + * Xor Based Zero Run Length Encoding unit tests. + * + * Copyright 2013 Red Hat, Inc. and/or its affiliates + * + * Authors: + * Orit Wasserman + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ +#include "qemu/osdep.h" +#include "qemu/cutils.h" +#include "../migration/xbzrle.h" + +#if defined(CONFIG_AVX512BW_OPT) +#define XBZRLE_PAGE_SIZE 4096 +static bool is_cpu_support_avx512bw; +#include "qemu/cpuid.h" +static void __attribute__((constructor)) init_cpu_flag(void) +{ + unsigned max = __get_cpuid_max(0, NULL); + int a, b, c, d; + is_cpu_support_avx512bw = false; + if (max >= 1) { + __cpuid(1, a, b, c, d); + /* We must check that AVX is not just available, but usable. */ + if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) { + int bv; + __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0)); + __cpuid_count(7, 0, a, b, c, d); + /* 0xe6: + * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15 + * and ZMM16-ZMM31 state are enabled by OS) + * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS) + */ + if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) { + is_cpu_support_avx512bw = true; + } + } + } + return ; +} + +struct ResTime { + float t_raw; + float t_512; +}; + + +/* Function prototypes +int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, + uint8_t *dst, int dlen); +*/ +static void encode_decode_zero(struct ResTime *res) +{ + uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *compressed = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *compressed512 = g_malloc0(XBZRLE_PAGE_SIZE); + int i = 0; + int dlen = 0, dlen512 = 0; + int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1006); + + for (i = diff_len; i > 0; i--) { + buffer[1000 + i] = i; + buffer512[1000 + i] = i; + } + + buffer[1000 + diff_len + 3] = 103; + buffer[1000 + diff_len + 5] = 105; + + buffer512[1000 + diff_len + 3] = 103; + buffer512[1000 + diff_len + 5] = 105; + + /* encode zero page */ + time_t t_start, t_end, t_start512, t_end512; + t_start = clock(); + dlen = xbzrle_encode_buffer(buffer, buffer, XBZRLE_PAGE_SIZE, compressed, + XBZRLE_PAGE_SIZE); + t_end = clock(); + float time_val = difftime(t_end, t_start); + g_assert(dlen == 0); + + t_start512 = clock(); + dlen512 = xbzrle_encode_buffer_avx512(buffer512, buffer512, XBZRLE_PAGE_SIZE, + compressed512, XBZRLE_PAGE_SIZE); + t_end512 = clock(); + float time_val512 = difftime(t_end512, t_start512); + g_assert(dlen512 == 0); + + res->t_raw = time_val; + res->t_512 = time_val512; + + g_free(buffer); + g_free(compressed); + g_free(buffer512); + g_free(compressed512); + +} + +static void test_encode_decode_zero_avx512(void) +{ + int i; + float time_raw = 0.0, time_512 = 0.0; + struct ResTime res; + for (i = 0; i < 10000; i++) { + encode_decode_zero(&res); + time_raw += res.t_raw; + time_512 += res.t_512; + } + printf("Zero test:\n"); + printf("Raw xbzrle_encode time is %f ms\n", time_raw); + printf("512 xbzrle_encode time is %f ms\n", time_512); +} + +static void encode_decode_unchanged(struct ResTime *res) +{ + uint8_t *compressed = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *compressed512 = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE); + int i = 0; + int dlen = 0, dlen512 = 0; + int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1006); + + for (i = diff_len; i > 0; i--) { + test[1000 + i] = i + 4; + test512[1000 + i] = i + 4; + } + + test[1000 + diff_len + 3] = 107; + test[1000 + diff_len + 5] = 109; + + test512[1000 + diff_len + 3] = 107; + test512[1000 + diff_len + 5] = 109; + + /* test unchanged buffer */ + time_t t_start, t_end, t_start512, t_end512; + t_start = clock(); + dlen = xbzrle_encode_buffer(test, test, XBZRLE_PAGE_SIZE, compressed, + XBZRLE_PAGE_SIZE); + t_end = clock(); + float time_val = difftime(t_end, t_start); + g_assert(dlen == 0); + + t_start512 = clock(); + dlen512 = xbzrle_encode_buffer_avx512(test512, test512, XBZRLE_PAGE_SIZE, + compressed512, XBZRLE_PAGE_SIZE); + t_end512 = clock(); + float time_val512 = difftime(t_end512, t_start512); + g_assert(dlen512 == 0); + + res->t_raw = time_val; + res->t_512 = time_val512; + + g_free(test); + g_free(compressed); + g_free(test512); + g_free(compressed512); + +} + +static void test_encode_decode_unchanged_avx512(void) +{ + int i; + float time_raw = 0.0, time_512 = 0.0; + struct ResTime res; + for (i = 0; i < 10000; i++) { + encode_decode_unchanged(&res); + time_raw += res.t_raw; + time_512 += res.t_512; + } + printf("Unchanged test:\n"); + printf("Raw xbzrle_encode time is %f ms\n", time_raw); + printf("512 xbzrle_encode time is %f ms\n", time_512); +} + +static void encode_decode_1_byte(struct ResTime *res) +{ + uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *compressed = g_malloc(XBZRLE_PAGE_SIZE); + uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *compressed512 = g_malloc(XBZRLE_PAGE_SIZE); + int dlen = 0, rc = 0, dlen512 = 0, rc512 = 0; + uint8_t buf[2]; + uint8_t buf512[2]; + + test[XBZRLE_PAGE_SIZE - 1] = 1; + test512[XBZRLE_PAGE_SIZE - 1] = 1; + + time_t t_start, t_end, t_start512, t_end512; + t_start = clock(); + dlen = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE, compressed, + XBZRLE_PAGE_SIZE); + t_end = clock(); + float time_val = difftime(t_end, t_start); + g_assert(dlen == (uleb128_encode_small(&buf[0], 4095) + 2)); + + rc = xbzrle_decode_buffer(compressed, dlen, buffer, XBZRLE_PAGE_SIZE); + g_assert(rc == XBZRLE_PAGE_SIZE); + g_assert(memcmp(test, buffer, XBZRLE_PAGE_SIZE) == 0); + + t_start512 = clock(); + dlen512 = xbzrle_encode_buffer_avx512(buffer512, test512, XBZRLE_PAGE_SIZE, + compressed512, XBZRLE_PAGE_SIZE); + t_end512 = clock(); + float time_val512 = difftime(t_end512, t_start512); + g_assert(dlen512 == (uleb128_encode_small(&buf512[0], 4095) + 2)); + + rc512 = xbzrle_decode_buffer(compressed512, dlen512, buffer512, + XBZRLE_PAGE_SIZE); + g_assert(rc512 == XBZRLE_PAGE_SIZE); + g_assert(memcmp(test512, buffer512, XBZRLE_PAGE_SIZE) == 0); + + res->t_raw = time_val; + res->t_512 = time_val512; + + g_free(buffer); + g_free(compressed); + g_free(test); + g_free(buffer512); + g_free(compressed512); + g_free(test512); + +} + +static void test_encode_decode_1_byte_avx512(void) +{ + int i; + float time_raw = 0.0, time_512 = 0.0; + struct ResTime res; + for (i = 0; i < 10000; i++) { + encode_decode_1_byte(&res); + time_raw += res.t_raw; + time_512 += res.t_512; + } + printf("1 byte test:\n"); + printf("Raw xbzrle_encode time is %f ms\n", time_raw); + printf("512 xbzrle_encode time is %f ms\n", time_512); +} + +static void encode_decode_overflow(struct ResTime *res) +{ + uint8_t *compressed = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *compressed512 = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE); + int i = 0, rc = 0, rc512 = 0; + + for (i = 0; i < XBZRLE_PAGE_SIZE / 2 - 1; i++) { + test[i * 2] = 1; + test512[i * 2] = 1; + } + + /* encode overflow */ + time_t t_start, t_end, t_start512, t_end512; + t_start = clock(); + rc = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE, compressed, + XBZRLE_PAGE_SIZE); + t_end = clock(); + float time_val = difftime(t_end, t_start); + g_assert(rc == -1); + + t_start512 = clock(); + rc512 = xbzrle_encode_buffer_avx512(buffer512, test512, XBZRLE_PAGE_SIZE, + compressed512, XBZRLE_PAGE_SIZE); + t_end512 = clock(); + float time_val512 = difftime(t_end512, t_start512); + g_assert(rc512 == -1); + + res->t_raw = time_val; + res->t_512 = time_val512; + + g_free(buffer); + g_free(compressed); + g_free(test); + g_free(buffer512); + g_free(compressed512); + g_free(test512); + +} + +static void test_encode_decode_overflow_avx512(void) +{ + int i; + float time_raw = 0.0, time_512 = 0.0; + struct ResTime res; + for (i = 0; i < 10000; i++) { + encode_decode_overflow(&res); + time_raw += res.t_raw; + time_512 += res.t_512; + } + printf("Overflow test:\n"); + printf("Raw xbzrle_encode time is %f ms\n", time_raw); + printf("512 xbzrle_encode time is %f ms\n", time_512); +} + +static void encode_decode_range_avx512(struct ResTime *res) +{ + uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *compressed = g_malloc(XBZRLE_PAGE_SIZE); + uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *compressed512 = g_malloc(XBZRLE_PAGE_SIZE); + uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE); + int i = 0, rc = 0, rc512 = 0; + int dlen = 0, dlen512 = 0; + + int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1006); + + for (i = diff_len; i > 0; i--) { + buffer[1000 + i] = i; + test[1000 + i] = i + 4; + buffer512[1000 + i] = i; + test512[1000 + i] = i + 4; + } + + buffer[1000 + diff_len + 3] = 103; + test[1000 + diff_len + 3] = 107; + + buffer[1000 + diff_len + 5] = 105; + test[1000 + diff_len + 5] = 109; + + buffer512[1000 + diff_len + 3] = 103; + test512[1000 + diff_len + 3] = 107; + + buffer512[1000 + diff_len + 5] = 105; + test512[1000 + diff_len + 5] = 109; + + /* test encode/decode */ + time_t t_start, t_end, t_start512, t_end512; + t_start = clock(); + dlen = xbzrle_encode_buffer(test, buffer, XBZRLE_PAGE_SIZE, compressed, + XBZRLE_PAGE_SIZE); + t_end = clock(); + float time_val = difftime(t_end, t_start); + rc = xbzrle_decode_buffer(compressed, dlen, test, XBZRLE_PAGE_SIZE); + g_assert(rc < XBZRLE_PAGE_SIZE); + g_assert(memcmp(test, buffer, XBZRLE_PAGE_SIZE) == 0); + + t_start512 = clock(); + dlen512 = xbzrle_encode_buffer_avx512(test512, buffer512, XBZRLE_PAGE_SIZE, + compressed512, XBZRLE_PAGE_SIZE); + t_end512 = clock(); + float time_val512 = difftime(t_end512, t_start512); + rc512 = xbzrle_decode_buffer(compressed512, dlen512, test512, XBZRLE_PAGE_SIZE); + g_assert(rc512 < XBZRLE_PAGE_SIZE); + g_assert(memcmp(test512, buffer512, XBZRLE_PAGE_SIZE) == 0); + + res->t_raw = time_val; + res->t_512 = time_val512; + + g_free(buffer); + g_free(compressed); + g_free(test); + g_free(buffer512); + g_free(compressed512); + g_free(test512); + +} + +static void test_encode_decode_avx512(void) +{ + int i; + float time_raw = 0.0, time_512 = 0.0; + struct ResTime res; + for (i = 0; i < 10000; i++) { + encode_decode_range_avx512(&res); + time_raw += res.t_raw; + time_512 += res.t_512; + } + printf("Encode decode test:\n"); + printf("Raw xbzrle_encode time is %f ms\n", time_raw); + printf("512 xbzrle_encode time is %f ms\n", time_512); +} + +static void encode_decode_random(struct ResTime *res) +{ + uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *compressed = g_malloc(XBZRLE_PAGE_SIZE); + uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE); + uint8_t *compressed512 = g_malloc(XBZRLE_PAGE_SIZE); + uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE); + int i = 0, rc = 0, rc512 = 0; + int dlen = 0, dlen512 = 0; + + int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1); + /* store the index of diff */ + int dirty_index[diff_len]; + for (int j = 0; j < diff_len; j++) { + dirty_index[j] = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1); + } + for (i = diff_len - 1; i >= 0; i--) { + buffer[dirty_index[i]] = i; + test[dirty_index[i]] = i + 4; + buffer512[dirty_index[i]] = i; + test512[dirty_index[i]] = i + 4; + } + + time_t t_start, t_end, t_start512, t_end512; + t_start = clock(); + dlen = xbzrle_encode_buffer(test, buffer, XBZRLE_PAGE_SIZE, compressed, + XBZRLE_PAGE_SIZE); + t_end = clock(); + float time_val = difftime(t_end, t_start); + rc = xbzrle_decode_buffer(compressed, dlen, test, XBZRLE_PAGE_SIZE); + g_assert(rc < XBZRLE_PAGE_SIZE); + + t_start512 = clock(); + dlen512 = xbzrle_encode_buffer_avx512(test512, buffer512, XBZRLE_PAGE_SIZE, + compressed512, XBZRLE_PAGE_SIZE); + t_end512 = clock(); + float time_val512 = difftime(t_end512, t_start512); + rc512 = xbzrle_decode_buffer(compressed512, dlen512, test512, XBZRLE_PAGE_SIZE); + g_assert(rc512 < XBZRLE_PAGE_SIZE); + + res->t_raw = time_val; + res->t_512 = time_val512; + + g_free(buffer); + g_free(compressed); + g_free(test); + g_free(buffer512); + g_free(compressed512); + g_free(test512); + +} + +static void test_encode_decode_random_avx512(void) +{ + int i; + float time_raw = 0.0, time_512 = 0.0; + struct ResTime res; + for (i = 0; i < 10000; i++) { + encode_decode_random(&res); + time_raw += res.t_raw; + time_512 += res.t_512; + } + printf("Random test:\n"); + printf("Raw xbzrle_encode time is %f ms\n", time_raw); + printf("512 xbzrle_encode time is %f ms\n", time_512); +} +#endif + +int main(int argc, char **argv) +{ + g_test_init(&argc, &argv, NULL); + g_test_rand_int(); + #if defined(CONFIG_AVX512BW_OPT) + if (likely(is_cpu_support_avx512bw)) { + g_test_add_func("/xbzrle/encode_decode_zero", test_encode_decode_zero_avx512); + g_test_add_func("/xbzrle/encode_decode_unchanged", + test_encode_decode_unchanged_avx512); + g_test_add_func("/xbzrle/encode_decode_1_byte", test_encode_decode_1_byte_avx512); + g_test_add_func("/xbzrle/encode_decode_overflow", + test_encode_decode_overflow_avx512); + g_test_add_func("/xbzrle/encode_decode", test_encode_decode_avx512); + g_test_add_func("/xbzrle/encode_decode_random", test_encode_decode_random_avx512); + } + #endif + return g_test_run(); +} diff --git a/tests/unit/test-xbzrle.c b/tests/unit/test-xbzrle.c index 795d6f1cbab..baa364b443b 100644 --- a/tests/unit/test-xbzrle.c +++ b/tests/unit/test-xbzrle.c @@ -17,6 +17,35 @@ #define XBZRLE_PAGE_SIZE 4096 +int (*xbzrle_encode_buffer_func)(uint8_t *, uint8_t *, int, + uint8_t *, int) = xbzrle_encode_buffer; +#if defined(CONFIG_AVX512BW_OPT) +#include "qemu/cpuid.h" +static void __attribute__((constructor)) init_cpu_flag(void) +{ + unsigned max = __get_cpuid_max(0, NULL); + int a, b, c, d; + if (max >= 1) { + __cpuid(1, a, b, c, d); + /* We must check that AVX is not just available, but usable. */ + if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) { + int bv; + __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0)); + __cpuid_count(7, 0, a, b, c, d); + /* 0xe6: + * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15 + * and ZMM16-ZMM31 state are enabled by OS) + * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS) + */ + if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) { + xbzrle_encode_buffer_func = xbzrle_encode_buffer_avx512; + } + } + } + return ; +} +#endif + static void test_uleb(void) { uint32_t i, val; @@ -55,7 +84,7 @@ static void test_encode_decode_zero(void) buffer[1000 + diff_len + 5] = 105; /* encode zero page */ - dlen = xbzrle_encode_buffer(buffer, buffer, XBZRLE_PAGE_SIZE, compressed, + dlen = xbzrle_encode_buffer_func(buffer, buffer, XBZRLE_PAGE_SIZE, compressed, XBZRLE_PAGE_SIZE); g_assert(dlen == 0); @@ -79,7 +108,7 @@ static void test_encode_decode_unchanged(void) test[1000 + diff_len + 5] = 109; /* test unchanged buffer */ - dlen = xbzrle_encode_buffer(test, test, XBZRLE_PAGE_SIZE, compressed, + dlen = xbzrle_encode_buffer_func(test, test, XBZRLE_PAGE_SIZE, compressed, XBZRLE_PAGE_SIZE); g_assert(dlen == 0); @@ -97,7 +126,7 @@ static void test_encode_decode_1_byte(void) test[XBZRLE_PAGE_SIZE - 1] = 1; - dlen = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE, compressed, + dlen = xbzrle_encode_buffer_func(buffer, test, XBZRLE_PAGE_SIZE, compressed, XBZRLE_PAGE_SIZE); g_assert(dlen == (uleb128_encode_small(&buf[0], 4095) + 2)); @@ -122,7 +151,7 @@ static void test_encode_decode_overflow(void) } /* encode overflow */ - rc = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE, compressed, + rc = xbzrle_encode_buffer_func(buffer, test, XBZRLE_PAGE_SIZE, compressed, XBZRLE_PAGE_SIZE); g_assert(rc == -1); @@ -153,7 +182,7 @@ static void encode_decode_range(void) test[1000 + diff_len + 5] = 109; /* test encode/decode */ - dlen = xbzrle_encode_buffer(test, buffer, XBZRLE_PAGE_SIZE, compressed, + dlen = xbzrle_encode_buffer_func(test, buffer, XBZRLE_PAGE_SIZE, compressed, XBZRLE_PAGE_SIZE); rc = xbzrle_decode_buffer(compressed, dlen, test, XBZRLE_PAGE_SIZE); -- Gitee From 572b9085b15a117fe33298d30c897a828f11c07c Mon Sep 17 00:00:00 2001 From: Matheus Tavares Bernardino Date: Mon, 13 Mar 2023 15:58:19 -0300 Subject: [PATCH 30/31] migration/xbzrle: use ctz64 to avoid undefined result commit d84a78d15d3af9ff28ceec6906a4b101bd545b55 upstream. __builtin_ctzll() produces undefined results when the argument is 0. This can be seen through test-xbzrle, which produces the following warning: ../migration/xbzrle.c:265: runtime error: passing zero to ctz(), which is not a valid argument Replace __builtin_ctzll() with our ctz64() wrapper which properly handles 0. Intel-SIG: commit d84a78d15d3a migration/xbzrle: use ctz64 to avoid undefined result Backport AVX512 support for xbzrle_encode_buffer. Signed-off-by: Matheus Tavares Bernardino Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela [ Aichun Shi: amend commit log ] Signed-off-by: Aichun Shi Signed-off-by: Aubrey Li --- migration/xbzrle.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/migration/xbzrle.c b/migration/xbzrle.c index 05366e86c05..21b92d4eae1 100644 --- a/migration/xbzrle.c +++ b/migration/xbzrle.c @@ -12,6 +12,7 @@ */ #include "qemu/osdep.h" #include "qemu/cutils.h" +#include "qemu/host-utils.h" #include "xbzrle.h" /* @@ -233,7 +234,7 @@ int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, break; } never_same = false; - num = __builtin_ctzll(~comp); + num = ctz64(~comp); num = (num < bytes_to_check) ? num : bytes_to_check; zrun_len += num; bytes_to_check -= num; @@ -262,7 +263,7 @@ int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, nzrun_len += 64; break; } - num = __builtin_ctzll(comp); + num = ctz64(comp); num = (num < bytes_to_check) ? num : bytes_to_check; nzrun_len += num; bytes_to_check -= num; -- Gitee From 4b79931b6c2b86efa71abfed09a51aecf3e28e3c Mon Sep 17 00:00:00 2001 From: Matheus Tavares Bernardino Date: Mon, 13 Mar 2023 15:58:20 -0300 Subject: [PATCH 31/31] migration/xbzrle: fix out-of-bounds write with axv512 commit 1776b70f55c75541e9cab3423650a59b085162a9 upstream. xbzrle_encode_buffer_avx512() checks for overflows too scarcely in its outer loop, causing out-of-bounds writes: $ ../configure --target-list=aarch64-softmmu --enable-sanitizers --enable-avx512bw $ make tests/unit/test-xbzrle && ./tests/unit/test-xbzrle ==5518==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x62100000b100 at pc 0x561109a7714d bp 0x7ffed712a440 sp 0x7ffed712a430 WRITE of size 1 at 0x62100000b100 thread T0 #0 0x561109a7714c in uleb128_encode_small ../util/cutils.c:831 #1 0x561109b67f6a in xbzrle_encode_buffer_avx512 ../migration/xbzrle.c:275 #2 0x5611099a7428 in test_encode_decode_overflow ../tests/unit/test-xbzrle.c:153 #3 0x7fb2fb65a58d (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x7a58d) #4 0x7fb2fb65a333 (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x7a333) #5 0x7fb2fb65aa79 in g_test_run_suite (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x7aa79) #6 0x7fb2fb65aa94 in g_test_run (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x7aa94) #7 0x5611099a3a23 in main ../tests/unit/test-xbzrle.c:218 #8 0x7fb2fa78c082 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x24082) #9 0x5611099a608d in _start (/qemu/build/tests/unit/test-xbzrle+0x28408d) 0x62100000b100 is located 0 bytes to the right of 4096-byte region [0x62100000a100,0x62100000b100) allocated by thread T0 here: #0 0x7fb2fb823a06 in __interceptor_calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cc:153 #1 0x7fb2fb637ef0 in g_malloc0 (/lib/x86_64-linux-gnu/libglib-2.0.so.0+0x57ef0) Fix that by performing the overflow check in the inner loop, instead. Intel-SIG: commit 1776b70f55c7 migration/xbzrle: fix out-of-bounds write with axv512 Backport AVX512 support for xbzrle_encode_buffer. Signed-off-by: Matheus Tavares Bernardino Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela [ Aichun Shi: amend commit log ] Signed-off-by: Aichun Shi Signed-off-by: Aubrey Li --- migration/xbzrle.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/migration/xbzrle.c b/migration/xbzrle.c index 21b92d4eae1..c6f8b209175 100644 --- a/migration/xbzrle.c +++ b/migration/xbzrle.c @@ -197,10 +197,6 @@ int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, __m512i r = _mm512_set1_epi32(0); while (count512s) { - if (d + 2 > dlen) { - return -1; - } - int bytes_to_check = 64; uint64_t mask = 0xffffffffffffffff; if (count512s == 1) { @@ -216,6 +212,9 @@ int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, bool is_same = (comp & 0x1); while (bytes_to_check) { + if (d + 2 > dlen) { + return -1; + } if (is_same) { if (nzrun_len) { d += uleb128_encode_small(dst + d, nzrun_len); -- Gitee