diff --git a/usecases/libvirtd-offload/libvirt-6.2.0/README b/usecases/libvirtd-offload/libvirt-6.2.0/README new file mode 100644 index 0000000000000000000000000000000000000000..b3cf7d4b6e8f2fc13f19483958fefbce67f534ff --- /dev/null +++ b/usecases/libvirtd-offload/libvirt-6.2.0/README @@ -0,0 +1,84 @@ +使用说明 + +1.安装依赖 +yum groupinstall "Development tools" -y +yum install -y rpcgen +yum install -y python3-docutils +yum install -y glib2-devel +yum install -y gnutls-devel +yum install -y libxml2-devel +yum install -y libpciaccess-devel +yum install -y libtirpc-devel +yum install -y yajl-devel +yum install -y systemd-devel +yum install -y dmidecode +yum install -y glusterfs-api +yum install -y numactl +yum install -y vim +yum install -y meson +yum install -y qemu +yum install -y qemu-img +yum install -y strace +yum install -y edk2-aarch64 +yum install -y tar +yum install -y device-mapper-devel +yum install -y audit-devel +yum install -y libcap-ng-devel +yum install -y fuse-devel +yum install -y libpcap-devel +yum install -y libnl3-devel +yum install -y numactl-devel +yum install -y dbus-devel + +2.下载源码 +git config --global http.sslVerify false +git clone https://gitee.com/openeuler/libvirt.git -b libvirt-6.2.0 + +3.安装补丁，把patch放在与源码目录底下 +patch -p1 < libvirt-6.2.0-offload.patch + +4.配置编译选项 +mkdir pre-build +cd pre-build +../autogen.sh +cd .. +mkdir build +cd build +../configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var \ +--enable-dependency-tracking \ +--libdir=/usr/lib64 \ +--with-qemu \ +--without-libxl \ +--with-polkit \ +--with-libvirtd \ +--without-esx \ +--without-vz \ +--without-bhyve \ +--with-remote-default-mode=legacy \ +--with-interface \ +--with-network \ +--without-storage-vstorage \ +--with-numactl \ +--with-fuse \ +--without-selinux \ +--without-apparmor \ +--without-hal \ +--with-udev \ +--with-yajl \ +--with-libpcap \ +--with-macvtap \ +--with-audit \ +--with-driver-modules \ +--without-pm-utils \ +--with-nss-plugin \ +--disable-werror \ +--with-init-script=systemd \ +--with-capng \ +--without-netcf \ +--without-test + +5.编译 +make -j64 + +6.安装 +make install \ No newline at end of file diff --git a/usecases/libvirtd-offload/libvirt-6.2.0/libvirt-6.2.0-offload.patch b/usecases/libvirtd-offload/libvirt-6.2.0/libvirt-6.2.0-offload.patch index 31f03617430d97fb0510a4c73bf25ab778f9e4ff..7952999ca5336a80c73ca5579ae3f5ea6dbf74b4 100644 --- a/usecases/libvirtd-offload/libvirt-6.2.0/libvirt-6.2.0-offload.patch +++ b/usecases/libvirtd-offload/libvirt-6.2.0/libvirt-6.2.0-offload.patch @@ -22,36 +22,112 @@ index 82d720c..e52a10d 100644 virCommandAddEnvPass; virCommandAddEnvPassCommon; diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c -index 9fcea9d..085cabb 100644 +index 9fcea9d..11f7788 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c -@@ -5117,7 +5117,7 @@ qemuBuildChrChardevFileStr(virLogManagerPtr logManager, +@@ -5117,43 +5117,14 @@ qemuBuildChrChardevFileStr(virLogManagerPtr logManager, const char *filearg, const char *fileval, const char *appendarg, int appendval) { - if (logManager) { -+ if (false) { - g_autofree char *fdset = NULL; - g_autofree char *fdpath = NULL; - int flags = 0; -@@ -5368,7 +5368,7 @@ qemuBuildChrChardevStr(virLogManagerPtr logManager, +- g_autofree char *fdset = NULL; +- g_autofree char *fdpath = NULL; +- int flags = 0; +- int logfd; +- +- if (appendval == VIR_TRISTATE_SWITCH_ABSENT || +- appendval == VIR_TRISTATE_SWITCH_OFF) +- flags |= VIR_LOG_MANAGER_PROTOCOL_DOMAIN_OPEN_LOG_FILE_TRUNCATE; +- +- if ((logfd = virLogManagerDomainOpenLogFile(logManager, +- "qemu", +- def->uuid, +- def->name, +- fileval, +- flags, +- NULL, NULL)) < 0) +- return -1; +- +- virCommandPassFD(cmd, logfd, VIR_COMMAND_PASS_FD_CLOSE_PARENT); +- if (!(fdset = qemuVirCommandGetFDSet(cmd, logfd))) +- return -1; +- +- virCommandAddArg(cmd, "-add-fd"); +- virCommandAddArg(cmd, fdset); +- +- if (!(fdpath = qemuVirCommandGetDevSet(cmd, logfd))) +- return -1; +- +- virBufferAsprintf(buf, ",%s=%s,%s=on", filearg, fdpath, appendarg); +- } else { +- virBufferAsprintf(buf, ",%s=", filearg); +- virQEMUBuildBufferEscapeComma(buf, fileval); +- if (appendval != VIR_TRISTATE_SWITCH_ABSENT) { +- virBufferAsprintf(buf, ",%s=%s", appendarg, +- virTristateSwitchTypeToString(appendval)); +- } ++ (void)logManager; ++ (void)cmd; ++ (void)def; ++ virBufferAsprintf(buf, ",%s=", filearg); ++ virQEMUBuildBufferEscapeComma(buf, fileval); ++ if (appendval != VIR_TRISTATE_SWITCH_ABSENT) { ++ virBufferAsprintf(buf, ",%s=%s", appendarg, ++ virTristateSwitchTypeToString(appendval)); + } + + return 0; +@@ -5252,6 +5223,7 @@ qemuBuildChrChardevStr(virLogManagerPtr logManager, + g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; + bool telnet; + g_autofree char *charAlias = NULL; ++ (void)secManager; + + if (!(charAlias = qemuAliasChardevFromDevAlias(alias))) + return NULL; +@@ -5368,26 +5340,8 @@ qemuBuildChrChardevStr(virLogManagerPtr logManager, case VIR_DOMAIN_CHR_TYPE_UNIX: virBufferAsprintf(&buf, "socket,id=%s", charAlias); - if (dev->data.nix.listen && -+ if (0 && dev->data.nix.listen && - (flags & QEMU_BUILD_CHARDEV_UNIX_FD_PASS) && - virQEMUCapsGet(qemuCaps, QEMU_CAPS_CHARDEV_FD_PASS)) { - if (qemuSecuritySetSocketLabel(secManager, (virDomainDefPtr)def) < 0) -@@ -5414,7 +5414,7 @@ qemuBuildChrChardevStr(virLogManagerPtr logManager, +- (flags & QEMU_BUILD_CHARDEV_UNIX_FD_PASS) && +- virQEMUCapsGet(qemuCaps, QEMU_CAPS_CHARDEV_FD_PASS)) { +- if (qemuSecuritySetSocketLabel(secManager, (virDomainDefPtr)def) < 0) +- return NULL; +- int fd = qemuOpenChrChardevUNIXSocket(dev); +- if (qemuSecurityClearSocketLabel(secManager, (virDomainDefPtr)def) < 0) { +- VIR_FORCE_CLOSE(fd); +- return NULL; +- } +- if (fd < 0) +- return NULL; +- +- virBufferAsprintf(&buf, ",fd=%d", fd); +- +- virCommandPassFD(cmd, fd, VIR_COMMAND_PASS_FD_CLOSE_PARENT); +- } else { +- virBufferAddLit(&buf, ",path="); +- virQEMUBuildBufferEscapeComma(&buf, dev->data.nix.path); +- } ++ virBufferAddLit(&buf, ",path="); ++ virQEMUBuildBufferEscapeComma(&buf, dev->data.nix.path); + if (dev->data.nix.listen) { + virBufferAddLit(&buf, ",server"); + if (flags & QEMU_BUILD_CHARDEV_TCP_NOWAIT) +@@ -5414,13 +5368,6 @@ qemuBuildChrChardevStr(virLogManagerPtr logManager, return NULL; } - if (dev->logfile) { -+ if (0 && dev->logfile) { - if (qemuBuildChrChardevFileStr(logManager, cmd, def, &buf, - "logfile", dev->logfile, - "logappend", dev->logappend) < 0) +- if (qemuBuildChrChardevFileStr(logManager, cmd, def, &buf, +- "logfile", dev->logfile, +- "logappend", dev->logappend) < 0) +- return NULL; +- } +- + return virBufferContentAndReset(&buf); + } + diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 04b9333..92c1a5d 100644 --- a/src/qemu/qemu_migration.c @@ -133,7 +209,7 @@ index 3bdfdbe..5d037e2 100644 return monfd; diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c -index 013c9eb..569274d 100644 +index 013c9eb..711a3b4 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -94,6 +94,8 @@ @@ -190,21 +266,24 @@ index 013c9eb..569274d 100644 VIR_DEBUG("Waiting for handshake from child"); if (virCommandHandshakeWait(cmd) < 0) { /* Read errors from child that occurred between fork and exec. */ -@@ -7163,6 +7163,22 @@ qemuProcessLaunch(virConnectPtr conn, +@@ -7163,6 +7163,25 @@ qemuProcessLaunch(virConnectPtr conn, _("Process exited prior to exec")); goto cleanup; } + + VIR_DEBUG("read pid from rexec pidfile"); + rexecPidfile = g_strdup_printf("/var/run/rexec/pids/%lld", (long long)vm->pid); ++ if (virFileIsLink(rexecPidfile)) { ++ goto cleanup; ++ } + if ((rv = virPidFileReadPath(rexecPidfile, &vm->pid)) < 0) { + virReportSystemError(-rv, + _("Domain %s didn't show up"), + vm->def->name); + goto cleanup; + } -+ VIR_DEBUG("Got real qemu pid, QEMU vm=%p name=%s running with pid=%lld", -+ vm, vm->def->name, (long long)vm->pid); ++ VIR_DEBUG("Got real qemu pid, QEMU name=%s running with pid=%lld", ++ vm->def->name, (long long)vm->pid); + + VIR_DEBUG("Writing early domain status to disk"); + if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0) @@ -213,7 +292,7 @@ index 013c9eb..569274d 100644 VIR_DEBUG("Setting up domain cgroup (if required)"); if (qemuSetupCgroup(vm, nnicindexes, nicindexes) < 0) -@@ -7591,7 +7607,7 @@ qemuProcessKill(virDomainObjPtr vm, unsigned int flags) +@@ -7591,7 +7610,7 @@ qemuProcessKill(virDomainObjPtr vm, unsigned int flags) } if (flags & VIR_QEMU_PROCESS_KILL_NOWAIT) { @@ -222,7 +301,7 @@ index 013c9eb..569274d 100644 (flags & VIR_QEMU_PROCESS_KILL_FORCE) ? SIGKILL : SIGTERM); return 0; -@@ -8682,7 +8698,7 @@ qemuProcessQMPStop(qemuProcessQMPPtr proc) +@@ -8682,7 +8701,7 @@ qemuProcessQMPStop(qemuProcessQMPPtr proc) if (proc->pid != 0) { VIR_DEBUG("Killing QMP caps process %lld", (long long)proc->pid); @@ -421,7 +500,7 @@ index 0000000..46d910a +int remote_sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t *mask); +int remote_bind(int sockfd, struct sockaddr *addr, int addrlen); diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c -index b075dd1..93dc5a9 100644 +index b075dd1..5d2fca0 100644 --- a/src/util/vircgroup.c +++ b/src/util/vircgroup.c @@ -453,6 +453,7 @@ virCgroupSetValueRaw(const char *path, @@ -443,21 +522,37 @@ index b075dd1..93dc5a9 100644 virReportSystemError(errno, _("Unable to write to '%s'"), path); return -1; -@@ -1239,7 +1244,7 @@ virCgroupNewMachine(const char *name, - int rv; - - *group = NULL; +@@ -1236,27 +1241,14 @@ virCgroupNewMachine(const char *name, + unsigned int maxthreads, + virCgroupPtr *group) + { +- int rv; - -+#if 0 - if ((rv = virCgroupNewMachineSystemd(name, - drivername, - uuid, -@@ -1256,7 +1261,7 @@ virCgroupNewMachine(const char *name, ++ (void)uuid; ++ (void)rootdir; ++ (void)isContainer; ++ (void)nnicindexes; ++ (void)nicindexes; ++ (void)maxthreads; + *group = NULL; - if (rv == -1) - return -1; +- if ((rv = virCgroupNewMachineSystemd(name, +- drivername, +- uuid, +- rootdir, +- pidleader, +- isContainer, +- nnicindexes, +- nicindexes, +- partition, +- controllers, +- maxthreads, +- group)) == 0) +- return 0; +- +- if (rv == -1) +- return -1; - -+#endif return virCgroupNewMachineManual(name, drivername, pidleader, @@ -562,10 +657,25 @@ index 12cb592..40f2e60 100644 if (virCommandRun(cmd, NULL) < 0) diff --git a/src/util/virnuma.c b/src/util/virnuma.c -index d5cc262..acd2477 100644 +index d5cc262..63963b5 100644 --- a/src/util/virnuma.c +++ b/src/util/virnuma.c -@@ -162,7 +162,7 @@ virNumaSetupMemoryPolicy(virDomainNumatuneMemMode mode, +@@ -49,9 +49,13 @@ + #include "virutil.h" + + #define VIR_FROM_THIS VIR_FROM_NONE ++#define MAX_CPU_PER_NUM (512) // 最大CPU数 ++#define NUMA_MAX_NODE_NUM MAX_CPU_PER_NUM // 在最大CPU数条件下, 每个CPU占一个node的情况, 最多的node数 ++#define NUMA_MAX_NODE_LEN (MAX_CPU_PER_NUM / sizeof(unsigned long) / 8) // 在最大CPU数条件下, g_numaNodeMaskde的列数, 先计算g_numaNodeMask每个元素数据类型大小, 再由字节转为位 + + VIR_LOG_INIT("util.numa"); +- ++int g_numaMaxCpuNum = 0; ++unsigned long g_numaNodeMask[NUMA_MAX_NODE_NUM][NUMA_MAX_NODE_LEN] = {0}; + + #if HAVE_NUMAD + char * +@@ -162,7 +166,7 @@ virNumaSetupMemoryPolicy(virDomainNumatuneMemMode mode, bool virNumaIsAvailable(void) { @@ -574,15 +684,16 @@ index d5cc262..acd2477 100644 } -@@ -232,6 +232,69 @@ virNumaGetNodeMemory(int node, +@@ -232,6 +236,101 @@ virNumaGetNodeMemory(int node, return 0; } -+int virNumaNodeToCpus(int node, unsigned long* mask) ++static int virNumaNodeToCpus(int node, unsigned long* mask, int mask_n_bytes) +{ -+#define NODE_MAX_LEN (32) -+ -+ char nodeStr[NODE_MAX_LEN] = {0}; ++ virBuffer buf = VIR_BUFFER_INITIALIZER; ++ char *nodeStr = NULL; ++ int outIndex = 0; ++ int outValue = 0; + virCommandPtr cmd = NULL; + char *disp = NULL; + int ret = 0; @@ -592,7 +703,20 @@ index d5cc262..acd2477 100644 + int num = 0; + unsigned int *output = (unsigned int*)mask; + -+ sprintf(nodeStr, "node %d cpus:", node); ++ if (node > NUMA_MAX_NODE_NUM) { ++ VIR_ERROR("virNumaNodeToCpus failed, node %d is oversize", node); ++ return -1; ++ } ++ ++ for (int i = 0; i < NUMA_MAX_NODE_LEN; i++) { ++ if (g_numaNodeMask[node][i] != 0) { ++ memcpy(mask, &g_numaNodeMask[node][0], mask_n_bytes); ++ return 0; ++ } ++ } ++ ++ virBufferAsprintf(&buf, "node %d cpus:", node); ++ nodeStr = virBufferContentAndReset(&buf); + + cmd = virCommandNew("/usr/bin/rexec"); + virCommandAddArgList(cmd, "numactl", "-H", NULL); @@ -620,6 +744,12 @@ index d5cc262..acd2477 100644 + } + + pNext = start + strlen(nodeStr) + 1; ++ if (pNext >= (disp + disLen)) { ++ VIR_ERROR("pNext is NULL"); ++ ret = -1; ++ goto out; ++ } ++ + while ((*pNext == ' ') || ((*pNext >= '0') && (*pNext <= '9'))) { + signed char c = *pNext - '0'; + if ((c >= 0) && (c <= 9)) { @@ -627,13 +757,25 @@ index d5cc262..acd2477 100644 + } else { + int index = num / (sizeof(unsigned int) * 8); + int value = num % (sizeof(unsigned int) * 8); ++ if (index >= (mask_n_bytes / sizeof(unsigned int))) { ++ break; ++ } + output[index] += (1 << value); + num = 0; + } + pNext++; ++ if (pNext >= (disp + disLen)) { ++ break; ++ } + } + -+ output[(num / (sizeof(unsigned int) * 8))] += (1 << (num % (sizeof(unsigned int) * 8))); ++ outIndex = num / (sizeof(unsigned int) * 8); ++ outValue = num % (sizeof(unsigned int) * 8); ++ if (outIndex >= (mask_n_bytes / sizeof(unsigned int))) { ++ goto out; ++ } ++ output[outIndex] += (1 << outValue); ++ memcpy(&g_numaNodeMask[node][0], mask, mask_n_bytes); + +out: + VIR_FREE(disp); @@ -644,16 +786,16 @@ index d5cc262..acd2477 100644 /** * virNumaGetNodeCPUs: -@@ -268,7 +331,7 @@ virNumaGetNodeCPUs(int node, +@@ -268,7 +367,7 @@ virNumaGetNodeCPUs(int node, if (VIR_ALLOC_N(mask, mask_n_bytes / sizeof(*mask)) < 0) return -1; - if (numa_node_to_cpus(node, mask, mask_n_bytes) < 0) { -+ if (virNumaNodeToCpus(node, mask) < 0) { ++ if (virNumaNodeToCpus(node, mask, mask_n_bytes) < 0) { VIR_WARN("NUMA topology for cell %d is not available, ignoring", node); return -2; } -@@ -419,7 +482,47 @@ virNumaNodesetToCPUset(virBitmapPtr nodeset G_GNUC_UNUSED, +@@ -419,7 +518,60 @@ virNumaNodesetToCPUset(virBitmapPtr nodeset G_GNUC_UNUSED, unsigned int virNumaGetMaxCPUs(void) { @@ -665,6 +807,10 @@ index d5cc262..acd2477 100644 + int disLen = 0; + int sum = 0; + ++ if (g_numaMaxCpuNum != 0) { ++ return g_numaMaxCpuNum; ++ } ++ + cmd = virCommandNew("/usr/bin/rexec"); + virCommandAddArgList(cmd, "numactl", "--show", NULL); + virCommandSetOutputBuffer(cmd, &disp); @@ -688,12 +834,21 @@ index d5cc262..acd2477 100644 + } + + pNext = start + strlen("physcpubind:"); ++ if (pNext >= (disp + disLen)) { ++ VIR_ERROR("pNext is NULL"); ++ goto out; ++ } ++ + while ((*pNext == ' ') || ((*pNext >= '0') && (*pNext <= '9'))) { + if ((*pNext != ' ') && (*(pNext - 1) == ' ')) { + sum++; + } + pNext++; ++ if (pNext >= (disp + disLen)) { ++ break; ++ } + } ++ g_numaMaxCpuNum = sum; + +out: + VIR_FREE(disp); @@ -702,7 +857,7 @@ index d5cc262..acd2477 100644 } -@@ -437,7 +540,8 @@ virNumaGetMaxCPUs(void) +@@ -437,7 +589,8 @@ virNumaGetMaxCPUs(void) bool virNumaNodeIsAvailable(int node) { @@ -734,7 +889,7 @@ index 2d34b61..13661af 100644 *used = true; ret = 0; diff --git a/src/util/virprocess.c b/src/util/virprocess.c -index c3f88d2..4ec1dc9 100644 +index c3f88d2..93d4c6a 100644 --- a/src/util/virprocess.c +++ b/src/util/virprocess.c @@ -98,6 +98,8 @@ VIR_LOG_INIT("util.process"); @@ -782,23 +937,22 @@ index c3f88d2..4ec1dc9 100644 virReportSystemError(errno, _("cannot get CPU affinity of process %d"), pid); goto cleanup; -@@ -718,7 +722,96 @@ virProcessPrLimit(pid_t pid, +@@ -718,7 +722,97 @@ virProcessPrLimit(pid_t pid, const struct rlimit *new_limit, struct rlimit *old_limit) { - return prlimit(pid, resource, new_limit, old_limit); -+#define PID_MAX_LEN (16) -+#define RES_MAX_LEN (64) -+ -+ char pidStr[PID_MAX_LEN] = {0}; -+ char resValStr[RES_MAX_LEN] = {0}; ++ virBuffer pidBuf = VIR_BUFFER_INITIALIZER; ++ virBuffer resValBuf = VIR_BUFFER_INITIALIZER; ++ char *pidStr = NULL; ++ char *resValStr = NULL; + virCommandPtr cmd = NULL; + char *disp = NULL; + int ret = 0; + int disLen = 0; + const char *res = NULL; + bool isSetCmd = false; -+ unsigned long long num = 0; ++ rlim_t num = 0; + int spaceNum = 0; + + if (new_limit != NULL) { @@ -818,11 +972,13 @@ index c3f88d2..4ec1dc9 100644 + return -1; + } + -+ sprintf(pidStr, "%d", pid); ++ virBufferAsprintf(&pidBuf, "%d", pid); ++ pidStr = virBufferContentAndReset(&pidBuf); + + cmd = virCommandNew("/usr/bin/rexec"); + if (isSetCmd) { -+ sprintf(resValStr, "%s=%ld", res, new_limit->rlim_cur); ++ virBufferAsprintf(&resValBuf, "%s=%ld", res, new_limit->rlim_cur); ++ resValStr = virBufferContentAndReset(&resValBuf); + virCommandAddArgList(cmd, "prlimit", "--pid", pidStr, resValStr, NULL); + } else { + virCommandAddArgList(cmd, "prlimit", "--pid", pidStr, res, "--noheadings", "-o", "SOFT,HARD,UNITS", NULL); @@ -880,12 +1036,15 @@ index c3f88d2..4ec1dc9 100644 } #elif HAVE_SETRLIMIT static int -@@ -977,7 +1070,7 @@ int virProcessGetStartTime(pid_t pid, +@@ -977,7 +1071,10 @@ int virProcessGetStartTime(pid_t pid, g_autofree char *buf = NULL; VIR_AUTOSTRINGLIST tokens = NULL; - filename = g_strdup_printf("/proc/%llu/stat", (long long)pid); + filename = g_strdup_printf("/local/proc/%llu/stat", (long long)pid); ++ if (virFileIsLink(filename)) { ++ return -1; ++ } if ((len = virFileReadAll(filename, 1024, &buf)) < 0) return -1;