From 7575a30103abfc7138efaa2d7ac50527412c60fe Mon Sep 17 00:00:00 2001 From: Jie Yang Date: Fri, 20 Aug 2021 21:59:20 +0800 Subject: [PATCH] Update version to 2.0.0-3 virtio: fix dev_id initialization for virtio-pci and vfio device on aarch64 platform vfio: fix the problem of dma mapping failed syscall: add syscall "newfstatat" in x86_64-unknown-linux-gnu target kernel_config: update kernel config 5.10 on aarch64 platform machine/standard_vm: fix inappropriate file open permissions migration: fix an errors during the PL011 device state restore migration: fix an error during migration interface on aarch64 fix spelling errors in project (cherry picked from commit a12a13829fa5d788667e11b886c254760e6a4579) --- 0001-fix-spelling-errors-in-project.patch | 449 +++++++++++++ ...-error-during-migration-interface-on.patch | 104 +++ ...-errors-during-the-PL011-device-stat.patch | 29 + ..._vm-fix-inappropriate-file-open-perm.patch | 122 ++++ ...date-kernel-config-5.10-on-aarch64-p.patch | 55 ++ ...all-newfstatat-in-x86_64-unknown-lin.patch | 99 +++ ...ix-the-problem-of-dma-mapping-failed.patch | 228 +++++++ ...d-initialization-for-virtio-pci-and-.patch | 614 ++++++++++++++++++ stratovirt.spec | 21 +- 9 files changed, 1720 insertions(+), 1 deletion(-) create mode 100644 0001-fix-spelling-errors-in-project.patch create mode 100644 0002-migration-fix-an-error-during-migration-interface-on.patch create mode 100644 0003-migration-fix-an-errors-during-the-PL011-device-stat.patch create mode 100644 0004-machine-standard_vm-fix-inappropriate-file-open-perm.patch create mode 100644 0005-kernel_config-update-kernel-config-5.10-on-aarch64-p.patch create mode 100644 0006-syscall-add-syscall-newfstatat-in-x86_64-unknown-lin.patch create mode 100644 0007-vfio-fix-the-problem-of-dma-mapping-failed.patch create mode 100644 0008-virtio-fix-dev_id-initialization-for-virtio-pci-and-.patch diff --git a/0001-fix-spelling-errors-in-project.patch b/0001-fix-spelling-errors-in-project.patch new file mode 100644 index 0000000..ab3f64d --- /dev/null +++ b/0001-fix-spelling-errors-in-project.patch @@ -0,0 +1,449 @@ +From 515c87cad45e324286b36b6c2f3d7f8319092a1e Mon Sep 17 00:00:00 2001 +From: DBJ <974658390@qq.com> +Date: Fri, 13 Aug 2021 02:20:28 -0700 +Subject: [PATCH 1/8] fix spelling errors in project + +--- + address_space/src/address_space.rs | 2 +- + boot_loader/src/x86_64/standard_boot/elf.rs | 2 +- + cpu/src/lib.rs | 8 ++++---- + devices/src/interrupt_controller/aarch64/state.rs | 2 +- + devices/src/legacy/fwcfg.rs | 6 +++--- + devices/src/legacy/pl011.rs | 2 +- + devices/src/legacy/rtc.rs | 6 +++--- + docs/config_guidebook.md | 6 +++--- + docs/design.md | 2 +- + docs/interconnect_with_libvirt.md | 2 +- + hypervisor/src/kvm/interrupt.rs | 2 +- + machine/src/micro_vm/mod.rs | 4 ++-- + machine/src/standard_vm/aarch64/mod.rs | 2 +- + migration/src/manager.rs | 4 ++-- + migration/src/snapshot.rs | 4 ++-- + ozone/src/syscall.rs | 2 +- + tests/hydropper/monitor/monitor_thread.py | 2 +- + .../microvm/functional/test_microvm_concurrency.py | 4 ++-- + tests/hydropper/utils/session.py | 2 +- + tests/hydropper/utils/utils_qmp.py | 2 +- + vfio/src/vfio_dev.rs | 8 ++++---- + virtio/src/queue.rs | 2 +- + virtio/src/virtio_mmio.rs | 2 +- + 23 files changed, 39 insertions(+), 39 deletions(-) + +diff --git a/address_space/src/address_space.rs b/address_space/src/address_space.rs +index c182281..85bf71f 100644 +--- a/address_space/src/address_space.rs ++++ b/address_space/src/address_space.rs +@@ -307,7 +307,7 @@ impl AddressSpace { + }) + } + +- /// Return the end address fo memory according to all Ram regions in AddressSpace. ++ /// Return the end address of memory according to all Ram regions in AddressSpace. + pub fn memory_end_address(&self) -> GuestAddress { + self.flat_view + .load() +diff --git a/boot_loader/src/x86_64/standard_boot/elf.rs b/boot_loader/src/x86_64/standard_boot/elf.rs +index 481b876..a76e789 100644 +--- a/boot_loader/src/x86_64/standard_boot/elf.rs ++++ b/boot_loader/src/x86_64/standard_boot/elf.rs +@@ -79,7 +79,7 @@ impl Elf64Header { + bail!("Invalid magic in ELF header"); + } + if self.e_ident[EI_DATA] != ELFDATA2LSB { +- bail!("Big endian ELF file is not suppored"); ++ bail!("Big endian ELF file is not supported"); + } + if self.e_ident[EI_CLASS] != ELFCLASS64 { + bail!("Only 64-bit ELF image is supported"); +diff --git a/cpu/src/lib.rs b/cpu/src/lib.rs +index 41e143f..cb1a41b 100644 +--- a/cpu/src/lib.rs ++++ b/cpu/src/lib.rs +@@ -494,7 +494,7 @@ impl CPUInterface for CPU { + .chain_err(|| "Some error occurred in guest shutdown")?; + } else { + error!( +- "Vcpu{} recevied unexpected system event with type 0x{:x}, flags 0x{:x}", ++ "Vcpu{} received unexpected system event with type 0x{:x}, flags 0x{:x}", + self.id(), + event, + flags +@@ -569,11 +569,11 @@ impl CPUThreadWorker { + } + + register_signal_handler(VCPU_EXIT_SIGNAL, handle_signal) +- .chain_err(|| "Failed to registe VCPU_EXIT_SIGNAL signal.")?; ++ .chain_err(|| "Failed to register VCPU_EXIT_SIGNAL signal.")?; + register_signal_handler(VCPU_PAUSE_SIGNAL, handle_signal) +- .chain_err(|| "Failed to registe VCPU_PAUSE_SIGNAL signal.")?; ++ .chain_err(|| "Failed to register VCPU_PAUSE_SIGNAL signal.")?; + register_signal_handler(VCPU_TASK_SIGNAL, handle_signal) +- .chain_err(|| "Failed to registe VCPU_TASK_SIGNAL signal.")?; ++ .chain_err(|| "Failed to register VCPU_TASK_SIGNAL signal.")?; + + Ok(()) + } +diff --git a/devices/src/interrupt_controller/aarch64/state.rs b/devices/src/interrupt_controller/aarch64/state.rs +index e1b05b6..d4da00b 100644 +--- a/devices/src/interrupt_controller/aarch64/state.rs ++++ b/devices/src/interrupt_controller/aarch64/state.rs +@@ -75,7 +75,7 @@ const ICC_CTLR_EL1: u64 = 0xc664; + const ICC_SRE_EL1: u64 = 0xc665; + const ICC_IGRPEN0_EL1: u64 = 0xc666; + const ICC_IGRPEN1_EL1: u64 = 0xc667; +-/// GICv3 CPU interface control regiter pribits[8:10] ++/// GICv3 CPU interface control register pribits[8:10] + const ICC_CTLR_EL1_PRIBITS_MASK: u64 = 0x700; + const ICC_CTLR_EL1_PRIBITS_SHIFT: u64 = 0x8; + +diff --git a/devices/src/legacy/fwcfg.rs b/devices/src/legacy/fwcfg.rs +index ed41581..34c1375 100644 +--- a/devices/src/legacy/fwcfg.rs ++++ b/devices/src/legacy/fwcfg.rs +@@ -365,7 +365,7 @@ impl FwCfgCommon { + ); + }; + +- // unwrap is safe bacause the count of arch_entries and entries is initialized ++ // unwrap is safe because the count of arch_entries and entries is initialized + // as `FW_CFG_FILE_FIRST + FW_CFG_FILE_SLOTS_DFLT`, which is equal to the return + // value of `max_entry` function. + if self.is_arch_local() { +@@ -1402,7 +1402,7 @@ mod test { + fwcfg_dev.lock().unwrap().read(&mut read_data, base, offset); + assert_eq!(read_data, sig_entry_data); + +- // Failed to write beacuse of offset overflow. ++ // Failed to write because of offset overflow. + let write_data = vec![0x0_u8]; + let offset = 0x18; + let f_back = fwcfg_dev.lock().unwrap().write(&write_data, base, offset); +@@ -1442,7 +1442,7 @@ mod test { + fwcfg_dev.lock().unwrap().read(&mut read_data, base, offset); + assert_eq!(read_data, sig_entry_data); + +- // Failed to write beacuse of offset overflow. ++ // Failed to write because of offset overflow. + let write_data = vec![0x0_u8]; + let offset = 0x0c; + let f_back = fwcfg_dev.lock().unwrap().write(&write_data, base, offset); +diff --git a/devices/src/legacy/pl011.rs b/devices/src/legacy/pl011.rs +index fa063c3..133487e 100644 +--- a/devices/src/legacy/pl011.rs ++++ b/devices/src/legacy/pl011.rs +@@ -74,7 +74,7 @@ pub struct PL011State { + ibrd: u32, + /// Fractional Baud Rate Register. + fbrd: u32, +- /// Interrut FIFO Level Select Register. ++ /// Interrupt FIFO Level Select Register. + ifl: u32, + /// Identifier Register. Length is 8. + id: [u8; 8], +diff --git a/devices/src/legacy/rtc.rs b/devices/src/legacy/rtc.rs +index 0b5e4aa..6241e68 100644 +--- a/devices/src/legacy/rtc.rs ++++ b/devices/src/legacy/rtc.rs +@@ -36,7 +36,7 @@ const RTC_MINUTES: u8 = 0x02; + const RTC_MINUTES_ALARM: u8 = 0x03; + const RTC_HOURS: u8 = 0x04; + const RTC_HOURS_ARARM: u8 = 0x05; +-const RTC_DAY_OF_WEAK: u8 = 0x06; ++const RTC_DAY_OF_WEEK: u8 = 0x06; + const RTC_DAY_OF_MONTH: u8 = 0x07; + const RTC_MONTH: u8 = 0x08; + const RTC_YEAR: u8 = 0x09; +@@ -61,7 +61,7 @@ const CMOS_MEM_ABOVE_4GB: (u8, u8, u8) = (0x5B, 0x5C, 0x5D); + fn get_utc_time() -> libc::tm { + let time_val: libc::time_t = 0_i64; + +- // Safe bacause `libc::time` only get time. ++ // Safe because `libc::time` only get time. + unsafe { libc::time(time_val as *mut i64) }; + + let mut dest_tm = libc::tm { +@@ -177,7 +177,7 @@ impl RTC { + RTC_HOURS => { + data[0] = bin_to_bcd(tm.tm_hour as u8); + } +- RTC_DAY_OF_WEAK => { ++ RTC_DAY_OF_WEEK => { + data[0] = bin_to_bcd((tm.tm_wday + 1) as u8); + } + RTC_DAY_OF_MONTH => { +diff --git a/docs/config_guidebook.md b/docs/config_guidebook.md +index 6d118ec..6a5cda7 100644 +--- a/docs/config_guidebook.md ++++ b/docs/config_guidebook.md +@@ -124,7 +124,7 @@ If you want to use initrd as rootfs, `root=/dev/ram` and `rdinit=/bin/sh` must b + ## 2. Device Configuration + + For machine type "microvm", only virtio-mmio and legacy devices are supported. +-Maximum number of user createable devices is 11 on x86_64 and 160 on aarch64. ++Maximum number of user creatable devices is 11 on x86_64 and 160 on aarch64. + + For standard VM (machine type "q35" on x86_64, and "virt" on aarch64) , virtio-pci devices are supported instead of virtio-mmio + devices. As for now pci bridges are not implemented yet, there is currently only one +@@ -829,7 +829,7 @@ About the arguments: + * `netns` : path to a existed network namespace. + * `source` : path to the source file, such as `rootfs` and `vmlinux`. + * `clean-resource` : a flag to clean resource. +-* `--` : these two dashes are used to splite args, the args followed are used to launched StratoVirt. ++* `--` : these two dashes are used to split args, the args followed are used to launched StratoVirt. + + ### 5.2 Example + As ozone uses a directory to mount as a root directory, after ozone is launched, the directory "/srv/zozne/{exec_file}/{name}" will be created. (Where, `exec_file` is the executable binary file, usually it is `stratovirt`, while `name` is the name of ozone, it is given by users, but the length of it should be no more than 255 bytes.) In order to run ozone normally, please make sure that the directory "/srv/zozne/{exec_file}/{name}" does not exists before launching ozone. +@@ -874,7 +874,7 @@ $ ./ozone \ + ``` + + ## 6. Libvirt +-Libvirt launchs StratoVirt by creating cmdlines. But some of these commands ++Libvirt launches StratoVirt by creating cmdlines. But some of these commands + such as: cpu, overcommit, uuid, no-user-config, nodefaults, sandbox, msg, rtc, no-shutdown, + nographic, realtime, display, usb, mem-prealloc and boot, are not supported by StratoVirt. + To launch StratoVirt from libvirt successfully, StratoVirt needs to put these arguments into +diff --git a/docs/design.md b/docs/design.md +index 4ffdd16..537d256 100644 +--- a/docs/design.md ++++ b/docs/design.md +@@ -45,6 +45,6 @@ The following figure shows StratoVirt's core architecture which consist of three + + - Only the Linux operating system is supported; The recommended kernel version is 4.19; + - Only Linux is supported as the client operating system, and the recommended kernel version is 4.19; +-- StratoVirt is fullly tested on openEuler; ++- StratoVirt is fully tested on openEuler; + - Supports a maximum of 254 CPUs; + +diff --git a/docs/interconnect_with_libvirt.md b/docs/interconnect_with_libvirt.md +index a448bb5..d5205ed 100644 +--- a/docs/interconnect_with_libvirt.md ++++ b/docs/interconnect_with_libvirt.md +@@ -20,7 +20,7 @@ CPU topology is not supported, please configure the number of VCPUs only. + ``` + 4 + ``` +-- Archtecture: ++- Architecture: + + Optional value of `arch` are: `aarch64` and `x86_64`. On X86 platform, supported machine is `q35`; on aarch 64 platform, supported machine is `virt`. + ``` +diff --git a/hypervisor/src/kvm/interrupt.rs b/hypervisor/src/kvm/interrupt.rs +index 45799f9..675d25c 100644 +--- a/hypervisor/src/kvm/interrupt.rs ++++ b/hypervisor/src/kvm/interrupt.rs +@@ -103,7 +103,7 @@ impl IrqRouteTable { + #[cfg(target_arch = "x86_64")] + pub fn init_irq_route_table(&mut self) { + // On x86, use `kvm_create_irqchip` to create an interrupt +- // controller module in the kernel. It creates a virual PIC, a virtual ioapic, ++ // controller module in the kernel. It creates a virtual PIC, a virtual ioapic, + // and sets up future vcpus to have a local APIC. IRQ routing for GSIs 0-15 is set + // to both PIC and IOAPIC. GSI 16-23 only go to the IOAPIC. + for i in 0..IOAPIC_NUM_PINS { +diff --git a/machine/src/micro_vm/mod.rs b/machine/src/micro_vm/mod.rs +index ea92164..f48a6dc 100644 +--- a/machine/src/micro_vm/mod.rs ++++ b/machine/src/micro_vm/mod.rs +@@ -659,7 +659,7 @@ impl MachineOps for LightMachine { + let index = MMIO_REPLACEABLE_BLK_NR + self.replaceable_info.net_count; + if index >= MMIO_REPLACEABLE_BLK_NR + MMIO_REPLACEABLE_NET_NR { + bail!( +- "A maximum of {} net replaceble devices are supported.", ++ "A maximum of {} net replaceable devices are supported.", + MMIO_REPLACEABLE_NET_NR + ); + } +@@ -677,7 +677,7 @@ impl MachineOps for LightMachine { + let device_cfg = parse_blk(vm_config, cfg_args)?; + if self.replaceable_info.block_count >= MMIO_REPLACEABLE_BLK_NR { + bail!( +- "A maximum of {} block replaceble devices are supported.", ++ "A maximum of {} block replaceable devices are supported.", + MMIO_REPLACEABLE_BLK_NR + ); + } +diff --git a/machine/src/standard_vm/aarch64/mod.rs b/machine/src/standard_vm/aarch64/mod.rs +index ee66bc9..54c103c 100644 +--- a/machine/src/standard_vm/aarch64/mod.rs ++++ b/machine/src/standard_vm/aarch64/mod.rs +@@ -179,7 +179,7 @@ impl StdMachineOps for StdMachine { + let pcihost_root = PciHostRoot::new(root_bus); + pcihost_root + .realize() +- .chain_err(|| "Faile to realize pcihost root device.")?; ++ .chain_err(|| "Failed to realize pcihost root device.")?; + + Ok(()) + } +diff --git a/migration/src/manager.rs b/migration/src/manager.rs +index 35ed975..480b59b 100644 +--- a/migration/src/manager.rs ++++ b/migration/src/manager.rs +@@ -21,7 +21,7 @@ use super::status::MigrationStatus; + use util::byte_code::ByteCode; + + lazy_static! { +- /// Glocal MigrationMananger to manage all migration combined interface. ++ /// Glocal MigrationManager to manage all migration combined interface. + pub(crate) static ref MIGRATION_MANAGER: Arc = Arc::new(MigrationManager { + entry: Arc::new(RwLock::new(BTreeMap::::new())), + desc_db: Arc::new(RwLock::new(HashMap::::new())), +@@ -135,7 +135,7 @@ pub enum MigrationEntry { + /// This structure is to manage all resource during migration. + /// It is also the only way to call on `MIGRATION_MANAGER`. + pub struct MigrationManager { +- /// The map offers the deivce_id and combined migratable device entry. ++ /// The map offers the device_id and combined migratable device entry. + pub(crate) entry: Arc>>, + /// The map offers the device type and its device state describe structure. + pub(crate) desc_db: Arc>>, +diff --git a/migration/src/snapshot.rs b/migration/src/snapshot.rs +index 470ea63..edea8ce 100644 +--- a/migration/src/snapshot.rs ++++ b/migration/src/snapshot.rs +@@ -94,7 +94,7 @@ impl MigrationManager { + /// # Notes + /// + /// Offers a interface for restore snapshot functions. This function will make VM +- /// back to the state restored in snapshot file incluing both device and memory. ++ /// back to the state restored in snapshot file including both device and memory. + /// + /// # Argument + /// +@@ -126,7 +126,7 @@ impl MigrationManager { + bail!("Invalid device state snapshot file"); + } + +- Self::load_memory(&mut memory_file).chain_err(|| "Failed to load snpashot memory")?; ++ Self::load_memory(&mut memory_file).chain_err(|| "Failed to load snapshot memory")?; + let snapshot_desc_db = + Self::load_descriptor_db(&mut device_state_file, device_state_header.desc_len) + .chain_err(|| "Failed to load device descriptor db")?; +diff --git a/ozone/src/syscall.rs b/ozone/src/syscall.rs +index 2627642..936661f 100644 +--- a/ozone/src/syscall.rs ++++ b/ozone/src/syscall.rs +@@ -140,7 +140,7 @@ pub fn set_host_name(host_name: &str) -> Result<()> { + /// + /// # Arguments + /// +-/// * `fd` - File descriptor refering to one of magic links in a /proc/[pid]/ns/ directory. ++/// * `fd` - File descriptor referring to one of magic links in a /proc/[pid]/ns/ directory. + /// * `nstype` - Namespace type. + pub fn setns(fd: i32, nstype: i32) -> Result<()> { + SyscallResult { +diff --git a/tests/hydropper/monitor/monitor_thread.py b/tests/hydropper/monitor/monitor_thread.py +index daad97d..5921ecf 100644 +--- a/tests/hydropper/monitor/monitor_thread.py ++++ b/tests/hydropper/monitor/monitor_thread.py +@@ -25,7 +25,7 @@ class MonitorThread(threading.Thread): + items = dict() + + def __init__(self): +- """Construtor""" ++ """Constructor""" + super(MonitorThread, self).__init__() + self.state_lock = threading.Lock() + self._state = 'init' +diff --git a/tests/hydropper/testcases/microvm/functional/test_microvm_concurrency.py b/tests/hydropper/testcases/microvm/functional/test_microvm_concurrency.py +index ded87ad..73de004 100644 +--- a/tests/hydropper/testcases/microvm/functional/test_microvm_concurrency.py ++++ b/tests/hydropper/testcases/microvm/functional/test_microvm_concurrency.py +@@ -20,10 +20,10 @@ def test_microvm_concurrency(microvms): + """ + Test multi microvms start: + +- 1) Set each VM vcpu_count = 4, then launch it and comfirm vcpu count is 4. ++ 1) Set each VM vcpu_count = 4, then launch it and confirm vcpu count is 4. + And increase _succ_sum. + 2) Execute step 1 concurrency by threads. +- 3) Comfirm each VM is execute successffully by _succ_sum ++ 3) Confirm each VM is execute successffully by _succ_sum + + Note: You can modify CONCURRENT_QUANTITY tag in config/config.ini to set vm quantity. + """ +diff --git a/tests/hydropper/utils/session.py b/tests/hydropper/utils/session.py +index 1eb3bad..bb9036f 100644 +--- a/tests/hydropper/utils/session.py ++++ b/tests/hydropper/utils/session.py +@@ -152,7 +152,7 @@ class ConsoleManager(): + LOG.debug("Got 'Warning added RSA to known host list") + continue + except aexpect.ExpectTimeoutError as err: +- # send a empty line to avoid unexpect login timeout ++ # send a empty line to avoid unexpected login timeout + # because some message from linux kernel maybe impact match + if not last_chance: + time.sleep(0.5) +diff --git a/tests/hydropper/utils/utils_qmp.py b/tests/hydropper/utils/utils_qmp.py +index 8acefed..7d0527d 100644 +--- a/tests/hydropper/utils/utils_qmp.py ++++ b/tests/hydropper/utils/utils_qmp.py +@@ -9,7 +9,7 @@ + # KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + # NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + # See the Mulan PSL v2 for more details. +-"""Some qmp fuctions""" ++"""Some qmp functions""" + + import re + from utils.exception import QMPError +diff --git a/vfio/src/vfio_dev.rs b/vfio/src/vfio_dev.rs +index 0603d49..30f7d5a 100644 +--- a/vfio/src/vfio_dev.rs ++++ b/vfio/src/vfio_dev.rs +@@ -464,13 +464,13 @@ impl VfioDevice { + .iter() + .collect::() + .read_link() +- .chain_err(|| "Invaild iommu group path")?; ++ .chain_err(|| "Invalid iommu group path")?; + let group_name = iommu_group + .file_name() +- .chain_err(|| "Invaild iommu group name")?; ++ .chain_err(|| "Invalid iommu group name")?; + let mut group_id = 0; + if let Some(n) = group_name.to_str() { +- group_id = n.parse::().chain_err(|| "Invaild iommu group id")?; ++ group_id = n.parse::().chain_err(|| "Invalid iommu group id")?; + } + + let mut groups = container.groups.lock().unwrap(); +@@ -489,7 +489,7 @@ impl VfioDevice { + fn vfio_get_device(group: &VfioGroup, name: &Path) -> Result { + let mut dev_name: &str = ""; + if let Some(n) = name.file_name() { +- dev_name = n.to_str().chain_err(|| "Invaild device path")?; ++ dev_name = n.to_str().chain_err(|| "Invalid device path")?; + } + let path: CString = CString::new(dev_name.as_bytes()) + .chain_err(|| "Failed to convert device name to CString type of data")?; +diff --git a/virtio/src/queue.rs b/virtio/src/queue.rs +index 9979b5c..6ac137c 100644 +--- a/virtio/src/queue.rs ++++ b/virtio/src/queue.rs +@@ -68,7 +68,7 @@ pub struct QueueConfig { + pub ready: bool, + /// Interrupt vector index of the queue for msix + pub vector: u16, +- /// The next index which can be poped in the available vring. ++ /// The next index which can be popped in the available vring. + next_avail: u16, + /// The next index which can be pushed in the used vring. + next_used: u16, +diff --git a/virtio/src/virtio_mmio.rs b/virtio/src/virtio_mmio.rs +index 099015f..ad9b32a 100644 +--- a/virtio/src/virtio_mmio.rs ++++ b/virtio/src/virtio_mmio.rs +@@ -80,7 +80,7 @@ const VENDOR_ID: u32 = 0; + const MMIO_MAGIC_VALUE: u32 = 0x7472_6976; + const MMIO_VERSION: u32 = 2; + +-/// The maximun of virtio queue within a virtio device. ++/// The maximum of virtio queue within a virtio device. + const MAXIMUM_NR_QUEUES: usize = 8; + + /// HostNotifyInfo includes the info needed for notifying backend from guest. +-- +2.25.1 + diff --git a/0002-migration-fix-an-error-during-migration-interface-on.patch b/0002-migration-fix-an-error-during-migration-interface-on.patch new file mode 100644 index 0000000..6274b1e --- /dev/null +++ b/0002-migration-fix-an-error-during-migration-interface-on.patch @@ -0,0 +1,104 @@ +From 7c9db6a2e3082b41af3e62b241d49553f5bb8492 Mon Sep 17 00:00:00 2001 +From: Wei Gao +Date: Wed, 18 Aug 2021 14:30:08 +0800 +Subject: [PATCH 2/8] migration: fix an error during migration interface on + aarch64. + +Change the order of GICv3 device register to the end. Because it rely on +vcpu_init if boot with multi vcpu. + +Signed-off-by: Wei Gao +--- + .../src/interrupt_controller/aarch64/gicv3.rs | 3 ++- + hypervisor/src/kvm/mod.rs | 1 + + machine/src/lib.rs | 2 +- + migration/src/manager.rs | 16 ++++++++++++---- + 4 files changed, 16 insertions(+), 6 deletions(-) + +diff --git a/devices/src/interrupt_controller/aarch64/gicv3.rs b/devices/src/interrupt_controller/aarch64/gicv3.rs +index 3303b32..6559e73 100644 +--- a/devices/src/interrupt_controller/aarch64/gicv3.rs ++++ b/devices/src/interrupt_controller/aarch64/gicv3.rs +@@ -389,13 +389,14 @@ impl GICDevice for GICv3 { + gic_conf: &GICConfig, + ) -> Result> { + let gicv3 = Arc::new(GICv3::new(gic_conf)?); +- MigrationManager::register_device_instance(GICv3State::descriptor(), gicv3.clone()); + if gicv3.its_dev.is_some() { + MigrationManager::register_device_instance( + GICv3ItsState::descriptor(), + gicv3.its_dev.as_ref().unwrap().clone(), ++ true, + ); + } ++ MigrationManager::register_device_instance(GICv3State::descriptor(), gicv3.clone(), true); + + Ok(gicv3) + } +diff --git a/hypervisor/src/kvm/mod.rs b/hypervisor/src/kvm/mod.rs +index 5b5be94..19193db 100644 +--- a/hypervisor/src/kvm/mod.rs ++++ b/hypervisor/src/kvm/mod.rs +@@ -60,6 +60,7 @@ impl KVMFds { + migration::MigrationManager::register_device_instance( + state::KvmDeviceState::descriptor(), + Arc::new(state::KvmDevice {}), ++ false, + ); + + kvm_fds +diff --git a/machine/src/lib.rs b/machine/src/lib.rs +index 8a095b3..9eb3039 100644 +--- a/machine/src/lib.rs ++++ b/machine/src/lib.rs +@@ -233,7 +233,7 @@ pub trait MachineOps { + )); + cpus.push(cpu.clone()); + +- MigrationManager::register_device_instance(cpu::ArchCPU::descriptor(), cpu); ++ MigrationManager::register_device_instance(cpu::ArchCPU::descriptor(), cpu, false); + } + + if let Some(boot_config) = boot_cfg { +diff --git a/migration/src/manager.rs b/migration/src/manager.rs +index 480b59b..7ec4767 100644 +--- a/migration/src/manager.rs ++++ b/migration/src/manager.rs +@@ -162,14 +162,22 @@ impl MigrationManager { + /// + /// * `device_desc` - The `DeviceStateDesc` of device instance. + /// * `entry` - Device instance with migratable interface. +- pub fn register_device_instance(device_desc: DeviceStateDesc, device_entry: Arc) +- where ++ /// * `reverse` - Register device in order or in the reverse order. ++ pub fn register_device_instance( ++ device_desc: DeviceStateDesc, ++ device_entry: Arc, ++ reverse: bool, ++ ) where + T: MigrationHook + Sync + Send + 'static, + { + Self::register_device_desc(device_desc); + + let entry = MigrationEntry::Safe(device_entry); +- let nr_entry = Self::entry_db_len(); ++ let nr_entry = if reverse { ++ !0 - Self::entry_db_len() ++ } else { ++ Self::entry_db_len() ++ }; + + MIGRATION_MANAGER + .entry +@@ -333,7 +341,7 @@ mod tests { + let device_v2 = Arc::new(DeviceV2::default()); + let device_v2_mutex = Arc::new(Mutex::new(DeviceV2::default())); + +- MigrationManager::register_device_instance(DeviceV1State::descriptor(), device_v1); ++ MigrationManager::register_device_instance(DeviceV1State::descriptor(), device_v1, false); + MigrationManager::register_memory_instance(device_v2); + MigrationManager::register_device_instance_mutex( + DeviceV2State::descriptor(), +-- +2.25.1 + diff --git a/0003-migration-fix-an-errors-during-the-PL011-device-stat.patch b/0003-migration-fix-an-errors-during-the-PL011-device-stat.patch new file mode 100644 index 0000000..982fa71 --- /dev/null +++ b/0003-migration-fix-an-errors-during-the-PL011-device-stat.patch @@ -0,0 +1,29 @@ +From 79a5a58e74e28c175ee836bda60090cf511f893d Mon Sep 17 00:00:00 2001 +From: Wei Gao +Date: Wed, 18 Aug 2021 15:30:33 +0800 +Subject: [PATCH 3/8] migration: fix an errors during the PL011 device state + restore. + +This bug is leaded by commit d24f510cf271f144084266ff292db7e94c004b53. +Add the function to register device again. + +Signed-off-by: Wei Gao +--- + devices/src/legacy/pl011.rs | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/devices/src/legacy/pl011.rs b/devices/src/legacy/pl011.rs +index 133487e..65bf64b 100644 +--- a/devices/src/legacy/pl011.rs ++++ b/devices/src/legacy/pl011.rs +@@ -172,6 +172,7 @@ impl PL011 { + param_type: "earlycon".to_string(), + value: format!("pl011,mmio,0x{:08x}", region_base), + }); ++ MigrationManager::register_device_instance_mutex(PL011State::descriptor(), dev.clone()); + let locked_dev = dev.lock().unwrap(); + locked_dev.chardev.lock().unwrap().set_input_callback(&dev); + EventLoop::update_event( +-- +2.25.1 + diff --git a/0004-machine-standard_vm-fix-inappropriate-file-open-perm.patch b/0004-machine-standard_vm-fix-inappropriate-file-open-perm.patch new file mode 100644 index 0000000..3357d6b --- /dev/null +++ b/0004-machine-standard_vm-fix-inappropriate-file-open-perm.patch @@ -0,0 +1,122 @@ +From 22362ed8a2d865e0b84bb20615eb1415086eb713 Mon Sep 17 00:00:00 2001 +From: Jiajie Li +Date: Wed, 18 Aug 2021 11:18:45 +0800 +Subject: [PATCH 4/8] machine/standard_vm: fix inappropriate file open + permissions + +For PFlash device, code file is read-only and vars file is +readable and writable. So using right permissions to open +these two files. + +Signed-off-by: Jiajie Li +--- + machine/src/standard_vm/aarch64/mod.rs | 9 ++++----- + machine/src/standard_vm/mod.rs | 17 ++++++++++++++++- + machine/src/standard_vm/x86_64/mod.rs | 8 +++----- + 3 files changed, 23 insertions(+), 11 deletions(-) + +diff --git a/machine/src/standard_vm/aarch64/mod.rs b/machine/src/standard_vm/aarch64/mod.rs +index 54c103c..d3c8670 100644 +--- a/machine/src/standard_vm/aarch64/mod.rs ++++ b/machine/src/standard_vm/aarch64/mod.rs +@@ -44,9 +44,9 @@ use virtio::{qmp_balloon, qmp_query_balloon}; + use vmm_sys_util::eventfd::EventFd; + + use super::{AcpiBuilder, StdMachineOps}; +-use crate::errors::Result as MachineResult; + use crate::errors::{ErrorKind, Result}; + use crate::MachineOps; ++use crate::{errors::Result as MachineResult, standard_vm::open_pflash_file}; + use pci_host_root::PciHostRoot; + use syscall::syscall_whitelist; + +@@ -397,6 +397,7 @@ impl MachineOps for StdMachine { + + /// Add pflash device. + fn add_pflash_device(&mut self, configs: &[PFlashConfig]) -> Result<()> { ++ use super::errors::ErrorKind as StdErrorKind; + use crate::errors::ResultExt; + + let mut configs_vec = configs.to_vec(); +@@ -407,10 +408,8 @@ impl MachineOps for StdMachine { + for i in 0..=1 { + let (fd, read_only) = if i < configs_vec.len() { + let config = &configs_vec[i]; +- let fd = std::fs::OpenOptions::new() +- .read(true) +- .write(true) +- .open(config.path_on_host.clone())?; ++ let fd = open_pflash_file(&config.path_on_host, config.unit) ++ .chain_err(|| StdErrorKind::OpenFileErr(config.path_on_host.clone()))?; + (Some(fd), config.read_only) + } else { + (None, false) +diff --git a/machine/src/standard_vm/mod.rs b/machine/src/standard_vm/mod.rs +index 5ba7929..ff469c3 100644 +--- a/machine/src/standard_vm/mod.rs ++++ b/machine/src/standard_vm/mod.rs +@@ -39,12 +39,15 @@ pub mod errors { + InitPCIeHostErr { + display("Failed to init PCIe host.") + } ++ OpenFileErr(path: String) { ++ display("Failed to open file: {}.", path) ++ } + } + } + } + +-use std::mem::size_of; + use std::sync::{Arc, Mutex}; ++use std::{fs::File, mem::size_of}; + + #[cfg(target_arch = "x86_64")] + use acpi::AcpiGenericAddress; +@@ -61,6 +64,18 @@ use aarch64::{LayoutEntryType, MEM_LAYOUT}; + #[cfg(target_arch = "x86_64")] + use x86_64::{LayoutEntryType, MEM_LAYOUT}; + ++fn open_pflash_file(file_name: &str, unit: usize) -> Result { ++ let fd = if unit == 0 { ++ std::fs::OpenOptions::new().read(true).open(file_name)? ++ } else { ++ std::fs::OpenOptions::new() ++ .read(true) ++ .write(true) ++ .open(file_name)? ++ }; ++ Ok(fd) ++} ++ + trait StdMachineOps: AcpiBuilder { + fn init_pci_host(&self) -> Result<()>; + +diff --git a/machine/src/standard_vm/x86_64/mod.rs b/machine/src/standard_vm/x86_64/mod.rs +index 0cb2b63..1c29703 100644 +--- a/machine/src/standard_vm/x86_64/mod.rs ++++ b/machine/src/standard_vm/x86_64/mod.rs +@@ -49,7 +49,7 @@ use vmm_sys_util::eventfd::EventFd; + use super::errors::{ErrorKind, Result}; + use super::{AcpiBuilder, StdMachineOps}; + use crate::errors::{ErrorKind as MachineErrorKind, Result as MachineResult}; +-use crate::MachineOps; ++use crate::{standard_vm::open_pflash_file, MachineOps}; + use mch::Mch; + use syscall::syscall_whitelist; + use util::byte_code::ByteCode; +@@ -398,10 +398,8 @@ impl MachineOps for StdMachine { + // of current PFlash device. + let mut flash_end: u64 = MEM_LAYOUT[LayoutEntryType::MemAbove4g as usize].0; + for config in configs_vec { +- let mut fd = std::fs::OpenOptions::new() +- .read(true) +- .write(true) +- .open(config.path_on_host.clone())?; ++ let mut fd = open_pflash_file(&config.path_on_host, config.unit) ++ .chain_err(|| ErrorKind::OpenFileErr(config.path_on_host.clone()))?; + let pfl_size = fd.metadata().unwrap().len(); + + if config.unit == 0 { +-- +2.25.1 + diff --git a/0005-kernel_config-update-kernel-config-5.10-on-aarch64-p.patch b/0005-kernel_config-update-kernel-config-5.10-on-aarch64-p.patch new file mode 100644 index 0000000..30211d3 --- /dev/null +++ b/0005-kernel_config-update-kernel-config-5.10-on-aarch64-p.patch @@ -0,0 +1,55 @@ +From cde02263b76a88407b23d0ca2e06f244c45dec42 Mon Sep 17 00:00:00 2001 +From: Wei Gao +Date: Thu, 19 Aug 2021 20:13:42 +0800 +Subject: [PATCH 5/8] kernel_config: update kernel config 5.10 on aarch64 + platform. + +1. Turn on memory hotplug feature. + +Signed-off-by: Wei Gao +--- + docs/kernel_config/micro_vm/kernel_config_5.10_aarch64 | 9 +++++++-- + 1 file changed, 7 insertions(+), 2 deletions(-) + +diff --git a/docs/kernel_config/micro_vm/kernel_config_5.10_aarch64 b/docs/kernel_config/micro_vm/kernel_config_5.10_aarch64 +index d9deb7a..bbd016f 100644 +--- a/docs/kernel_config/micro_vm/kernel_config_5.10_aarch64 ++++ b/docs/kernel_config/micro_vm/kernel_config_5.10_aarch64 +@@ -698,7 +698,9 @@ CONFIG_SPARSEMEM_VMEMMAP_ENABLE=y + CONFIG_SPARSEMEM_VMEMMAP=y + CONFIG_HAVE_FAST_GUP=y + CONFIG_ARCH_KEEP_MEMBLOCK=y +-# CONFIG_MEMORY_HOTPLUG is not set ++CONFIG_MEMORY_ISOLATION=y ++CONFIG_MEMORY_HOTPLUG=y ++CONFIG_MEMORY_HOTPLUG_SPARSE=y + CONFIG_SPLIT_PTLOCK_CPUS=4 + CONFIG_MEMORY_BALLOON=y + # CONFIG_COMPACTION is not set +@@ -1767,6 +1769,7 @@ CONFIG_COMMON_CLK=y + # CONFIG_CLK_QORIQ is not set + # CONFIG_COMMON_CLK_XGENE is not set + # CONFIG_COMMON_CLK_FIXED_MMIO is not set ++# CONFIG_MCHP_CLK_PFSOC is not set + # CONFIG_HWSPINLOCK is not set + + # +@@ -2072,6 +2075,8 @@ CONFIG_INIT_STACK_NONE=y + # CONFIG_INIT_ON_FREE_DEFAULT_ON is not set + # end of Memory initialization + # end of Kernel hardening options ++ ++# CONFIG_SECURITY_BOOT_INIT is not set + # end of Security options + + CONFIG_CRYPTO=y +@@ -2431,4 +2436,4 @@ CONFIG_CC_HAS_SANCOV_TRACE_PC=y + # CONFIG_RUNTIME_TESTING_MENU is not set + # CONFIG_MEMTEST is not set + # end of Kernel Testing and Coverage +-# end of Kernel hacking +\ No newline at end of file ++# end of Kernel hacking +-- +2.25.1 + diff --git a/0006-syscall-add-syscall-newfstatat-in-x86_64-unknown-lin.patch b/0006-syscall-add-syscall-newfstatat-in-x86_64-unknown-lin.patch new file mode 100644 index 0000000..497f2f0 --- /dev/null +++ b/0006-syscall-add-syscall-newfstatat-in-x86_64-unknown-lin.patch @@ -0,0 +1,99 @@ +From 2a70e217561e64f460e95d4d89d145fc615f12ec Mon Sep 17 00:00:00 2001 +From: Qi Xi +Date: Fri, 20 Aug 2021 15:43:00 +0800 +Subject: [PATCH 6/8] syscall: add syscall "newfstatat" in + x86_64-unknown-linux-gnu target + +When we run stratovirt, built with GNU toolchain on x86 platform, and +use serial port for IO, it will be blocked. The bug is caused by when +it is the first time for rtc device to call "libc::gmtime_r" after +seccomp taking effect, it will use the syscall "newfstatat" which is +not in the syscall whitelist. + +The bug is fixed by adding "newfstatat" to syscall whitelist with +x86_64-unknown-linux-gnu target. + +Signed-off-by: Qi Xi +--- + docs/config_guidebook.md | 18 +++++++++++++++--- + machine/src/micro_vm/syscall.rs | 4 +++- + machine/src/standard_vm/x86_64/syscall.rs | 4 +++- + 3 files changed, 21 insertions(+), 5 deletions(-) + +diff --git a/docs/config_guidebook.md b/docs/config_guidebook.md +index 6a5cda7..72b4c7e 100644 +--- a/docs/config_guidebook.md ++++ b/docs/config_guidebook.md +@@ -679,10 +679,22 @@ And you can also restore StratoVirt's **pid number** to a file by: + ### 4.2 Seccomp + + StratoVirt use [seccomp(2)](https://man7.org/linux/man-pages/man2/seccomp.2.html) to limit the syscalls +-in StratoVirt process by default. StratoVirt use only 40 syscalls in x86_64 (39 syscalls in aarch64) after running. +-It will make a slight influence on performance to StratoVirt. If you want to disable seccomp, you can +-run StratoVirt with `-disable-seccomp`. ++in StratoVirt process by default. It will make a slight influence on performance to StratoVirt. ++* X86_64 + ++| Number of Syscalls | GNU Toolchain | MUSL Toolchain | ++| :----------------: | :-----------: | :------------: | ++| Micro_vm | 41 | 41 | ++| Standard_vm | 46 | 43 | ++ ++* AArch64 ++ ++| Number of Syscalls | GNU Toolchain | MUSL Toolchain | ++| :----------------: | :-----------: | :------------: | ++| Micro_vm | 39 | 40 | ++| Standard_vm | 43 | 42 | ++ ++If you want to disable seccomp, you can run StratoVirt with `-disable-seccomp`. + ```shell + # cmdline + -disable-seccomp +diff --git a/machine/src/micro_vm/syscall.rs b/machine/src/micro_vm/syscall.rs +index eb52df6..ddc9023 100644 +--- a/machine/src/micro_vm/syscall.rs ++++ b/machine/src/micro_vm/syscall.rs +@@ -92,7 +92,7 @@ ioctl_iowr_nr!(KVM_GET_REG_LIST, KVMIO, 0xb0, kvm_reg_list); + /// + /// # Notes + /// This allowlist limit syscall with: +-/// * x86_64-unknown-gnu: 40 syscalls ++/// * x86_64-unknown-gnu: 41 syscalls + /// * x86_64-unknown-musl: 41 syscalls + /// * aarch64-unknown-gnu: 39 syscalls + /// * aarch64-unknown-musl: 40 syscalls +@@ -153,6 +153,8 @@ pub fn syscall_whitelist() -> Vec { + BpfRule::new(libc::SYS_statx), + #[cfg(all(target_env = "musl", target_arch = "x86_64"))] + BpfRule::new(libc::SYS_stat), ++ #[cfg(all(target_env = "gnu", target_arch = "x86_64"))] ++ BpfRule::new(libc::SYS_newfstatat), + #[cfg(all(target_env = "musl", target_arch = "aarch64"))] + BpfRule::new(libc::SYS_newfstatat), + #[cfg(target_arch = "x86_64")] +diff --git a/machine/src/standard_vm/x86_64/syscall.rs b/machine/src/standard_vm/x86_64/syscall.rs +index d794c78..dddeb8a 100644 +--- a/machine/src/standard_vm/x86_64/syscall.rs ++++ b/machine/src/standard_vm/x86_64/syscall.rs +@@ -78,7 +78,7 @@ ioctl_iowr_nr!(KVM_GET_MSRS, KVMIO, 0x88, kvm_msrs); + /// + /// # Notes + /// This allowlist limit syscall with: +-/// * x86_64-unknown-gnu: 45 syscalls ++/// * x86_64-unknown-gnu: 46 syscalls + /// * x86_64-unknown-musl: 43 syscalls + /// To reduce performance losses, the syscall rules is ordered by frequency. + pub fn syscall_whitelist() -> Vec { +@@ -132,6 +132,8 @@ pub fn syscall_whitelist() -> Vec { + BpfRule::new(libc::SYS_gettid), + BpfRule::new(libc::SYS_getpid), + BpfRule::new(libc::SYS_fstat), ++ #[cfg(all(target_env = "gnu"))] ++ BpfRule::new(libc::SYS_newfstatat), + BpfRule::new(libc::SYS_pread64), + BpfRule::new(libc::SYS_pwrite64), + BpfRule::new(libc::SYS_statx), +-- +2.25.1 + diff --git a/0007-vfio-fix-the-problem-of-dma-mapping-failed.patch b/0007-vfio-fix-the-problem-of-dma-mapping-failed.patch new file mode 100644 index 0000000..39b58b7 --- /dev/null +++ b/0007-vfio-fix-the-problem-of-dma-mapping-failed.patch @@ -0,0 +1,228 @@ +From 023dde42b55a58c7a41293566e95a0fc94efa2c6 Mon Sep 17 00:00:00 2001 +From: "Xinle.Guo" +Date: Thu, 19 Aug 2021 20:48:25 +0800 +Subject: [PATCH 7/8] vfio: fix the problem of dma mapping failed + +Failed to use libvirt to manager more than two vfio devices. +The reason is that creating a container for every vfio device, +the process stratovirt uses more than the memory locked limit. + +Signed-off-by: Xinle.Guo +--- + machine/src/lib.rs | 24 +++++++++++------------- + vfio/src/vfio_dev.rs | 25 +++++++++++++++---------- + vfio/src/vfio_pci.rs | 31 ++++++++++++++++++------------- + 3 files changed, 44 insertions(+), 36 deletions(-) + +diff --git a/machine/src/lib.rs b/machine/src/lib.rs +index 9eb3039..8ba0ba3 100644 +--- a/machine/src/lib.rs ++++ b/machine/src/lib.rs +@@ -121,7 +121,7 @@ use devices::legacy::FwCfgOps; + #[cfg(target_arch = "aarch64")] + use devices::InterruptController; + use hypervisor::KVM_FDS; +-use kvm_ioctls::{DeviceFd, VcpuFd}; ++use kvm_ioctls::VcpuFd; + use machine_manager::config::{ + get_pci_bdf, parse_balloon, parse_blk, parse_net, parse_rng_dev, parse_root_port, parse_vfio, + parse_virtconsole, parse_virtio_serial, parse_vsock, MachineMemConfig, PFlashConfig, PciBdf, +@@ -132,7 +132,7 @@ use machine_manager::machine::{KvmVmState, MachineInterface}; + use migration::MigrationManager; + use util::loop_context::{EventNotifier, NotifierCallback, NotifierOperation}; + use util::seccomp::{BpfRule, SeccompOpt, SyscallFilter}; +-use vfio::vfio_pci::create_vfio_device; ++use vfio::vfio_pci::create_vfio_container; + use vfio::{VfioContainer, VfioPciDevice}; + use virtio::{balloon_allow_list, Balloon, Block, Console, Rng, VirtioMmioDevice, VirtioPciDevice}; + use vmm_sys_util::epoll::EventSet; +@@ -478,13 +478,8 @@ pub trait MachineOps { + &mut self, + vm_config: &VmConfig, + cfg_args: &str, +- dev_fd: Arc, ++ container: Arc, + ) -> Result<()> { +- let sys_mem = self.get_sys_mem().clone(); +- let container = Arc::new( +- VfioContainer::new(dev_fd, &sys_mem).chain_err(|| "Failed to create vfio container")?, +- ); +- + let device_cfg: VfioConfig = parse_vfio(vm_config, cfg_args)?; + let path = "/sys/bus/pci/devices/".to_string() + &device_cfg.host; + let name = device_cfg.id; +@@ -543,10 +538,7 @@ pub trait MachineOps { + .chain_err(|| ErrorKind::AddDevErr("pflash".to_string()))?; + } + +- // Create an emulated kvm device that is used for VFIO. It should be created only once. +- // See the kernel docs for `KVM_CREATE_DEVICE` to get more info. +- let vfio_dev = create_vfio_device().chain_err(|| "Failed to create kvm device for VFIO")?; +- ++ let mut container: Option> = None; + for dev in &cloned_vm_config.devices { + let cfg_args = dev.1.as_str(); + match dev.0.as_str() { +@@ -581,7 +573,13 @@ pub trait MachineOps { + self.add_virtio_rng(vm_config, cfg_args)?; + } + "vfio-pci" => { +- self.add_vfio_device(&vm_config, cfg_args, vfio_dev.clone())?; ++ if container.is_none() { ++ container = Some( ++ create_vfio_container(self.get_sys_mem().clone()) ++ .chain_err(|| "Failed to create vfio container")?, ++ ); ++ } ++ self.add_vfio_device(&vm_config, cfg_args, container.clone().unwrap())?; + } + _ => { + bail!("Unsupported device: {:?}", dev.0.as_str()); +diff --git a/vfio/src/vfio_dev.rs b/vfio/src/vfio_dev.rs +index 30f7d5a..93922ba 100644 +--- a/vfio/src/vfio_dev.rs ++++ b/vfio/src/vfio_dev.rs +@@ -96,8 +96,8 @@ pub struct VfioMemoryRegion { + pub memory_size: u64, + // Host virtual address. + pub userspace_addr: u64, +- // No flags specified for now. +- flags_padding: u64, ++ // IOMMU mapped flag. ++ pub iommu_mapped: bool, + } + + /// `VfioMemInfo` structure contains pinning pages information. If any pages need to be zapped from +@@ -131,7 +131,7 @@ impl VfioMemInfo { + guest_phys_addr, + memory_size, + userspace_addr, +- flags_padding: 0_u64, ++ iommu_mapped: false, + }); + + Ok(()) +@@ -150,7 +150,7 @@ impl VfioMemInfo { + guest_phys_addr: fr.addr_range.base.raw_value(), + memory_size: fr.addr_range.size, + userspace_addr: hva + fr.offset_in_region, +- flags_padding: 0_u64, ++ iommu_mapped: false, + }; + let mut mem_regions = self.regions.lock().unwrap(); + for (index, mr) in mem_regions.iter().enumerate() { +@@ -378,9 +378,11 @@ impl VfioGroup { + return Err(ErrorKind::VfioIoctl("VFIO_GROUP_SET_CONTAINER".to_string(), ret).into()); + } + +- if let Err(e) = container.set_iommu(vfio::VFIO_TYPE1v2_IOMMU) { +- unsafe { ioctl_with_ref(&self.group, VFIO_GROUP_UNSET_CONTAINER(), &raw_fd) }; +- return Err(e).chain_err(|| "Failed to set IOMMU"); ++ if container.groups.lock().unwrap().is_empty() { ++ if let Err(e) = container.set_iommu(vfio::VFIO_TYPE1v2_IOMMU) { ++ unsafe { ioctl_with_ref(&self.group, VFIO_GROUP_UNSET_CONTAINER(), &raw_fd) }; ++ return Err(e).chain_err(|| "Failed to set IOMMU"); ++ } + } + + if let Err(e) = container.kvm_device_add_group(&self.group.as_raw_fd()) { +@@ -473,15 +475,18 @@ impl VfioDevice { + group_id = n.parse::().chain_err(|| "Invalid iommu group id")?; + } + +- let mut groups = container.groups.lock().unwrap(); +- if let Some(g) = groups.get(&group_id) { ++ if let Some(g) = container.groups.lock().unwrap().get(&group_id) { + return Ok(g.clone()); + } + let group = Arc::new(VfioGroup::new(group_id)?); + group + .connect_container(&container) + .chain_err(|| "Fail to connect container")?; +- groups.insert(group_id, group.clone()); ++ container ++ .groups ++ .lock() ++ .unwrap() ++ .insert(group_id, group.clone()); + + Ok(group) + } +diff --git a/vfio/src/vfio_pci.rs b/vfio/src/vfio_pci.rs +index 5cc674a..7bec445 100644 +--- a/vfio/src/vfio_pci.rs ++++ b/vfio/src/vfio_pci.rs +@@ -19,13 +19,12 @@ use std::sync::{Arc, Mutex, Weak}; + use byteorder::{ByteOrder, LittleEndian}; + use error_chain::ChainedError; + use kvm_bindings::{kvm_create_device, kvm_device_type_KVM_DEV_TYPE_VFIO}; +-use kvm_ioctls::DeviceFd; + use vfio_bindings::bindings::vfio; + use vmm_sys_util::eventfd::EventFd; + use vmm_sys_util::ioctl::ioctl_with_mut_ref; + + use super::errors::{ErrorKind, Result, ResultExt}; +-use address_space::{FileBackend, GuestAddress, HostMemMapping, Region, RegionOps}; ++use address_space::{AddressSpace, FileBackend, GuestAddress, HostMemMapping, Region, RegionOps}; + use hypervisor::{MsiVector, KVM_FDS}; + #[cfg(target_arch = "aarch64")] + use pci::config::SECONDARY_BUS_NUM; +@@ -380,7 +379,7 @@ impl VfioPciDevice { + .register_bar(i as usize, bar_region, vfio_bar.region_type, false, size); + } + +- self.map_guest_memory()?; ++ self.do_dma_map()?; + + Ok(()) + } +@@ -569,14 +568,17 @@ impl VfioPciDevice { + } + + /// Add all guest memory regions into IOMMU table. +- fn map_guest_memory(&mut self) -> Result<()> { ++ fn do_dma_map(&mut self) -> Result<()> { + let container = &self.vfio_device.container; +- let regions = container.vfio_mem_info.regions.lock().unwrap(); +- +- for r in regions.iter() { +- container +- .vfio_dma_map(r.guest_phys_addr, r.memory_size, r.userspace_addr) +- .chain_err(|| "Failed to add guest memory region map into IOMMU table")?; ++ let mut regions = container.vfio_mem_info.regions.lock().unwrap(); ++ ++ for r in regions.iter_mut() { ++ if !r.iommu_mapped { ++ container ++ .vfio_dma_map(r.guest_phys_addr, r.memory_size, r.userspace_addr) ++ .chain_err(|| "Failed to add guest memory region map into IOMMU table")?; ++ r.iommu_mapped = true; ++ } + } + Ok(()) + } +@@ -872,7 +874,7 @@ fn get_irq_rawfds(gsi_msi_routes: &[GsiMsiRoute]) -> Vec { + rawfds + } + +-pub fn create_vfio_device() -> Result> { ++pub fn create_vfio_container(sys_mem: Arc) -> Result> { + let mut vfio_device = kvm_create_device { + type_: kvm_device_type_KVM_DEV_TYPE_VFIO, + fd: 0, +@@ -884,7 +886,10 @@ pub fn create_vfio_device() -> Result> { + .as_ref() + .unwrap() + .create_device(&mut vfio_device) +- .chain_err(|| "Failed to create VFIO type KVM device")?; ++ .chain_err(|| "Failed to create kvm device for VFIO")?; + +- Ok(Arc::new(dev_fd)) ++ Ok(Arc::new( ++ VfioContainer::new(Arc::new(dev_fd), &sys_mem) ++ .chain_err(|| "Failed to create vfio container")?, ++ )) + } +-- +2.25.1 + diff --git a/0008-virtio-fix-dev_id-initialization-for-virtio-pci-and-.patch b/0008-virtio-fix-dev_id-initialization-for-virtio-pci-and-.patch new file mode 100644 index 0000000..7a28031 --- /dev/null +++ b/0008-virtio-fix-dev_id-initialization-for-virtio-pci-and-.patch @@ -0,0 +1,614 @@ +From 4ba032f8967c660b8eafb3adaf9b44b95474c7c3 Mon Sep 17 00:00:00 2001 +From: Ming Yang +Date: Wed, 11 Aug 2021 14:52:26 +0800 +Subject: [PATCH 8/8] virtio: fix dev_id initialization for virtio-pci and vfio + device on aarch64 platform + +On aarch64 platform, dev_id information is necessary for virtio-pci device +to send msix. + +This commit fixes the dev_id initialization in virtio_pci transport module, +and set dev_id information according to the number of bus where the device +is attached to. + +Signed-off-by: Ming Yang +--- + pci/src/config.rs | 2 + + pci/src/msix.rs | 63 ++++++++++++++------ + pci/src/root_port.rs | 10 +--- + vfio/src/vfio_pci.rs | 34 +++++++---- + virtio/src/virtio_pci.rs | 123 ++++++++++++++++++++++++--------------- + 5 files changed, 150 insertions(+), 82 deletions(-) + +diff --git a/pci/src/config.rs b/pci/src/config.rs +index d7bd348..873fe7a 100644 +--- a/pci/src/config.rs ++++ b/pci/src/config.rs +@@ -221,6 +221,7 @@ pub enum RegionType { + } + + /// Registered bar. ++#[derive(Clone)] + pub struct Bar { + region_type: RegionType, + address: u64, +@@ -266,6 +267,7 @@ pub enum PcieDevType { + } + + /// Configuration space of PCI/PCIe device. ++#[derive(Clone)] + pub struct PciConfig { + /// Configuration space data. + pub config: Vec, +diff --git a/pci/src/msix.rs b/pci/src/msix.rs +index 2831af4..5c04575 100644 +--- a/pci/src/msix.rs ++++ b/pci/src/msix.rs +@@ -10,16 +10,19 @@ + // NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + // See the Mulan PSL v2 for more details. + +-use std::sync::{Arc, Mutex}; ++use std::sync::atomic::{AtomicU16, Ordering}; ++use std::sync::{Arc, Mutex, Weak}; + + use address_space::{GuestAddress, Region, RegionOps}; + use hypervisor::{MsiVector, KVM_FDS}; + use migration::{DeviceStateDesc, FieldDesc, MigrationHook, MigrationManager, StateTransfer}; + use util::{byte_code::ByteCode, num_ops::round_up}; + +-use crate::config::{CapId, PciConfig, RegionType}; ++use crate::config::{CapId, PciConfig, RegionType, SECONDARY_BUS_NUM}; + use crate::errors::{Result, ResultExt}; +-use crate::{le_read_u16, le_read_u32, le_read_u64, le_write_u16, le_write_u32, le_write_u64}; ++use crate::{ ++ le_read_u16, le_read_u32, le_read_u64, le_write_u16, le_write_u32, le_write_u64, PciBus, ++}; + + pub const MSIX_TABLE_ENTRY_SIZE: u16 = 16; + pub const MSIX_TABLE_SIZE_MAX: u16 = 0x7ff; +@@ -71,7 +74,7 @@ pub struct Msix { + pub func_masked: bool, + pub enabled: bool, + pub msix_cap_offset: u16, +- pub dev_id: u16, ++ pub dev_id: Arc, + } + + impl Msix { +@@ -90,7 +93,7 @@ impl Msix { + func_masked: true, + enabled: true, + msix_cap_offset, +- dev_id, ++ dev_id: Arc::new(AtomicU16::new(dev_id)), + }; + msix.mask_all_vectors(); + msix +@@ -148,7 +151,11 @@ impl Msix { + le_write_u64(&mut self.pba, offset, old_val & pending_bit).unwrap(); + } + +- fn register_memory_region(msix: Arc>, region: &Region, dev_id: u16) -> Result<()> { ++ fn register_memory_region( ++ msix: Arc>, ++ region: &Region, ++ dev_id: Arc, ++ ) -> Result<()> { + let locked_msix = msix.lock().unwrap(); + let table_size = locked_msix.table.len() as u64; + let pba_size = locked_msix.pba.len() as u64; +@@ -170,7 +177,7 @@ impl Msix { + let is_masked: bool = locked_msix.is_vector_masked(vector); + if was_masked && !is_masked { + locked_msix.clear_pending_vector(vector); +- locked_msix.notify(vector, dev_id); ++ locked_msix.notify(vector, dev_id.load(Ordering::Acquire)); + } + + true +@@ -264,7 +271,7 @@ impl StateTransfer for Msix { + state.func_masked = self.func_masked; + state.enabled = self.enabled; + state.msix_cap_offset = self.msix_cap_offset; +- state.dev_id = self.dev_id; ++ state.dev_id = self.dev_id.load(Ordering::Acquire); + + Ok(state.as_bytes().to_vec()) + } +@@ -280,7 +287,7 @@ impl StateTransfer for Msix { + self.func_masked = msix_state.func_masked; + self.enabled = msix_state.enabled; + self.msix_cap_offset = msix_state.msix_cap_offset; +- self.dev_id = msix_state.dev_id; ++ self.dev_id = Arc::new(AtomicU16::new(msix_state.dev_id)); + + Ok(()) + } +@@ -318,7 +325,7 @@ impl MigrationHook for Msix { + msg_data: msg.data, + masked: false, + #[cfg(target_arch = "aarch64")] +- dev_id: self.dev_id as u32, ++ dev_id: self.dev_id.load(Ordering::Acquire) as u32, + }; + if let Err(e) = locked_irq_table.add_msi_route(allocated_gsi, msi_vector) { + bail!("Failed to add msi route to global irq routing table: {}", e); +@@ -330,7 +337,7 @@ impl MigrationHook for Msix { + + if self.is_vector_pending(vector) { + self.clear_pending_vector(vector); +- send_msix(msg, self.dev_id); ++ send_msix(msg, self.dev_id.load(Ordering::Acquire)); + } + } + } +@@ -377,7 +384,12 @@ fn send_msix(msg: Message, dev_id: u16) { + } + + /// MSI-X initialization. +-pub fn init_msix(bar_id: usize, vector_nr: u32, config: &mut PciConfig, dev_id: u16) -> Result<()> { ++pub fn init_msix( ++ bar_id: usize, ++ vector_nr: u32, ++ config: &mut PciConfig, ++ dev_id: Arc, ++) -> Result<()> { + if vector_nr > MSIX_TABLE_SIZE_MAX as u32 + 1 { + bail!("Too many msix vectors."); + } +@@ -401,7 +413,7 @@ pub fn init_msix(bar_id: usize, vector_nr: u32, config: &mut PciConfig, dev_id: + table_size, + pba_size, + msix_cap_offset as u16, +- dev_id, ++ dev_id.load(Ordering::Acquire), + ))); + let bar_size = ((table_size + pba_size) as u64).next_power_of_two(); + let region = Region::init_container_region(bar_size); +@@ -415,6 +427,17 @@ pub fn init_msix(bar_id: usize, vector_nr: u32, config: &mut PciConfig, dev_id: + Ok(()) + } + ++pub fn update_dev_id(parent_bus: &Weak>, devfn: u8, dev_id: &Arc) { ++ let bus_num = parent_bus ++ .upgrade() ++ .unwrap() ++ .lock() ++ .unwrap() ++ .number(SECONDARY_BUS_NUM as usize); ++ let device_id = ((bus_num as u16) << 8) | (devfn as u16); ++ dev_id.store(device_id, Ordering::Release); ++} ++ + #[cfg(test)] + mod tests { + use super::*; +@@ -425,9 +448,15 @@ mod tests { + let mut pci_config = PciConfig::new(PCI_CONFIG_SPACE_SIZE, 2); + + // Too many vectors. +- assert!(init_msix(0, MSIX_TABLE_SIZE_MAX as u32 + 2, &mut pci_config, 0).is_err()); +- +- init_msix(1, 2, &mut pci_config, 0).unwrap(); ++ assert!(init_msix( ++ 0, ++ MSIX_TABLE_SIZE_MAX as u32 + 2, ++ &mut pci_config, ++ Arc::new(AtomicU16::new(0)) ++ ) ++ .is_err()); ++ ++ init_msix(1, 2, &mut pci_config, Arc::new(AtomicU16::new(0))).unwrap(); + let msix_cap_start = 64_u8; + assert_eq!(pci_config.last_cap_end, 64 + MSIX_CAP_SIZE as u16); + // Capabilities pointer +@@ -492,7 +521,7 @@ mod tests { + #[test] + fn test_write_config() { + let mut pci_config = PciConfig::new(PCI_CONFIG_SPACE_SIZE, 2); +- init_msix(0, 2, &mut pci_config, 0).unwrap(); ++ init_msix(0, 2, &mut pci_config, Arc::new(AtomicU16::new(0))).unwrap(); + let msix = pci_config.msix.as_ref().unwrap(); + let mut locked_msix = msix.lock().unwrap(); + locked_msix.enabled = false; +diff --git a/pci/src/root_port.rs b/pci/src/root_port.rs +index ba1b5f7..948e31d 100644 +--- a/pci/src/root_port.rs ++++ b/pci/src/root_port.rs +@@ -10,6 +10,7 @@ + // NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + // See the Mulan PSL v2 for more details. + ++use std::sync::atomic::AtomicU16; + use std::sync::{Arc, Mutex, Weak}; + + use address_space::Region; +@@ -117,13 +118,8 @@ impl PciDevOps for RootPort { + config_space[PREF_MEMORY_LIMIT as usize] = PREF_MEM_RANGE_64BIT; + self.config + .add_pcie_cap(self.devfn, self.port_num, PcieDevType::RootPort as u8)?; +- #[cfg(target_arch = "aarch64")] +- { +- self.dev_id = self.set_dev_id(0, self.devfn); +- init_msix(0, 1, &mut self.config, self.dev_id)?; +- } +- #[cfg(target_arch = "x86_64")] +- init_msix(0, 1, &mut self.config, 0)?; ++ ++ init_msix(0, 1, &mut self.config, Arc::new(AtomicU16::new(0)))?; + + let parent_bus = self.parent_bus.upgrade().unwrap(); + let mut locked_parent_bus = parent_bus.lock().unwrap(); +diff --git a/vfio/src/vfio_pci.rs b/vfio/src/vfio_pci.rs +index 5cc674a..640ff20 100644 +--- a/vfio/src/vfio_pci.rs ++++ b/vfio/src/vfio_pci.rs +@@ -14,6 +14,7 @@ use std::collections::HashMap; + use std::mem::size_of; + use std::os::unix::io::{AsRawFd, RawFd}; + use std::path::Path; ++use std::sync::atomic::{AtomicU16, Ordering}; + use std::sync::{Arc, Mutex, Weak}; + + use byteorder::{ByteOrder, LittleEndian}; +@@ -36,9 +37,9 @@ use pci::config::{ + }; + use pci::errors::Result as PciResult; + use pci::msix::{ +- is_msix_enabled, Msix, MSIX_CAP_CONTROL, MSIX_CAP_ENABLE, MSIX_CAP_FUNC_MASK, MSIX_CAP_ID, +- MSIX_CAP_SIZE, MSIX_CAP_TABLE, MSIX_TABLE_BIR, MSIX_TABLE_ENTRY_SIZE, MSIX_TABLE_OFFSET, +- MSIX_TABLE_SIZE_MAX, ++ is_msix_enabled, update_dev_id, Msix, MSIX_CAP_CONTROL, MSIX_CAP_ENABLE, MSIX_CAP_FUNC_MASK, ++ MSIX_CAP_ID, MSIX_CAP_SIZE, MSIX_CAP_TABLE, MSIX_TABLE_BIR, MSIX_TABLE_ENTRY_SIZE, ++ MSIX_TABLE_OFFSET, MSIX_TABLE_SIZE_MAX, + }; + use pci::{ + le_read_u16, le_read_u32, le_write_u16, le_write_u32, ranges_overlap, PciBus, PciDevOps, +@@ -93,7 +94,7 @@ pub struct VfioPciDevice { + // Maintains a list of GSI with irqfds that are registered to kvm. + gsi_msi_routes: Arc>>, + devfn: u8, +- dev_id: u16, ++ dev_id: Arc, + name: String, + parent_bus: Weak>, + } +@@ -119,7 +120,7 @@ impl VfioPciDevice { + vfio_bars: Arc::new(Mutex::new(Vec::with_capacity(PCI_NUM_BARS as usize))), + gsi_msi_routes: Arc::new(Mutex::new(Vec::new())), + devfn, +- dev_id: 0, ++ dev_id: Arc::new(AtomicU16::new(0)), + name, + parent_bus, + }) +@@ -404,7 +405,7 @@ impl VfioPciDevice { + table_size, + table_size / 128, + cap_offset as u16, +- self.dev_id, ++ self.dev_id.load(Ordering::Acquire), + ))); + self.pci_config.msix = Some(msix.clone()); + +@@ -418,6 +419,9 @@ impl VfioPciDevice { + + let cloned_dev = self.vfio_device.clone(); + let cloned_gsi_routes = self.gsi_msi_routes.clone(); ++ let parent_bus = self.parent_bus.clone(); ++ let dev_id = self.dev_id.clone(); ++ let devfn = self.devfn; + let write = move |data: &[u8], _: GuestAddress, offset: u64| -> bool { + let mut locked_msix = msix.lock().unwrap(); + locked_msix.table[offset as usize..(offset as usize + data.len())] +@@ -429,13 +433,15 @@ impl VfioPciDevice { + } + + let entry = locked_msix.get_message(vector as u16); ++ ++ update_dev_id(&parent_bus, devfn, &dev_id); + let msix_vector = MsiVector { + msg_addr_lo: entry.address_lo, + msg_addr_hi: entry.address_hi, + msg_data: entry.data, + masked: false, + #[cfg(target_arch = "aarch64")] +- dev_id: locked_msix.dev_id as u32, ++ dev_id: dev_id.load(Ordering::Acquire) as u32, + }; + + let mut locked_gsi_routes = cloned_gsi_routes.lock().unwrap(); +@@ -707,7 +713,7 @@ impl PciDevOps for VfioPciDevice { + .lock() + .unwrap() + .number(SECONDARY_BUS_NUM as usize); +- self.dev_id = self.set_dev_id(bus_num, self.devfn); ++ self.dev_id = Arc::new(AtomicU16::new(self.set_dev_id(bus_num, self.devfn))); + } + + self.msix_info = Some(PciResultExt::chain_err(self.get_msix_info(), || { +@@ -799,7 +805,8 @@ impl PciDevOps for VfioPciDevice { + } + + if ranges_overlap(offset, end, COMMAND as usize, COMMAND as usize + 4) { +- self.pci_config.write(offset, data, self.dev_id); ++ self.pci_config ++ .write(offset, data, self.dev_id.load(Ordering::Acquire)); + + if le_read_u32(&self.pci_config.config, offset).unwrap() & COMMAND_MEMORY_SPACE as u32 + != 0 +@@ -822,7 +829,8 @@ impl PciDevOps for VfioPciDevice { + } + } + } else if ranges_overlap(offset, end, BAR_0 as usize, (BAR_5 as usize) + REG_SIZE) { +- self.pci_config.write(offset, data, self.dev_id); ++ self.pci_config ++ .write(offset, data, self.dev_id.load(Ordering::Acquire)); + + if size == 4 && LittleEndian::read_u32(data) != 0xffff_ffff { + let parent_bus = self.parent_bus.upgrade().unwrap(); +@@ -838,7 +846,8 @@ impl PciDevOps for VfioPciDevice { + } + } else if ranges_overlap(offset, end, cap_offset, cap_offset + MSIX_CAP_SIZE as usize) { + let was_enable = is_msix_enabled(cap_offset, &self.pci_config.config); +- self.pci_config.write(offset, data, self.dev_id); ++ self.pci_config ++ .write(offset, data, self.dev_id.load(Ordering::Acquire)); + let is_enable = is_msix_enabled(cap_offset, &self.pci_config.config); + + if !was_enable && is_enable { +@@ -853,7 +862,8 @@ impl PciDevOps for VfioPciDevice { + } + } + } else { +- self.pci_config.write(offset, data, self.dev_id); ++ self.pci_config ++ .write(offset, data, self.dev_id.load(Ordering::Acquire)); + } + } + +diff --git a/virtio/src/virtio_pci.rs b/virtio/src/virtio_pci.rs +index 03645dc..aaf009e 100644 +--- a/virtio/src/virtio_pci.rs ++++ b/virtio/src/virtio_pci.rs +@@ -24,6 +24,7 @@ use pci::config::{ + VENDOR_ID, + }; + use pci::errors::{ErrorKind, Result as PciResult, ResultExt}; ++use pci::msix::update_dev_id; + use pci::{config::PciConfig, init_msix, le_write_u16, ranges_overlap, PciBus, PciDevOps}; + use util::byte_code::ByteCode; + use vmm_sys_util::eventfd::EventFd; +@@ -459,13 +460,14 @@ pub struct VirtioPciState { + } + + /// Virtio-PCI device structure ++#[derive(Clone)] + pub struct VirtioPciDevice { + /// Name of this device + name: String, + /// The entity of virtio device + device: Arc>, + /// Device id +- dev_id: u16, ++ dev_id: Arc, + /// Devfn + devfn: u8, + /// If this device is activated or not. +@@ -500,7 +502,7 @@ impl VirtioPciDevice { + VirtioPciDevice { + name, + device, +- dev_id: 0_u16, ++ dev_id: Arc::new(AtomicU16::new(0)), + devfn, + device_activated: Arc::new(AtomicBool::new(false)), + sys_mem, +@@ -518,7 +520,7 @@ impl VirtioPciDevice { + fn assign_interrupt_cb(&mut self) { + let cloned_common_cfg = self.common_config.clone(); + let cloned_msix = self.config.msix.clone(); +- let dev_id = self.dev_id; ++ let dev_id = self.dev_id.clone(); + let cb = Arc::new(Box::new( + move |int_type: &VirtioInterruptType, queue: Option<&Queue>| { + let vector = match int_type { +@@ -533,7 +535,9 @@ impl VirtioPciDevice { + }; + + if let Some(msix) = &cloned_msix { +- msix.lock().unwrap().notify(vector, dev_id); ++ msix.lock() ++ .unwrap() ++ .notify(vector, dev_id.load(Ordering::Acquire)); + } else { + bail!("Failed to send interrupt, msix does not exist"); + } +@@ -618,14 +622,7 @@ impl VirtioPciDevice { + true + }; + +- let cloned_virtio_dev = self.device.clone(); +- let cloned_common_cfg = self.common_config.clone(); +- let cloned_virtio_queue = self.queues.clone(); +- let cloned_activated_flag = self.device_activated.clone(); +- let cloned_notify_evts = self.notify_eventfds.clone(); +- let cloned_sys_mem = self.sys_mem.clone(); +- let cloned_int_cb = self.interrupt_cb.clone(); +- let cloned_msix = self.config.msix.as_ref().unwrap().clone(); ++ let cloned_pci_device = self.clone(); + let common_write = move |data: &[u8], _addr: GuestAddress, offset: u64| -> bool { + let value = match data.len() { + 1 => data[0] as u32, +@@ -639,13 +636,18 @@ impl VirtioPciDevice { + return false; + } + }; +- let old_dev_status = cloned_common_cfg.lock().unwrap().device_status; ++ let old_dev_status = cloned_pci_device ++ .common_config ++ .lock() ++ .unwrap() ++ .device_status; + +- if let Err(e) = cloned_common_cfg.lock().unwrap().write_common_config( +- &cloned_virtio_dev, +- offset, +- value, +- ) { ++ if let Err(e) = cloned_pci_device ++ .common_config ++ .lock() ++ .unwrap() ++ .write_common_config(&cloned_pci_device.device.clone(), offset, value) ++ { + error!( + "Failed to read common config of virtio-pci device, error is {}", + e.display_chain(), +@@ -653,21 +655,29 @@ impl VirtioPciDevice { + return false; + } + +- if !cloned_activated_flag.load(Ordering::Acquire) +- && cloned_common_cfg.lock().unwrap().check_device_status( +- CONFIG_STATUS_ACKNOWLEDGE +- | CONFIG_STATUS_DRIVER +- | CONFIG_STATUS_DRIVER_OK +- | CONFIG_STATUS_FEATURES_OK, +- CONFIG_STATUS_FAILED, +- ) ++ if !cloned_pci_device.device_activated.load(Ordering::Acquire) ++ && cloned_pci_device ++ .common_config ++ .lock() ++ .unwrap() ++ .check_device_status( ++ CONFIG_STATUS_ACKNOWLEDGE ++ | CONFIG_STATUS_DRIVER ++ | CONFIG_STATUS_DRIVER_OK ++ | CONFIG_STATUS_FEATURES_OK, ++ CONFIG_STATUS_FAILED, ++ ) + { +- let queue_type = cloned_common_cfg.lock().unwrap().queue_type; +- let queues_config = &cloned_common_cfg.lock().unwrap().queues_config; +- let mut locked_queues = cloned_virtio_queue.lock().unwrap(); ++ let queue_type = cloned_pci_device.common_config.lock().unwrap().queue_type; ++ let queues_config = &cloned_pci_device ++ .common_config ++ .lock() ++ .unwrap() ++ .queues_config; ++ let mut locked_queues = cloned_pci_device.queues.lock().unwrap(); + for q_config in queues_config.iter() { + let queue = Queue::new(*q_config, queue_type).unwrap(); +- if !queue.is_valid(&cloned_sys_mem) { ++ if !queue.is_valid(&cloned_pci_device.sys_mem) { + error!("Failed to activate device: Invalid queue"); + return false; + } +@@ -675,10 +685,10 @@ impl VirtioPciDevice { + locked_queues.push(arc_queue.clone()); + } + +- let queue_evts = cloned_notify_evts.clone().events; +- if let Some(cb) = cloned_int_cb.clone() { +- if let Err(e) = cloned_virtio_dev.lock().unwrap().activate( +- cloned_sys_mem.clone(), ++ let queue_evts = cloned_pci_device.notify_eventfds.clone().events; ++ if let Some(cb) = cloned_pci_device.interrupt_cb.clone() { ++ if let Err(e) = cloned_pci_device.device.lock().unwrap().activate( ++ cloned_pci_device.sys_mem.clone(), + cb, + &locked_queues, + queue_evts, +@@ -689,20 +699,43 @@ impl VirtioPciDevice { + error!("Failed to activate device: No interrupt callback"); + return false; + } +- cloned_activated_flag.store(true, Ordering::Release); ++ cloned_pci_device ++ .device_activated ++ .store(true, Ordering::Release); ++ ++ update_dev_id( ++ &cloned_pci_device.parent_bus, ++ cloned_pci_device.devfn, ++ &cloned_pci_device.dev_id, ++ ); + } + +- if old_dev_status != 0 && cloned_common_cfg.lock().unwrap().device_status == 0 { +- let mut locked_queues = cloned_virtio_queue.lock().unwrap(); ++ if old_dev_status != 0 ++ && cloned_pci_device ++ .common_config ++ .lock() ++ .unwrap() ++ .device_status ++ == 0 ++ { ++ let mut locked_queues = cloned_pci_device.queues.lock().unwrap(); + locked_queues.clear(); +- cloned_activated_flag.store(false, Ordering::Release); ++ cloned_pci_device ++ .device_activated ++ .store(false, Ordering::Release); ++ let cloned_msix = cloned_pci_device.config.msix.as_ref().unwrap().clone(); + cloned_msix.lock().unwrap().reset(); +- if let Err(e) = cloned_virtio_dev.lock().unwrap().reset() { ++ if let Err(e) = cloned_pci_device.device.lock().unwrap().reset() { + error!( + "Failed to reset virtio device, error is {}", + e.display_chain() + ); + } ++ update_dev_id( ++ &cloned_pci_device.parent_bus, ++ cloned_pci_device.devfn, ++ &cloned_pci_device.dev_id, ++ ); + } + + true +@@ -874,15 +907,12 @@ impl PciDevOps for VirtioPciDevice { + self.modern_mem_region_map(notify_cap)?; + + let nvectors = self.device.lock().unwrap().queue_num() + 1; +- #[cfg(target_arch = "aarch64")] +- { +- self.dev_id = self.set_dev_id(0, self.devfn); +- } ++ + init_msix( + VIRTIO_PCI_MSIX_BAR_IDX as usize, + nvectors as u32, + &mut self.config, +- self.dev_id, ++ self.dev_id.clone(), + )?; + + self.assign_interrupt_cb(); +@@ -950,7 +980,8 @@ impl PciDevOps for VirtioPciDevice { + return; + } + +- self.config.write(offset, data, self.dev_id); ++ self.config ++ .write(offset, data, self.dev_id.clone().load(Ordering::Acquire)); + if ranges_overlap( + offset, + end, +@@ -1388,7 +1419,7 @@ mod tests { + VIRTIO_PCI_MSIX_BAR_IDX as usize, + virtio_pci.device.lock().unwrap().queue_num() as u32 + 1, + &mut virtio_pci.config, +- virtio_pci.dev_id, ++ virtio_pci.dev_id.clone(), + ) + .unwrap(); + // Prepare valid queue config +-- +2.25.1 + diff --git a/stratovirt.spec b/stratovirt.spec index 3f39253..79cac58 100644 --- a/stratovirt.spec +++ b/stratovirt.spec @@ -6,7 +6,7 @@ Name: stratovirt Version: 2.0.0 -Release: 2 +Release: 3 Summary: StratoVirt is an opensource VMM(Virtual Machine Manager) which aims to perform next generation virtualization. License: Mulan PSL v2 @@ -30,6 +30,15 @@ Based on Rust programming language, StratoVirt is lightweight, efficient and saf Summary: %{summary} +Patch001: 0001-fix-spelling-errors-in-project.patch +Patch002: 0002-migration-fix-an-error-during-migration-interface-on.patch +Patch003: 0003-migration-fix-an-errors-during-the-PL011-device-stat.patch +Patch004: 0004-machine-standard_vm-fix-inappropriate-file-open-perm.patch +Patch005: 0005-kernel_config-update-kernel-config-5.10-on-aarch64-p.patch +Patch006: 0006-syscall-add-syscall-newfstatat-in-x86_64-unknown-lin.patch +Patch007: 0007-vfio-fix-the-problem-of-dma-mapping-failed.patch +Patch008: 0008-virtio-fix-dev_id-initialization-for-virtio-pci-and-.patch + %files -n stratovirt %{_bindir}/stratovirt @@ -64,6 +73,16 @@ sed -i 's/rustflags = \[/rustflags = \["-Clink-arg=-lgcc", /g' ./.cargo/config chmod 555 ${RPM_BUILD_ROOT}/usr/bin/stratovirt %changelog +* Fri Aug 20 2021 Jie Yang - 2.0.0-3 +- virtio: fix dev_id initialization for virtio-pci and vfio device on aarch64 platform +- vfio: fix the problem of dma mapping failed +- syscall: add syscall "newfstatat" in x86_64-unknown-linux-gnu target +- kernel_config: update kernel config 5.10 on aarch64 platform +- machine/standard_vm: fix inappropriate file open permissions +- migration: fix an errors during the PL011 device state restore +- migration: fix an error during migration interface on aarch64 +- fix spelling errors in project + * Wed Aug 18 2021 Xinle.Guo - 2.0.0-2 - switch stratovirt permission from 550 to 555 -- Gitee