diff --git a/Cargo.lock b/Cargo.lock index 6ceed0bb0827195ed3d64cfd5cf2266c5692ef25..9f4125e7a85a07fbcf4687b6d060b02753d4e9ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1504,6 +1504,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.163" @@ -1794,10 +1803,14 @@ dependencies = [ "libc", "log", "machine_manager", + "migration", + "migration_derive", "once_cell", "rustls", "rustls-pemfile", "sasl2-sys", + "serde", + "serde-big-array", "serde_json", "sscanf", "thiserror", diff --git a/devices/src/misc/ivshmem.rs b/devices/src/misc/ivshmem.rs index 86b92381757430a0ffe321a822787fcbeb0ec6d6..0d7a7d7ae94469e356b25bd0465e72a4e836534c 100644 --- a/devices/src/misc/ivshmem.rs +++ b/devices/src/misc/ivshmem.rs @@ -15,17 +15,24 @@ use std::sync::{ Arc, Mutex, RwLock, Weak, }; -use anyhow::Result; +use anyhow::{bail, Context, Result}; use log::error; +use serde::{Deserialize, Serialize}; -use crate::pci::config::{ - PciConfig, RegionType, DEVICE_ID, PCI_CLASS_MEMORY_RAM, PCI_CONFIG_SPACE_SIZE, - PCI_VENDOR_ID_REDHAT_QUMRANET, REVISION_ID, SUB_CLASS_CODE, VENDOR_ID, -}; use crate::pci::msix::init_msix; +use crate::pci::{ + config::{ + PciConfig, RegionType, DEVICE_ID, PCI_CLASS_MEMORY_RAM, PCI_CONFIG_SPACE_SIZE, + PCI_VENDOR_ID_REDHAT_QUMRANET, REVISION_ID, SUB_CLASS_CODE, VENDOR_ID, + }, + PciState, +}; use crate::pci::{le_write_u16, PciBus, PciDevBase, PciDevOps}; use crate::{convert_bus_ref, Bus, Device, DeviceBase, PCI_BUS}; -use address_space::{GuestAddress, Region, RegionOps}; +use address_space::{register_ram_region, GuestAddress, Region, RegionOps}; +use migration::StateTransfer; +use migration::{DeviceStateDesc, MigrationHook, MigrationManager}; +use migration_derive::DescSerde; use util::gen_base_func; const PCI_VENDOR_ID_IVSHMEM: u16 = PCI_VENDOR_ID_REDHAT_QUMRANET; @@ -153,7 +160,9 @@ impl Ivshmem { RegionType::Mem64Bit, true, self.ram_mem_region.size(), - ) + )?; + let mem_mapping = self.ram_mem_region.mem_mapping.as_ref().unwrap().clone(); + register_ram_region(self.device_base().id.clone(), mem_mapping) } pub fn trigger_msix(&self, vector: u16) { @@ -210,6 +219,12 @@ impl Device for Ivshmem { .store(pci_bus.generate_dev_id(self.base.devfn), Ordering::Release); let dev = Arc::new(Mutex::new(self)); locked_bus.attach_child(u64::from(dev.lock().unwrap().base.devfn), dev.clone())?; + + MigrationManager::register_device_instance( + IvshmemState::descriptor(), + dev.clone(), + &dev.lock().unwrap().device_base().id, + ); Ok(dev) } @@ -219,6 +234,14 @@ impl Device for Ivshmem { } Ok(()) } + + fn unrealize(&mut self) -> Result<()> { + MigrationManager::unregister_device_instance( + IvshmemState::descriptor(), + &self.device_base().id, + ); + Ok(()) + } } impl PciDevOps for Ivshmem { @@ -238,3 +261,50 @@ impl PciDevOps for Ivshmem { ); } } + +/// Migration of ivshmem device. +#[derive(Clone, Deserialize, Serialize, DescSerde)] +#[desc_version(current_version = "0.1.0")] +struct IvshmemState { + dev_id: u16, + pci_state: PciState, +} + +impl StateTransfer for Ivshmem { + fn get_state_vec(&self) -> Result> { + let state = IvshmemState { + dev_id: self.dev_id.load(Ordering::Acquire), + pci_state: self.base.get_pci_state(), + }; + Ok(serde_json::to_vec(&state)?) + } + + fn set_state_mut(&mut self, state: &[u8]) -> Result<()> { + let ivshm_state: IvshmemState = serde_json::from_slice(state) + .with_context(|| migration::error::MigrationError::FromBytesError("Ivshmem"))?; + + self.dev_id.store(ivshm_state.dev_id, Ordering::Release); + self.base.set_pci_state(&ivshm_state.pci_state); + Ok(()) + } + + fn get_device_alias(&self) -> u64 { + MigrationManager::get_desc_alias(&IvshmemState::descriptor().name).unwrap_or(!0) + } +} + +impl MigrationHook for Ivshmem { + fn resume(&mut self) -> Result<()> { + let parent_bus = self.parent_bus().unwrap().upgrade().unwrap(); + PCI_BUS!(parent_bus, locked_bus, pci_bus); + if let Err(e) = self.base.config.update_bar_mapping( + #[cfg(target_arch = "x86_64")] + Some(&pci_bus.io_region), + Some(&pci_bus.mem_region), + ) { + bail!("Failed to update bar mapping for ivshmem, error is {:?}", e); + } + + Ok(()) + } +} diff --git a/devices/src/misc/scream/mod.rs b/devices/src/misc/scream/mod.rs index 73905abe63a07b32c643e5230007e28e0ad63702..67cb8eddd5605a2b1ab6874da2a1abb60338f8ce 100644 --- a/devices/src/misc/scream/mod.rs +++ b/devices/src/misc/scream/mod.rs @@ -28,6 +28,7 @@ use clap::{ArgAction, Parser}; use core::time; use log::{error, info, warn}; use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; #[cfg(feature = "scream_alsa")] use self::alsa::AlsaStreamData; @@ -39,6 +40,9 @@ use address_space::{GuestAddress, HostMemMapping, Region}; use machine_manager::config::{get_pci_df, parse_bool, valid_id}; use machine_manager::notifier::register_vm_pause_notifier; use machine_manager::state_query::register_state_query_callback; +use migration::StateTransfer; +use migration::{DeviceStateDesc, MigrationHook, MigrationManager}; +use migration_derive::DescSerde; #[cfg(all(target_env = "ohos", feature = "scream_ohaudio"))] use ohaudio::{OhAudio, OhAudioVolume}; #[cfg(feature = "scream_pulseaudio")] @@ -294,6 +298,19 @@ impl ScreamCond { fn stream_paused(&self) -> bool { *self.paused.lock().unwrap() != 0 } + + fn restore(&self, paused: u8) { + *self.paused.lock().unwrap() = paused; + } + + fn save(&self) -> u8 { + let ret = *self.paused.lock().unwrap(); + ret & !Self::VM_PAUSE_BIT + } + + fn notify(&self) { + self.cond.notify_all(); + } } /// Audio stream data structure. @@ -535,7 +552,7 @@ pub struct ScreamConfig { #[arg(long)] pub classtype: String, #[arg(long, value_parser = valid_id)] - id: String, + pub id: String, #[arg(long)] pub bus: String, #[arg(long, value_parser = get_pci_df)] @@ -559,6 +576,9 @@ pub struct Scream { config: ScreamConfig, token_id: Option>>, interface_resource: Vec>>, + play_cond: Arc, + capt_cond: Arc, + audio_ext: Option>, } impl Scream { @@ -582,6 +602,9 @@ impl Scream { config, token_id, interface_resource: Vec::new(), + play_cond: ScreamCond::new(), + capt_cond: ScreamCond::new(), + audio_ext: None, }) } @@ -703,9 +726,9 @@ impl Scream { let ivshmem = ivshmem.realize()?; let ivshmem_cloned = ivshmem.clone(); - let play_cond = ScreamCond::new(); - let capt_cond = ScreamCond::new(); - self.set_ivshmem_ops(ivshmem, play_cond.clone(), capt_cond.clone()); + let play_cond = self.play_cond.clone(); + let capt_cond = self.capt_cond.clone(); + self.audio_ext = Some(self.set_ivshmem_ops(ivshmem, play_cond.clone(), capt_cond.clone())); let author_notify = Arc::new(move || { ivshmem_cloned @@ -724,7 +747,7 @@ impl Scream { ivshmem: Arc>, play_cond: Arc, capt_cond: Arc, - ) { + ) -> Arc { let cloned_play_cond = play_cond.clone(); let cloned_capt_cond = capt_cond.clone(); let cb = Box::new(move || { @@ -735,6 +758,7 @@ impl Scream { ivshmem.lock().unwrap().register_reset_callback(cb); let interface = self.create_audio_extension(ivshmem.clone()); + let ret = interface.clone(); let interface2 = interface.clone(); let bar0_write = Arc::new(move |data: &[u8], offset: u64| { match offset { @@ -773,6 +797,8 @@ impl Scream { .lock() .unwrap() .set_bar0_ops((bar0_write, bar0_read)); + + ret } fn create_audio_extension(&self, _ivshmem: Arc>) -> Arc { @@ -804,6 +830,7 @@ pub trait AudioExtension: Send + Sync { false => 0, } } + fn notify_guest_sync(&self) {} } struct AudioExtensionDummy; @@ -812,3 +839,48 @@ impl AudioExtension for AudioExtensionDummy {} unsafe impl Send for AudioExtensionDummy {} // SAFETY: it is a dummy unsafe impl Sync for AudioExtensionDummy {} + +/// Migration support of Scream device. +#[derive(Clone, DescSerde, Serialize, Deserialize)] +#[desc_version(current_version = "0.1.0")] +pub struct ScreamState { + play_state: u8, + capt_state: u8, +} + +impl StateTransfer for Scream { + fn get_state_vec(&self) -> Result> { + let state = ScreamState { + play_state: self.play_cond.save(), + capt_state: self.capt_cond.save(), + }; + Ok(serde_json::to_vec(&state)?) + } + + fn set_state_mut(&mut self, state: &[u8]) -> Result<()> { + let scream_state: ScreamState = serde_json::from_slice(state) + .with_context(|| migration::error::MigrationError::FromBytesError("Scream"))?; + self.play_cond.restore(scream_state.play_state); + self.capt_cond.restore(scream_state.capt_state); + Ok(()) + } + + fn get_device_alias(&self) -> u64 { + MigrationManager::get_desc_alias(&ScreamState::descriptor().name).unwrap_or(!0) + } +} + +impl MigrationHook for Scream { + fn resume(&mut self) -> Result<()> { + self.play_cond.notify(); + self.capt_cond.notify(); + // Notify the guest to synchronize volume. + if let Some(ext) = self.audio_ext.as_ref() { + ext.notify_guest_sync(); + } + // Synchronize record authorization. + let auth = get_record_authority(); + set_record_authority(auth); + Ok(()) + } +} diff --git a/devices/src/misc/scream/ohaudio.rs b/devices/src/misc/scream/ohaudio.rs index 315c6c337977cd39f4ab6d883962a40ebd3badc0..0e0b1cb04328bf217587ab26bd5996216fe86761 100755 --- a/devices/src/misc/scream/ohaudio.rs +++ b/devices/src/misc/scream/ohaudio.rs @@ -680,6 +680,13 @@ impl AudioExtension for OhAudioVolume { fn set_host_volume(&self, vol: u32) { set_ohos_volume(self.to_host_vol(vol)); } + + fn notify_guest_sync(&self) { + self.shm_dev + .lock() + .unwrap() + .trigger_msix(IVSHMEM_VOLUME_SYNC_VECTOR); + } } impl OhAudioVolume { diff --git a/machine/src/aarch64/standard.rs b/machine/src/aarch64/standard.rs index 1e6123bc73b8634d0923492f6b1f02277f50ebc5..09a332e7d88108ece2563485a86fdf7b3c2dadea 100644 --- a/machine/src/aarch64/standard.rs +++ b/machine/src/aarch64/standard.rs @@ -312,10 +312,7 @@ impl StdMachine { if dpy.display_type != "ohui" { return Ok(()); } - self.ohui_server = Some(Arc::new(OhUiServer::new( - dpy.get_ui_path(), - dpy.get_sock_path(), - )?)); + self.ohui_server = Some(OhUiServer::new(dpy.get_ui_path(), dpy.get_sock_path())?); } Ok(()) } diff --git a/machine/src/lib.rs b/machine/src/lib.rs index 69a0a1618b4d41a9221b8bcdc4b3578d9230afd4..ec7628aedc08fffae418418a521ae751ef211104 100644 --- a/machine/src/lib.rs +++ b/machine/src/lib.rs @@ -62,7 +62,7 @@ use devices::legacy::FwCfgOps; #[cfg(feature = "pvpanic")] use devices::misc::pvpanic::{PvPanicPci, PvpanicDevConfig}; #[cfg(feature = "scream")] -use devices::misc::scream::{Scream, ScreamConfig}; +use devices::misc::scream::{Scream, ScreamConfig, ScreamState}; #[cfg(feature = "demo_device")] use devices::pci::demo_device::{DemoDev, DemoDevConfig}; use devices::pci::{ @@ -1802,10 +1802,17 @@ pub trait MachineOps: MachineLifecycle { bail!("Object for share config is not on"); } + let id = config.id.clone(); let mut scream = Scream::new(mem_cfg.size, config, token_id)?; scream .realize(parent_bus) - .with_context(|| "Failed to realize scream device") + .with_context(|| "Failed to realize scream device")?; + MigrationManager::register_device_instance( + ScreamState::descriptor(), + Arc::new(Mutex::new(scream)), + &id, + ); + Ok(()) } /// Get the corresponding device from the PCI bus based on the device id and device type name. diff --git a/migration/src/snapshot.rs b/migration/src/snapshot.rs index 986244b1818cda1e595f8d4ead425a68e39ba79e..377dc8521ed9c5c66ab8e4dd737cddab53764b13 100644 --- a/migration/src/snapshot.rs +++ b/migration/src/snapshot.rs @@ -31,6 +31,7 @@ pub const PL031_SNAPSHOT_ID: &str = "pl031"; pub const RAMFB_SNAPSHOT_ID: &str = "ramfb"; pub const FWCFG_SNAPSHOT_ID: &str = "fwcfg"; pub const GED_SNAPSHOT_ID: &str = "ged"; +pub const OHUI_SNAPSHOT_ID: &str = "ohui"; /// The suffix used for snapshot memory storage. const MEMORY_PATH_SUFFIX: &str = "memory"; diff --git a/ui/Cargo.toml b/ui/Cargo.toml index b7faa42406adbf0ba8776cfcc0d385c97f8d1860..e347d2fbd22189257e5bdc9b63e64626257b3655 100644 --- a/ui/Cargo.toml +++ b/ui/Cargo.toml @@ -12,6 +12,8 @@ anyhow = "1.0" libc = "0.2" log = "0.4" serde_json = "1.0" +serde = { version = "1.0", features = ["derive"] } +serde-big-array = "0.5.1" vmm-sys-util = "0.12.1" once_cell = "1.18.0" sscanf = "0.4.1" @@ -26,6 +28,8 @@ machine_manager = { path = "../machine_manager" } util = { path = "../util" } trace = { path = "../trace" } address_space = { path = "../address_space" } +migration = { path = "../migration" } +migration_derive = { path = "../migration/migration_derive" } [features] keycode = [] diff --git a/ui/src/ohui_srv/mod.rs b/ui/src/ohui_srv/mod.rs index 7fc8b631330a3950a626eeb503ae71b67b1a8edb..eda582c5a9f8a2287aba020a5f1d35bac9b1162b 100755 --- a/ui/src/ohui_srv/mod.rs +++ b/ui/src/ohui_srv/mod.rs @@ -26,6 +26,7 @@ use std::sync::{ use anyhow::{anyhow, bail, Context, Result}; use log::{error, info}; use once_cell::sync::OnceCell; +use serde::{Deserialize, Serialize}; use vmm_sys_util::epoll::EventSet; use crate::{ @@ -43,6 +44,9 @@ use machine_manager::{ event_loop::register_event_helper, temp_cleaner::TempCleaner, }; +use migration::snapshot::OHUI_SNAPSHOT_ID; +use migration::{DeviceStateDesc, MigrationHook, MigrationManager, StateTransfer}; +use migration_derive::DescSerde; use msg_handle::*; use util::{ loop_context::{ @@ -85,6 +89,15 @@ impl GuestSurface { const CURSOR_SIZE: u64 = 16 * 1024; +struct CursorInfo { + buffer: u64, + width: u32, + height: u32, + hot_x: u32, + hot_y: u32, + _file: FileBackend, +} + pub struct OhUiServer { // framebuffer passthru to the guest passthru: OnceCell, @@ -98,14 +111,14 @@ pub struct OhUiServer { connected: AtomicBool, // iothread processing unix socket iothread: OnceCell>, - // address of cursor buffer - cursorbuffer: u64, //address of framebuffer framebuffer: u64, // framebuffer file backend fb_file: Option, // tokenID of OHUI client pub token_id: Arc>, + // Cursor + cursor: Arc>, } impl OhUiServer { @@ -144,7 +157,7 @@ impl OhUiServer { Ok((Some(fb_backend), host_addr)) } - fn init_cursor_file(path: &String) -> Result { + fn init_cursor_file(path: &String) -> Result<(FileBackend, u64)> { let file_path = Path::new(path.as_str()).join("ohui-cursor"); let cursor_file = file_path .to_str() @@ -167,26 +180,44 @@ impl OhUiServer { false, )?; - Ok(cursorbuffer) + Ok((cursor_backend, cursorbuffer)) } - pub fn new(ui_path: String, sock_path: String) -> Result { + pub fn new(ui_path: String, sock_path: String) -> Result> { let channel = Self::init_channel(&sock_path)?; let (fb_file, framebuffer) = Self::init_fb_file(&ui_path)?; - let cursorbuffer = Self::init_cursor_file(&ui_path)?; - - Ok(OhUiServer { + let (cursor_file, cursorbuffer) = Self::init_cursor_file(&ui_path)?; + let cursor = Arc::new(Mutex::new(CursorInfo { + buffer: cursorbuffer, + width: 0, + height: 0, + hot_x: 0, + hot_y: 0, + _file: cursor_file, + })); + let ohui_srv = Arc::new(OhUiServer { passthru: OnceCell::new(), surface: RwLock::new(GuestSurface::new()), channel, msg_handler: OhUiMsgHandler::new(), connected: AtomicBool::new(false), iothread: OnceCell::new(), - cursorbuffer, framebuffer, fb_file, token_id: Arc::new(RwLock::new(0)), - }) + cursor: cursor.clone(), + }); + + MigrationManager::register_device_instance( + OhUiMigrationState::descriptor(), + Arc::new(Mutex::new(OhUiMigration { + cursor, + ohui_srv: ohui_srv.clone(), + })), + OHUI_SNAPSHOT_ID, + ); + + Ok(ohui_srv) } pub fn set_passthru(&self, passthru: bool) { @@ -365,7 +396,8 @@ impl DisplayChangeListenerOperations for OhUiServer { } fn dpy_cursor_update(&self, cursor: &DisplayMouse) -> Result<()> { - if self.cursorbuffer == 0 { + let mut locked_cursor = self.cursor.lock().unwrap(); + if locked_cursor.buffer == 0 { error!("Hwcursor not set."); // No need to return Err for this situation is not fatal return Ok(()); @@ -387,10 +419,14 @@ impl DisplayChangeListenerOperations for OhUiServer { unsafe { ptr::copy_nonoverlapping( cursor.data.as_ptr(), - self.cursorbuffer as *mut u8, + locked_cursor.buffer as *mut u8, len as usize, ); } + locked_cursor.width = cursor.width; + locked_cursor.height = cursor.height; + locked_cursor.hot_x = cursor.hot_x; + locked_cursor.hot_y = cursor.hot_y; self.msg_handler.handle_cursor_define( cursor.width, @@ -556,3 +592,78 @@ fn ohui_start_listener(server: Arc) -> Result<()> { info!("Successfully start listener."); Ok(()) } + +/// Migration +#[derive(Clone, Debug, Default, DescSerde, Serialize, Deserialize)] +#[desc_version(current_version = "0.1.0")] +struct OhUiMigrationState { + cursor_img: Vec, + width: u32, + height: u32, + hot_x: u32, + hot_y: u32, +} + +struct OhUiMigration { + cursor: Arc>, + ohui_srv: Arc, +} + +impl StateTransfer for OhUiMigration { + fn get_state_vec(&self) -> Result> { + let mut state = OhUiMigrationState::default(); + state.cursor_img.resize(CURSOR_SIZE as usize, 0); + + let cursor = self.cursor.lock().unwrap(); + // SAFETY: the buffer is initialized and being kept for the whole VM lifecycle. + unsafe { + ptr::copy_nonoverlapping( + cursor.buffer as *const u8, + state.cursor_img.as_mut_ptr(), + CURSOR_SIZE as usize, + ); + } + state.width = cursor.width; + state.height = cursor.height; + state.hot_x = cursor.hot_x; + state.hot_y = cursor.hot_y; + Ok(serde_json::to_vec(&state)?) + } + + fn set_state_mut(&mut self, state: &[u8]) -> Result<()> { + let mgt_state: OhUiMigrationState = serde_json::from_slice(state) + .with_context(|| migration::error::MigrationError::FromBytesError("OHUI"))?; + let mut cursor = self.cursor.lock().unwrap(); + // SAFETY: the buffer is initialized and being kept for the whole VM lifecycle. + unsafe { + ptr::copy_nonoverlapping( + mgt_state.cursor_img.as_ptr(), + cursor.buffer as *mut u8, + CURSOR_SIZE as usize, + ); + } + cursor.width = mgt_state.width; + cursor.height = mgt_state.height; + cursor.hot_x = mgt_state.hot_x; + cursor.hot_y = mgt_state.hot_y; + Ok(()) + } + + fn get_device_alias(&self) -> u64 { + MigrationManager::get_desc_alias(&OhUiMigrationState::descriptor().name).unwrap_or(0) + } +} + +impl MigrationHook for OhUiMigration { + fn resume(&mut self) -> Result<()> { + let cursor = self.cursor.lock().unwrap(); + self.ohui_srv.msg_handler.handle_cursor_define( + cursor.width, + cursor.height, + cursor.hot_x, + cursor.hot_y, + bytes_per_pixel() as u32, + ); + Ok(()) + } +}