diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 9021c1180637464daa581c63488fde25b0cdf289..a188690400f1e48f3f3ae6e16c1fbd63c827e678 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -5830,19 +5830,24 @@ CONFIG_VIRTIO_PCI_LIB=m CONFIG_VIRTIO_MENU=y CONFIG_VIRTIO_PCI=m CONFIG_VIRTIO_PCI_LEGACY=y +# CONFIG_VIRTIO_VDPA is not set # CONFIG_VIRTIO_PMEM is not set CONFIG_VIRTIO_BALLOON=m CONFIG_VIRTIO_INPUT=m CONFIG_VIRTIO_MMIO=m # CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES is not set CONFIG_VIRTIO_DMA_SHARED_BUFFER=m -# CONFIG_VDPA is not set +CONFIG_VDPA=m +# CONFIG_IFCVF is not set +# CONFIG_MLX5_VDPA_NET is not set +# CONFIG_VP_VDPA is not set CONFIG_VHOST_IOTLB=m CONFIG_VHOST=m CONFIG_VHOST_MENU=y CONFIG_VHOST_NET=m CONFIG_VHOST_SCSI=m CONFIG_VHOST_VSOCK=m +CONFIG_VHOST_VDPA=m # CONFIG_VHOST_CROSS_ENDIAN_LEGACY is not set # diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index b888efd65e92964041ec2ce8fc1f187f2a9acc72..6d4f515294f9c6e4a0f866a390941725412750e6 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -3040,6 +3040,7 @@ void iommu_get_resv_regions(struct device *dev, struct list_head *list) if (ops && ops->get_resv_regions) ops->get_resv_regions(dev, list); } +EXPORT_SYMBOL_GPL(iommu_get_resv_regions); void iommu_put_resv_regions(struct device *dev, struct list_head *list) { @@ -3048,6 +3049,7 @@ void iommu_put_resv_regions(struct device *dev, struct list_head *list) if (ops && ops->put_resv_regions) ops->put_resv_regions(dev, list); } +EXPORT_SYMBOL_GPL(iommu_put_resv_regions); /** * generic_iommu_put_resv_regions - Reserved region driver helper diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 4afd4ee4f7f04f8d4aed5c7d0f9fc79dd547be0d..08db0610da7680f80e03f3f1f4586e5f4913b841 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -32,6 +32,20 @@ int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id) dev->sriov->stride * vf_id) & 0xff; } +int pci_iov_vf_id(struct pci_dev *dev) +{ + struct pci_dev *pf; + + if (!dev->is_virtfn) + return -EINVAL; + + pf = pci_physfn(dev); + return (((dev->bus->number << 8) + dev->devfn) - + ((pf->bus->number << 8) + pf->devfn + pf->sriov->offset)) / + pf->sriov->stride; +} +EXPORT_SYMBOL_GPL(pci_iov_vf_id); + /* * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may * change when NumVFs changes. diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index 862f4591fff2a2b4c77fecc912fb15b0c99ed970..2a4991464cfd1cabd103994638b7d482e8330b9b 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -233,7 +233,7 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) ifcvf_set_status(vf, status); } -static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev) +static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev, int state) { struct ifcvf_adapter *adapter; struct ifcvf_hw *vf; diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index 76e7b4bab9d3b28f31df7396b75c0b8fd4fd8e0c..c676bb70a45379cc38576d653d8f083ff8d14809 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -1800,7 +1800,7 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; } -static int mlx5_vdpa_reset(struct vdpa_device *vdev) +static int mlx5_vdpa_reset(struct vdpa_device *vdev, int state) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index c141f08b82740b3dc562d0e72b9d050dbb7ad6fb..be925462922d8c2cee08887e6b1402f1b74bd4e4 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -454,7 +454,7 @@ static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status) spin_unlock(&vdpasim->lock); } -static int vdpasim_reset(struct vdpa_device *vdpa) +static int vdpasim_reset(struct vdpa_device *vdpa, int state) { struct vdpasim *vdpasim = vdpa_to_sim(vdpa); diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c index f4e375b1d903c3d34b11126a3f73561945c59ae4..bc0bdcc7bceae9ebbe5bd3ea4c31d6813193fd23 100644 --- a/drivers/vdpa/virtio_pci/vp_vdpa.c +++ b/drivers/vdpa/virtio_pci/vp_vdpa.c @@ -220,7 +220,7 @@ static void vp_vdpa_set_status(struct vdpa_device *vdpa, u8 status) vp_modern_set_status(mdev, status); } -static int vp_vdpa_reset(struct vdpa_device *vdpa) +static int vp_vdpa_reset(struct vdpa_device *vdpa, int state) { struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index ebafc05d2b74f1d14b52b0d9dc8b379c2cbe03e4..eed51d0045313598b582bcac21c639df25563c9c 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "vhost.h" @@ -49,6 +50,7 @@ struct vhost_vdpa { struct completion completion; struct vdpa_device *vdpa; struct hlist_head as[VHOST_VDPA_IOTLB_BUCKETS]; + struct vhost_iotlb resv_iotlb; struct device dev; struct cdev cdev; atomic_t opened; @@ -65,6 +67,10 @@ static DEFINE_IDA(vhost_vdpa_ida); static dev_t vhost_vdpa_major; +static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, + struct vhost_iotlb *iotlb, u64 start, + u64 last, u32 asid); + static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb) { struct vhost_vdpa_as *as = container_of(iotlb, struct @@ -135,7 +141,7 @@ static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid) return -EINVAL; hlist_del(&as->hash_link); - vhost_iotlb_reset(&as->iotlb); + vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid); kfree(as); return 0; @@ -206,13 +212,35 @@ static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid) irq_bypass_unregister_producer(&vq->call_ctx.producer); } -static int vhost_vdpa_reset(struct vhost_vdpa *v) +static int vhost_vdpa_reset(struct vhost_vdpa *v, int state) { struct vdpa_device *vdpa = v->vdpa; v->in_batch = 0; - return vdpa_reset(vdpa); + return vdpa_reset(vdpa, state); +} + +static long vhost_vdpa_bind_mm(struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + if (!vdpa->use_va || !ops->bind_mm) + return 0; + + return ops->bind_mm(vdpa, v->vdev.mm); +} + +static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + if (!vdpa->use_va || !ops->unbind_mm) + return; + + ops->unbind_mm(vdpa); } static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp) @@ -269,7 +297,7 @@ static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) vhost_vdpa_unsetup_vq_irq(v, i); if (status == 0) { - ret = vdpa_reset(vdpa); + ret = vdpa_reset(vdpa, VDPA_DEV_RESET_VIRTIO); if (ret) return ret; } else @@ -355,6 +383,14 @@ static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v) return ops->suspend; } +static bool vhost_vdpa_can_resume(const struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + return ops->resume; +} + static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep) { struct vdpa_device *vdpa = v->vdpa; @@ -369,25 +405,52 @@ static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep) return 0; } +static u64 vhost_vdpa_get_backend_features(const struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + if (!ops->get_backend_features) + return 0; + else + return ops->get_backend_features(vdpa); +} + static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep) { struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; + struct vhost_dev *d = &v->vdev; + u64 actual_features; u64 features; + int i; + + if (copy_from_user(&features, featurep, sizeof(features))) + return -EFAULT; + + actual_features = ops->get_driver_features(vdpa); /* * It's not allowed to change the features after they have - * been negotiated. + * been negotiated. But log start/end is allowed. */ - if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK) + if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK) && + (features & ~(BIT_ULL(VHOST_F_LOG_ALL))) != actual_features) return -EBUSY; - if (copy_from_user(&features, featurep, sizeof(features))) - return -EFAULT; - if (vdpa_set_features(vdpa, features)) return -EINVAL; + /* let the vqs know what has been configured */ + actual_features = ops->get_driver_features(vdpa); + for (i = 0; i < d->nvqs; ++i) { + struct vhost_virtqueue *vq = d->vqs[i]; + + mutex_lock(&vq->mutex); + vq->acked_features = actual_features; + mutex_unlock(&vq->mutex); + } + return 0; } @@ -494,6 +557,139 @@ static long vhost_vdpa_suspend(struct vhost_vdpa *v) return ops->suspend(vdpa); } +/* After a successful return of this ioctl the device resumes processing + * virtqueue descriptors. The device becomes fully operational the same way it + * was before it was suspended. + */ +static long vhost_vdpa_resume(struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + if (!ops->resume) + return -EOPNOTSUPP; + + return ops->resume(vdpa); +} + +static int vhost_vdpa_get_dev_buffer_size(struct vhost_vdpa *v, + uint32_t __user *argp) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + uint32_t size; + + if (!ops->get_dev_buffer_size) + return -EOPNOTSUPP; + + size = ops->get_dev_buffer_size(vdpa); + + if (copy_to_user(argp, &size, sizeof(size))) + return -EFAULT; + + return 0; +} + +static int vhost_vdpa_get_dev_buffer(struct vhost_vdpa *v, + struct vhost_vdpa_config __user *c) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vhost_vdpa_config config; + int ret; + unsigned long size = offsetof(struct vhost_vdpa_config, buf); + + if (copy_from_user(&config, c, size)) + return -EFAULT; + + if (!ops->get_dev_buffer) + return -EOPNOTSUPP; + + down_read(&vdpa->cf_lock); + ret = ops->get_dev_buffer(vdpa, config.off, c->buf, config.len); + up_read(&vdpa->cf_lock); + + return ret; +} + +static int vhost_vdpa_set_dev_buffer(struct vhost_vdpa *v, + struct vhost_vdpa_config __user *c) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vhost_vdpa_config config; + int ret; + unsigned long size = offsetof(struct vhost_vdpa_config, buf); + + if (copy_from_user(&config, c, size)) + return -EFAULT; + + if (!ops->set_dev_buffer) + return -EOPNOTSUPP; + + down_write(&vdpa->cf_lock); + ret = ops->set_dev_buffer(vdpa, config.off, c->buf, config.len); + up_write(&vdpa->cf_lock); + + return ret; +} + +static int vhost_vdpa_set_mig_state(struct vhost_vdpa *v, u8 __user *c) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + u8 state; + + if (!ops->set_mig_state) + return -EOPNOTSUPP; + + if (get_user(state, c)) + return -EFAULT; + + return ops->set_mig_state(vdpa, state); +} + +static long vhost_vdpa_set_log_base(struct vhost_vdpa *v, u64 __user *argp) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + u64 log; + + if (!ops->set_log_base) + return -EOPNOTSUPP; + + if (copy_from_user(&log, argp, sizeof(uint64_t))) + return -EFAULT; + + return ops->set_log_base(vdpa, log); +} + +static long vhost_vdpa_set_log_size(struct vhost_vdpa *v, u64 __user *sizep) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + u64 log_size; + + if (!ops->set_log_size) + return -EOPNOTSUPP; + + if (copy_from_user(&log_size, sizep, sizeof(log_size))) + return -EFAULT; + + return ops->set_log_size(vdpa, log_size); +} + +static long vhost_vdpa_log_sync(struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + if (!ops->log_sync) + return -EOPNOTSUPP; + + return ops->log_sync(vdpa); +} + static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, void __user *argp) { @@ -545,7 +741,14 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, if (r) return r; - vq->last_avail_idx = vq_state.split.avail_index; + if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) { + vq->last_avail_idx = vq_state.packed.last_avail_idx | + (vq_state.packed.last_avail_counter << 15); + vq->last_used_idx = vq_state.packed.last_used_idx | + (vq_state.packed.last_used_counter << 15); + } else { + vq->last_avail_idx = vq_state.split.avail_index; + } break; } @@ -563,9 +766,15 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, break; case VHOST_SET_VRING_BASE: - vq_state.split.avail_index = vq->last_avail_idx; - if (ops->set_vq_state(vdpa, idx, &vq_state)) - r = -EINVAL; + if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) { + vq_state.packed.last_avail_idx = vq->last_avail_idx & 0x7fff; + vq_state.packed.last_avail_counter = !!(vq->last_avail_idx & 0x8000); + vq_state.packed.last_used_idx = vq->last_used_idx & 0x7fff; + vq_state.packed.last_used_counter = !!(vq->last_used_idx & 0x8000); + } else { + vq_state.split.avail_index = vq->last_avail_idx; + } + r = ops->set_vq_state(vdpa, idx, &vq_state); break; case VHOST_SET_VRING_CALL: @@ -602,11 +811,16 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, if (copy_from_user(&features, featurep, sizeof(features))) return -EFAULT; if (features & ~(VHOST_VDPA_BACKEND_FEATURES | - BIT_ULL(VHOST_BACKEND_F_SUSPEND))) + BIT_ULL(VHOST_BACKEND_F_SUSPEND) | + BIT_ULL(VHOST_BACKEND_F_RESUME) | + BIT_ULL(VHOST_BACKEND_F_BYTEMAPLOG))) return -EOPNOTSUPP; if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) && !vhost_vdpa_can_suspend(v)) return -EOPNOTSUPP; + if ((features & BIT_ULL(VHOST_BACKEND_F_RESUME)) && + !vhost_vdpa_can_resume(v)) + return -EOPNOTSUPP; vhost_set_backend_features(&v->vdev, features); return 0; } @@ -648,6 +862,14 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, r = -EFAULT; break; case VHOST_SET_LOG_BASE: + r = vhost_vdpa_set_log_base(v, argp); + break; + case VHOST_SET_LOG_SIZE: + r = vhost_vdpa_set_log_size(v, argp); + break; + case VHOST_LOG_SYNC: + r = vhost_vdpa_log_sync(v); + break; case VHOST_SET_LOG_FD: r = -ENOIOCTLCMD; break; @@ -658,6 +880,9 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, features = VHOST_VDPA_BACKEND_FEATURES; if (vhost_vdpa_can_suspend(v)) features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND); + if (vhost_vdpa_can_resume(v)) + features |= BIT_ULL(VHOST_BACKEND_F_RESUME); + features |= vhost_vdpa_get_backend_features(v); if (copy_to_user(featurep, &features, sizeof(features))) r = -EFAULT; break; @@ -673,6 +898,21 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, case VHOST_VDPA_SUSPEND: r = vhost_vdpa_suspend(v); break; + case VHOST_VDPA_RESUME: + r = vhost_vdpa_resume(v); + break; + case VHOST_GET_DEV_BUFFER_SIZE: + r = vhost_vdpa_get_dev_buffer_size(v, argp); + break; + case VHOST_GET_DEV_BUFFER: + r = vhost_vdpa_get_dev_buffer(v, argp); + break; + case VHOST_SET_DEV_BUFFER: + r = vhost_vdpa_set_dev_buffer(v, argp); + break; + case VHOST_VDPA_SET_MIG_STATE: + r = vhost_vdpa_set_mig_state(v, argp); + break; default: r = vhost_dev_ioctl(&v->vdev, cmd, argp); if (r == -ENOIOCTLCMD) @@ -680,13 +920,34 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, break; } + if (r) + goto out; + + switch (cmd) { + case VHOST_SET_OWNER: + r = vhost_vdpa_bind_mm(v); + if (r) + vhost_dev_reset_owner(d, NULL); + break; + } +out: mutex_unlock(&d->mutex); return r; } +static void vhost_vdpa_general_unmap(struct vhost_vdpa *v, + struct vhost_iotlb_map *map, u32 asid) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + if (ops->dma_map) { + ops->dma_unmap(vdpa, asid, map->start, map->size); + } else if (ops->set_map == NULL) { + iommu_unmap(v->domain, map->start, map->size); + } +} -static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, - struct vhost_iotlb *iotlb, - u64 start, u64 last) +static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb, + u64 start, u64 last, u32 asid) { struct vhost_dev *dev = &v->vdev; struct vhost_iotlb_map *map; @@ -703,13 +964,13 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, unpin_user_page(page); } atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm); + vhost_vdpa_general_unmap(v, map, asid); vhost_iotlb_map_free(iotlb, map); } } -static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, - struct vhost_iotlb *iotlb, - u64 start, u64 last) +static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb, + u64 start, u64 last, u32 asid) { struct vhost_iotlb_map *map; struct vdpa_map_file *map_file; @@ -718,20 +979,21 @@ static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, map_file = (struct vdpa_map_file *)map->opaque; fput(map_file->file); kfree(map_file); + vhost_vdpa_general_unmap(v, map, asid); vhost_iotlb_map_free(iotlb, map); } } static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, - struct vhost_iotlb *iotlb, - u64 start, u64 last) + struct vhost_iotlb *iotlb, u64 start, + u64 last, u32 asid) { struct vdpa_device *vdpa = v->vdpa; if (vdpa->use_va) - return vhost_vdpa_va_unmap(v, iotlb, start, last); + return vhost_vdpa_va_unmap(v, iotlb, start, last, asid); - return vhost_vdpa_pa_unmap(v, iotlb, start, last); + return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid); } static int perm_to_iommu_flags(u32 perm) @@ -798,22 +1060,13 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v, const struct vdpa_config_ops *ops = vdpa->config; u32 asid = iotlb_to_asid(iotlb); - vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1); + vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid); - if (ops->dma_map) { - ops->dma_unmap(vdpa, asid, iova, size); - } else if (ops->set_map) { + if (ops->set_map) { if (!v->in_batch) ops->set_map(vdpa, asid, iotlb); - } else { - iommu_unmap(v->domain, iova, size); } - /* If we are in the middle of batch processing, delay the free - * of AS until BATCH_END. - */ - if (!v->in_batch && !iotlb->nmaps) - vhost_vdpa_remove_as(v, asid); } static int vhost_vdpa_va_map(struct vhost_vdpa *v, @@ -1003,6 +1256,10 @@ static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v, msg->iova + msg->size - 1 > v->range.last) return -EINVAL; + if (vhost_iotlb_itree_first(&v->resv_iotlb, msg->iova, + msg->iova + msg->size - 1)) + return -EINVAL; + if (vhost_iotlb_itree_first(iotlb, msg->iova, msg->iova + msg->size - 1)) return -EEXIST; @@ -1070,8 +1327,6 @@ static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid, if (v->in_batch && ops->set_map) ops->set_map(vdpa, asid, iotlb); v->in_batch = false; - if (!iotlb->nmaps) - vhost_vdpa_remove_as(v, asid); break; default: r = -EINVAL; @@ -1093,6 +1348,46 @@ static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb, return vhost_chr_write_iter(dev, from); } +static int vhost_vdpa_resv_iommu_region(struct iommu_domain *domain, struct device *dma_dev, + struct vhost_iotlb *resv_iotlb) +{ + struct list_head dev_resv_regions; + phys_addr_t resv_msi_base = 0; + struct iommu_resv_region *region; + int ret = 0; + bool with_sw_msi = false; + bool with_hw_msi = false; + + INIT_LIST_HEAD(&dev_resv_regions); + iommu_get_resv_regions(dma_dev, &dev_resv_regions); + + list_for_each_entry(region, &dev_resv_regions, list) { + ret = vhost_iotlb_add_range_ctx(resv_iotlb, region->start, + region->start + region->length - 1, + 0, 0, NULL); + if (ret) { + vhost_iotlb_reset(resv_iotlb); + break; + } + + if (region->type == IOMMU_RESV_MSI) + with_hw_msi = true; + + if (region->type == IOMMU_RESV_SW_MSI) { + resv_msi_base = region->start; + with_sw_msi = true; + } + + } + + if (!ret && !with_hw_msi && with_sw_msi) + ret = iommu_get_msi_cookie(domain, resv_msi_base); + + iommu_put_resv_regions(dma_dev, &dev_resv_regions); + + return ret; +} + static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v) { struct vdpa_device *vdpa = v->vdpa; @@ -1118,12 +1413,18 @@ static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v) ret = iommu_attach_device(v->domain, dma_dev); if (ret) - goto err_attach; + goto err_alloc_domain; - return 0; + ret = vhost_vdpa_resv_iommu_region(v->domain, dma_dev, &v->resv_iotlb); + if (ret) + goto err_attach_device; -err_attach: + return 0; +err_attach_device: + iommu_detach_device(v->domain, dma_dev); +err_alloc_domain: iommu_domain_free(v->domain); + v->domain = NULL; return ret; } @@ -1166,14 +1467,15 @@ static void vhost_vdpa_cleanup(struct vhost_vdpa *v) struct vhost_vdpa_as *as; u32 asid; - vhost_dev_cleanup(&v->vdev); - kfree(v->vdev.vqs); - for (asid = 0; asid < v->vdpa->nas; asid++) { as = asid_to_as(v, asid); if (as) vhost_vdpa_remove_as(v, asid); } + + vhost_vdpa_free_domain(v); + vhost_dev_cleanup(&v->vdev); + kfree(v->vdev.vqs); } static int vhost_vdpa_open(struct inode *inode, struct file *filep) @@ -1189,9 +1491,12 @@ static int vhost_vdpa_open(struct inode *inode, struct file *filep) opened = atomic_cmpxchg(&v->opened, 0, 1); if (opened) return -EBUSY; + r = vhost_vdpa_alloc_domain(v); + if (r) + return r; nvqs = v->nvqs; - r = vhost_vdpa_reset(v); + r = vhost_vdpa_reset(v, VDPA_DEV_RESET_OPEN); if (r) goto err; @@ -1209,19 +1514,14 @@ static int vhost_vdpa_open(struct inode *inode, struct file *filep) vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false, vhost_vdpa_process_iotlb_msg); - r = vhost_vdpa_alloc_domain(v); - if (r) - goto err_alloc_domain; - vhost_vdpa_set_iova_range(v); filep->private_data = v; return 0; -err_alloc_domain: - vhost_vdpa_cleanup(v); err: + vhost_vdpa_free_domain(v); atomic_dec(&v->opened); return r; } @@ -1242,9 +1542,9 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep) mutex_lock(&d->mutex); filep->private_data = NULL; vhost_vdpa_clean_irq(v); - vhost_vdpa_reset(v); + vhost_vdpa_reset(v, VDPA_DEV_RESET_CLOSE); vhost_dev_stop(&v->vdev); - vhost_vdpa_free_domain(v); + vhost_vdpa_unbind_mm(v); vhost_vdpa_config_put(v); vhost_vdpa_cleanup(v); mutex_unlock(&d->mutex); @@ -1379,6 +1679,8 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa) goto err; } + vhost_iotlb_init(&v->resv_iotlb, 0, 0); + r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor); if (r) goto err; diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 46c71653f508ba841e5bdcc7d7b211baba34db44..8126a9be22cd41516f482f65bbd1cc5d9f53bddf 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -99,7 +99,7 @@ static void virtio_vdpa_reset(struct virtio_device *vdev) { struct vdpa_device *vdpa = vd_get_vdpa(vdev); - vdpa_reset(vdpa); + vdpa_reset(vdpa, VDPA_DEV_RESET_VIRTIO); } static bool virtio_vdpa_notify(struct virtqueue *vq) diff --git a/include/linux/pci.h b/include/linux/pci.h index 46de0ade5c69272215b4ba4ded80f817c919567f..f1665a454553d53706920e3b2e43bb28cf7f3d90 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2119,7 +2119,7 @@ void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar); #ifdef CONFIG_PCI_IOV int pci_iov_virtfn_bus(struct pci_dev *dev, int id); int pci_iov_virtfn_devfn(struct pci_dev *dev, int id); - +int pci_iov_vf_id(struct pci_dev *dev); int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn); void pci_disable_sriov(struct pci_dev *dev); @@ -2147,6 +2147,12 @@ static inline int pci_iov_virtfn_devfn(struct pci_dev *dev, int id) { return -ENOSYS; } + +static inline int pci_iov_vf_id(struct pci_dev *dev) +{ + return -ENOSYS; +} + static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) { return -ENODEV; } diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 3751b672451d9139dca2785dc6f406d4d5686818..ef53829c7a7aa91c34e818cb0b0de700028b7c16 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -120,6 +120,12 @@ struct vdpa_map_file { u64 offset; }; +enum vdpa_reset_state { + VDPA_DEV_RESET_VIRTIO = 0, + VDPA_DEV_RESET_OPEN = 1, + VDPA_DEV_RESET_CLOSE = 2, +}; + /** * struct vdpa_config_ops - operations for configuring a vDPA device. * Note: vDPA device drivers are required to implement all of the @@ -185,6 +191,9 @@ struct vdpa_map_file { * @vdev: vdpa device * Returns the virtio features support by the * device + * @get_backend_features: Get parent-specific backend features (optional) + * Returns the vdpa features supported by the + * device. * @set_driver_features: Set virtio features supported by the driver * @vdev: vdpa device * @features: feature support by the driver @@ -215,8 +224,15 @@ struct vdpa_map_file { * @status: virtio device status * @reset: Reset device * @vdev: vdpa device + * @state: state for reset + * VDPA_DEV_RESET_VIRTIO for virtio reset + * VDPA_DEV_RESET_OPEN for vhost-vdpa device open + * VDPA_DEV_RESET_CLOSE for vhost-vdpa device close + * Returns integer: success (0) or error (< 0) + * @suspend: Suspend the device (optional) + * @vdev: vdpa device * Returns integer: success (0) or error (< 0) - * @suspend: Suspend or resume the device (optional) + * @resume: Resume the device (optional) * @vdev: vdpa device * Returns integer: success (0) or error (< 0) * @get_config_size: Get the size of the configuration space includes @@ -279,6 +295,39 @@ struct vdpa_map_file { * @iova: iova to be unmapped * @size: size of the area * Returns integer: success (0) or error (< 0) + * @bind_mm: Bind the device to a specific address space + * so the vDPA framework can use VA when this + * callback is implemented. (optional) + * @vdev: vdpa device + * @mm: address space to bind + * @unbind_mm: Unbind the device from the address space + * bound using the bind_mm callback. (optional) + * @vdev: vdpa device + * @set_log_base Set base address for logging. (optional) + * @vdev: vdpa device + * @base: base address + * @set_log_size Set buffer size for logging. (optional) + * @vdev: vdpa device + * @size: logging buffer size + * @log_sync Synchronize logging buffer from kernel space to + * user space. (optional) + * @vdev: vdpa device + * @get_dev_buffer_size Get device state buffer size. (optional) + * @vdev: vdpa device + * Return device status buffer size of vdpa device. + * @get_dev_buffer Get device state buffer. (optional) + * @vdev: vdpa device + * @offset: offset of dest for saving device state. + * @dest: userspace address for saving device state. + * @len: device state buffer length. + * @set_dev_buffer Set device state buffer. (opetional) + * @vdev: vdpa device + * @offset: offset of src addr of device state. + * @src: userspace addr of device state + * @len: device state buffer length. + * @set_mig_state Set device migration status. (optional) + * @vdev: vdpa device + * @status: migration status * @free: Free resources that belongs to vDPA (optional) * @vdev: vdpa device */ @@ -309,6 +358,7 @@ struct vdpa_config_ops { u32 (*get_vq_align)(struct vdpa_device *vdev); u32 (*get_vq_group)(struct vdpa_device *vdev, u16 idx); u64 (*get_device_features)(struct vdpa_device *vdev); + u64 (*get_backend_features)(const struct vdpa_device *vdev); int (*set_driver_features)(struct vdpa_device *vdev, u64 features); u64 (*get_driver_features)(struct vdpa_device *vdev); void (*set_config_cb)(struct vdpa_device *vdev, @@ -319,8 +369,9 @@ struct vdpa_config_ops { u32 (*get_vendor_id)(struct vdpa_device *vdev); u8 (*get_status)(struct vdpa_device *vdev); void (*set_status)(struct vdpa_device *vdev, u8 status); - int (*reset)(struct vdpa_device *vdev); + int (*reset)(struct vdpa_device *vdev, int state); int (*suspend)(struct vdpa_device *vdev); + int (*resume)(struct vdpa_device *vdev); size_t (*get_config_size)(struct vdpa_device *vdev); void (*get_config)(struct vdpa_device *vdev, unsigned int offset, void *buf, unsigned int len); @@ -338,6 +389,23 @@ struct vdpa_config_ops { u64 iova, u64 size); int (*set_group_asid)(struct vdpa_device *vdev, unsigned int group, unsigned int asid); + int (*bind_mm)(struct vdpa_device *vdev, struct mm_struct *mm); + void (*unbind_mm)(struct vdpa_device *vdev); + + /* Log ops */ + int (*set_log_base)(struct vdpa_device *vdev, uint64_t base); + int (*set_log_size)(struct vdpa_device *vdev, uint64_t size); + int (*log_sync)(struct vdpa_device *vdev); + + /* device state ops */ + uint32_t (*get_dev_buffer_size)(struct vdpa_device *vdpa); + int (*get_dev_buffer)(struct vdpa_device *vdev, unsigned int offset, + void __user *dest, unsigned int len); + int (*set_dev_buffer)(struct vdpa_device *vdev, unsigned int offset, + const void __user *src, unsigned int len); + + /* device mig state ops */ + int (*set_mig_state)(struct vdpa_device *v, u8 state); /* Free device resources */ void (*free)(struct vdpa_device *vdev); @@ -424,14 +492,14 @@ static inline struct device *vdpa_get_dma_dev(struct vdpa_device *vdev) return vdev->dma_dev; } -static inline int vdpa_reset(struct vdpa_device *vdev) +static inline int vdpa_reset(struct vdpa_device *vdev, int state) { const struct vdpa_config_ops *ops = vdev->config; int ret; down_write(&vdev->cf_lock); vdev->features_valid = false; - ret = ops->reset(vdev); + ret = ops->reset(vdev, state); up_write(&vdev->cf_lock); return ret; } diff --git a/include/linux/vdpa_vmstate.h b/include/linux/vdpa_vmstate.h new file mode 100644 index 0000000000000000000000000000000000000000..0e577a9605f17990871cb795a6fe6611ecb0ded7 --- /dev/null +++ b/include/linux/vdpa_vmstate.h @@ -0,0 +1,182 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * Description: vdpa vmstate header file + * Author: jiangdongxu + * Create: 2023-12-3 + * Note: + * History: 2023-12-3: Create file + */ + +#ifndef VDPA_VMSTATE_H +#define VDPA_VMSTATE_H + +#include +#include + + +#define VIRTIO_MIG_STATE_TYPE_DEVICE 0 +#define VIRTIO_MIG_STATE_TYPE_VQ 1 +#define VIRTIO_MIG_STATE_TYPE_CONFIG 2 +#define VIRTIO_MIG_STATE_TYPE_FEATURE 3 +#define VIRTIO_MIG_STATE_TYPE_PLATFORM 4 +#define VIRTIO_MIG_STATE_TYPE_VENDOR 255 + +#define VIRTIO_MIG_DEVICE_T_COMMON 0 +#define VIRTIO_MIG_DEVICE_T_NET 1 +#define VIRTIO_MIG_DEVICE_T_BLK 2 +#define VIRTIO_MIG_DEVICE_T_SCSI 8 + +#define VIRTIO_MIG_CONFIG_T_NET 1 +#define VIRTIO_MIG_CONFIG_T_BLK 2 +#define VIRTIO_MIG_CONFIG_T_SCSI 8 + +#define MAC_LEN 6 + +struct virtio_mig_state_header { + le32 type; + le32 len; +}; + +struct virtio_mig_dev_common_data { + le32 vendor_id; + le32 device_id; + le32 device_features_l; + le32 device_features_h; + le32 driver_features_l; + le32 driver_features_h; + le32 status; + le32 generation; + le32 msix_en; +}; + +struct virtio_mig_dev_common_state { + struct virtio_mig_state_header hdr; + struct virtio_mig_dev_common_data data; +}; + +struct virtio_mig_vq_split_state { + le16 avail_index; + le16 used_index; +}; +struct virtio_mig_vq_packed_state { + le16 avail_wrapped : 1; + le16 avail_index : 15; + le16 used_wrapped : 1; + le16 used_index : 15; +}; + +struct virtio_mig_per_vq_data { + le32 qsize; + + u8 qenabled; + le16 msix_vector; + + le32 desc_l; + le32 desc_h; + le32 avail_l; + le32 avail_h; + le32 used_l; + le32 used_h; + union { + struct virtio_mig_vq_split_state split; + struct virtio_mig_vq_packed_state packed; + } version; +}; + +/* vq state */ +struct virtio_mig_vq_state { + struct virtio_mig_state_header hdr; + + le16 msix_config; + le16 valid_queues; + + le16 num_queues; + + struct virtio_mig_per_vq_data vq_state[]; +}; + +/* config space */ +struct virtio_mig_config_state { + struct virtio_mig_state_header hdr; + union { + struct virtio_net_config net; + struct virtio_blk_config blk; + struct virtio_scsi_config scsi; + } dev; +}; + +struct virtio_mig_cfg_blk_features { + +}; + +struct virtio_mig_cfg_scsi_features { + +}; + +struct virtio_mig_cfg_net_ctrl_guest_offloads { + struct virtio_mig_state_header hdr; + le64 offloads; + le64 reserved; +}; + +struct virtio_mig_cfg_net_ctrl_mq_vq_pairs { + struct virtio_mig_state_header hdr; + le16 cur_virtqueue_pairs; +}; + +struct virtio_mig_cfg_net_ctrl_mac_table { + struct virtio_mig_state_header hdr; + le16 num_unicast; + /* TODO: need to be implemented later */ + // u8 unicast_macs[][6]; + le16 num_multicast; + /* TODO: need to be implemented later */ + // u8 multicast_macs[][6]; +}; + +struct virtio_mig_cfg_net_ctrl_vlan { + struct virtio_mig_state_header hdr; + le32 vlans[128]; +}; + +struct virtio_mig_cfg_net_data { + le32 nfeatures; + struct virtio_mig_cfg_net_ctrl_guest_offloads offloads; + struct virtio_mig_cfg_net_ctrl_mq_vq_pairs mq_pairs; + struct virtio_mig_cfg_net_ctrl_mac_table mac_table; + struct virtio_mig_cfg_net_ctrl_vlan vlan_table; +}; + +struct virtio_mig_cfg_net_features { + struct virtio_mig_state_header hdr; + struct virtio_mig_cfg_net_data data; +}; + +/* feature */ +struct virtio_mig_feat_state { + union { + struct virtio_mig_cfg_net_features net; + struct virtio_mig_cfg_blk_features blk; + struct virtio_mig_cfg_scsi_features scsi; + }; +}; + +struct vdpa_mig_state { + struct virtio_mig_dev_common_state dev_state; + struct virtio_mig_config_state cfg_state; + struct virtio_mig_feat_state feat_state; + struct virtio_mig_vq_state vq_state; +}; + +#endif /* VDPA_VMSTATE_H */ diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h index f9f115a7c75b8a3060f0678599e7662d0015b878..e916ac7a10ce66755ca7b182250c87649f5eaa96 100644 --- a/include/uapi/linux/vhost.h +++ b/include/uapi/linux/vhost.h @@ -43,6 +43,10 @@ * The bit is set using an atomic 32 bit operation. */ /* Set base address for logging. */ #define VHOST_SET_LOG_BASE _IOW(VHOST_VIRTIO, 0x04, __u64) +/* Set buffer size for logging */ +#define VHOST_SET_LOG_SIZE _IOW(VHOST_VIRTIO, 0x05, __u64) +/* Synchronize logging buffer from kernel space to user space */ +#define VHOST_LOG_SYNC _IO(VHOST_VIRTIO, 0x06) /* Specify an eventfd file descriptor to signal on log write. */ #define VHOST_SET_LOG_FD _IOW(VHOST_VIRTIO, 0x07, int) @@ -180,4 +184,20 @@ */ #define VHOST_VDPA_SUSPEND _IO(VHOST_VIRTIO, 0x7D) +/* Resume a device so it can resume processing virtqueue requests + * + * After the return of this ioctl the device will have restored all the + * necessary states and it is fully operational to continue processing the + * virtqueue descriptors. + */ +#define VHOST_VDPA_RESUME _IO(VHOST_VIRTIO, 0x7E) + +/* set and get device buffer */ +#define VHOST_GET_DEV_BUFFER _IOR(VHOST_VIRTIO, 0xb0, struct vhost_vdpa_config) +#define VHOST_SET_DEV_BUFFER _IOW(VHOST_VIRTIO, 0xb1, struct vhost_vdpa_config) +#define VHOST_GET_DEV_BUFFER_SIZE _IOR(VHOST_VIRTIO, 0xb3, __u32) + +/* set device migtration state */ +#define VHOST_VDPA_SET_MIG_STATE _IOW(VHOST_VIRTIO, 0xb2, __u8) + #endif diff --git a/include/uapi/linux/vhost_types.h b/include/uapi/linux/vhost_types.h index 1bdd6e363f4c9f1557608bf2218363e7024b1489..95cfaf6678e6c14ba7d825bb946b58a9a778bbe1 100644 --- a/include/uapi/linux/vhost_types.h +++ b/include/uapi/linux/vhost_types.h @@ -147,6 +147,22 @@ struct vhost_vdpa_iova_range { __u64 last; }; +/* vhost vdpa device migration statue */ +enum { + VHOST_VDPA_DEVICE_START, + VHOST_VDPA_DEVICE_STOP, + VHOST_VDPA_DEVICE_PRE_START, + VHOST_VDPA_DEVICE_PRE_STOP, + VHOST_VDPA_DEVICE_CANCEL, + VHOST_VDPA_DEVICE_POST_START, + VHOST_VDPA_DEVICE_START_ASYNC, + VHOST_VDPA_DEVICE_STOP_ASYNC, + VHOST_VDPA_DEVICE_PRE_START_ASYNC, + VHOST_VDPA_DEVICE_QUERY_OP_STATE, + VHOST_VDPA_DEVICE_MSIX_MASK, + VHOST_VDPA_DEVICE_MSIX_UNMASK, +}; + /* Feature bits */ /* Log all write descriptors. Can be changed while device is active. */ #define VHOST_F_LOG_ALL 26 @@ -163,5 +179,10 @@ struct vhost_vdpa_iova_range { #define VHOST_BACKEND_F_IOTLB_ASID 0x3 /* Device can be suspended */ #define VHOST_BACKEND_F_SUSPEND 0x4 +/* Device can be resumed */ +#define VHOST_BACKEND_F_RESUME 0x5 + +/* Device can use bytemap to deal log */ +#define VHOST_BACKEND_F_BYTEMAPLOG 0x3f #endif diff --git a/include/uapi/linux/virtio_ids.h b/include/uapi/linux/virtio_ids.h index b052355ac7a324e173f4ea44c48d8b757fa687cf..60555f7d169cf80df0b8379d3ab5f363bc7274f9 100644 --- a/include/uapi/linux/virtio_ids.h +++ b/include/uapi/linux/virtio_ids.h @@ -49,4 +49,16 @@ #define VIRTIO_ID_PMEM 27 /* virtio pmem */ #define VIRTIO_ID_MAC80211_HWSIM 29 /* virtio mac80211-hwsim */ +/* + * Virtio Transitional IDs + */ + +#define VIRTIO_TRANS_ID_NET 0x1000 /* transitional virtio net */ +#define VIRTIO_TRANS_ID_BLOCK 0x1001 /* transitional virtio block */ +#define VIRTIO_TRANS_ID_BALLOON 0x1002 /* transitional virtio balloon */ +#define VIRTIO_TRANS_ID_CONSOLE 0x1003 /* transitional virtio console */ +#define VIRTIO_TRANS_ID_SCSI 0x1004 /* transitional virtio SCSI */ +#define VIRTIO_TRANS_ID_RNG 0x1005 /* transitional virtio rng */ +#define VIRTIO_TRANS_ID_9P 0x1009 /* transitional virtio 9p console */ + #endif /* _LINUX_VIRTIO_IDS_H */ diff --git a/tools/include/uapi/linux/vhost.h b/tools/include/uapi/linux/vhost.h index f9f115a7c75b8a3060f0678599e7662d0015b878..92e1b700b51cbdf66aed6b3d17bc32688cd4706b 100644 --- a/tools/include/uapi/linux/vhost.h +++ b/tools/include/uapi/linux/vhost.h @@ -180,4 +180,12 @@ */ #define VHOST_VDPA_SUSPEND _IO(VHOST_VIRTIO, 0x7D) +/* Resume a device so it can resume processing virtqueue requests + * + * After the return of this ioctl the device will have restored all the + * necessary states and it is fully operational to continue processing the + * virtqueue descriptors. + */ +#define VHOST_VDPA_RESUME _IO(VHOST_VIRTIO, 0x7E) + #endif