| Message ID | 20201008171558.410886-10-jean-philippe@linaro.org |
|---|---|
| State | New |
| Headers | show |
| Series | virtio-iommu: VFIO integration | expand |
Hi Jean, On 10/8/20 7:15 PM, Jean-Philippe Brucker wrote: > From: Bharat Bhushan <bbhushan2@marvell.com> > > The virtio-iommu device can deal with arbitrary page sizes for virtual > endpoints, but for endpoints assigned with VFIO it must follow the page > granule used by the host IOMMU driver. > > Implement the interface to set the vIOMMU page size mask, called by VFIO > for each endpoint. We assume that all host IOMMU drivers use the same > page granule (the host page granule). Override the page_size_mask field > in the virtio config space. > > Signed-off-by: Bharat Bhushan <bbhushan2@marvell.com> > Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org> > --- > v10: Use global page mask, allowing VFIO to override it until boot. > --- > hw/virtio/virtio-iommu.c | 51 ++++++++++++++++++++++++++++++++++++++++ > 1 file changed, 51 insertions(+) > > diff --git a/hw/virtio/virtio-iommu.c b/hw/virtio/virtio-iommu.c > index 8823bfc804a..dd0b3093d1b 100644 > --- a/hw/virtio/virtio-iommu.c > +++ b/hw/virtio/virtio-iommu.c > @@ -914,6 +914,56 @@ static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr, > return 0; > } > > +static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr, > + uint64_t page_size_mask, > + Error **errp) > +{ > + int new_granule, old_granule; > + IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); > + VirtIOIOMMU *s = sdev->viommu; > + > + if (!page_size_mask) { set errp > + return -1; > + } > + > + new_granule = ctz64(page_size_mask); > + old_granule = ctz64(s->config.page_size_mask); I think this would be interesting to add a trace point > + > + /* > + * Modifying the page size after machine initialization isn't supported. > + * Having a different mask is possible but the guest will use sub-optimal > + * block sizes, so warn about it. > + */ > + if (qdev_hotplug) { > + if (new_granule != old_granule) { > + error_setg(errp, > + "virtio-iommu page mask 0x%"PRIx64 > + " is incompatible with mask 0x%"PRIx64, > + s->config.page_size_mask, page_size_mask); > + return -1; > + } else if (page_size_mask != s->config.page_size_mask) { > + warn_report("virtio-iommu page mask 0x%"PRIx64 > + " does not match 0x%"PRIx64, > + s->config.page_size_mask, page_size_mask); > + } > + return 0; > + } > + > + /* > + * Disallow shrinking the page size. For example if an endpoint only > + * supports 64kB pages, we can't globally enable 4kB pages. But that > + * shouldn't happen, the host is unlikely to setup differing page granules. > + * The other bits are only hints describing optimal block sizes. > + */ > + if (new_granule < old_granule) { > + error_setg(errp, "memory region shrinks the virtio-iommu page granule"); > + return -1; > + } > + > + s->config.page_size_mask = page_size_mask; > + return 0; > +} > + > static void virtio_iommu_device_realize(DeviceState *dev, Error **errp) > { > VirtIODevice *vdev = VIRTIO_DEVICE(dev); > @@ -1146,6 +1196,7 @@ static void virtio_iommu_memory_region_class_init(ObjectClass *klass, > imrc->translate = virtio_iommu_translate; > imrc->replay = virtio_iommu_replay; > imrc->notify_flag_changed = virtio_iommu_notify_flag_changed; > + imrc->iommu_set_page_size_mask = virtio_iommu_set_page_size_mask; > } > > static const TypeInfo virtio_iommu_info = { > Thanks Eric
On Thu, Oct 08, 2020 at 07:15:57PM +0200, Jean-Philippe Brucker wrote: > From: Bharat Bhushan <bbhushan2@marvell.com> > > The virtio-iommu device can deal with arbitrary page sizes for virtual > endpoints, but for endpoints assigned with VFIO it must follow the page > granule used by the host IOMMU driver. > > Implement the interface to set the vIOMMU page size mask, called by VFIO > for each endpoint. We assume that all host IOMMU drivers use the same > page granule (the host page granule). Override the page_size_mask field > in the virtio config space. > > Signed-off-by: Bharat Bhushan <bbhushan2@marvell.com> > Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org> > --- > v10: Use global page mask, allowing VFIO to override it until boot. > --- > hw/virtio/virtio-iommu.c | 51 ++++++++++++++++++++++++++++++++++++++++ > 1 file changed, 51 insertions(+) > > diff --git a/hw/virtio/virtio-iommu.c b/hw/virtio/virtio-iommu.c > index 8823bfc804a..dd0b3093d1b 100644 > --- a/hw/virtio/virtio-iommu.c > +++ b/hw/virtio/virtio-iommu.c > @@ -914,6 +914,56 @@ static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr, > return 0; > } > > +static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr, > + uint64_t page_size_mask, > + Error **errp) > +{ > + int new_granule, old_granule; > + IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); > + VirtIOIOMMU *s = sdev->viommu; > + > + if (!page_size_mask) { > + return -1; > + } > + > + new_granule = ctz64(page_size_mask); > + old_granule = ctz64(s->config.page_size_mask); > + > + /* > + * Modifying the page size after machine initialization isn't supported. > + * Having a different mask is possible but the guest will use sub-optimal > + * block sizes, so warn about it. > + */ > + if (qdev_hotplug) { > + if (new_granule != old_granule) { > + error_setg(errp, > + "virtio-iommu page mask 0x%"PRIx64 > + " is incompatible with mask 0x%"PRIx64, > + s->config.page_size_mask, page_size_mask); > + return -1; > + } else if (page_size_mask != s->config.page_size_mask) { > + warn_report("virtio-iommu page mask 0x%"PRIx64 > + " does not match 0x%"PRIx64, > + s->config.page_size_mask, page_size_mask); > + } > + return 0; > + } > + > + /* > + * Disallow shrinking the page size. For example if an endpoint only > + * supports 64kB pages, we can't globally enable 4kB pages. But that > + * shouldn't happen, the host is unlikely to setup differing page granules. > + * The other bits are only hints describing optimal block sizes. > + */ > + if (new_granule < old_granule) { > + error_setg(errp, "memory region shrinks the virtio-iommu page granule"); > + return -1; > + } My understanding is that shrink is actually allowed, instead we should forbid growing of the mask? For example, initially the old_granule will always points to the guest page size. Then as long as the host page size (which new_granule represents) is smaller than the old_granule, then it seems fine... Or am I wrong? Another thing, IIUC this function will be majorly called in vfio code when the container page mask will be passed into it. If there're multiple vfio containers that support different host IOMMU page sizes, then IIUC the order of the call to virtio_iommu_set_page_size_mask() is undefined. It's probably related to which "-device vfio-pci,..." parameter is earlier. To make this simpler, I'm thinking whether we should just forbid the case where devices have different iommu page sizes. So when assigned devices are used, we make sure all host iommu page sizes are the same, and the value should be smaller than guest page size. Otherwise we'll simply fall back to guest psize. Thanks, > + > + s->config.page_size_mask = page_size_mask; > + return 0; > +} > + > static void virtio_iommu_device_realize(DeviceState *dev, Error **errp) > { > VirtIODevice *vdev = VIRTIO_DEVICE(dev); > @@ -1146,6 +1196,7 @@ static void virtio_iommu_memory_region_class_init(ObjectClass *klass, > imrc->translate = virtio_iommu_translate; > imrc->replay = virtio_iommu_replay; > imrc->notify_flag_changed = virtio_iommu_notify_flag_changed; > + imrc->iommu_set_page_size_mask = virtio_iommu_set_page_size_mask; > } > > static const TypeInfo virtio_iommu_info = { > -- > 2.28.0 > -- Peter Xu
On Fri, Oct 16, 2020 at 03:08:03PM +0200, Auger Eric wrote: > > +static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr, > > + uint64_t page_size_mask, > > + Error **errp) > > +{ > > + int new_granule, old_granule; > > + IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); > > + VirtIOIOMMU *s = sdev->viommu; > > + > > + if (!page_size_mask) { > set errp Woops, fixed > > + return -1; > > + } > > + > > + new_granule = ctz64(page_size_mask); > > + old_granule = ctz64(s->config.page_size_mask); > > I think this would be interesting to add a trace point Agreed Thanks, Jean
diff --git a/hw/virtio/virtio-iommu.c b/hw/virtio/virtio-iommu.c index 8823bfc804a..dd0b3093d1b 100644 --- a/hw/virtio/virtio-iommu.c +++ b/hw/virtio/virtio-iommu.c @@ -914,6 +914,56 @@ static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr, return 0; } +static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr, + uint64_t page_size_mask, + Error **errp) +{ + int new_granule, old_granule; + IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); + VirtIOIOMMU *s = sdev->viommu; + + if (!page_size_mask) { + return -1; + } + + new_granule = ctz64(page_size_mask); + old_granule = ctz64(s->config.page_size_mask); + + /* + * Modifying the page size after machine initialization isn't supported. + * Having a different mask is possible but the guest will use sub-optimal + * block sizes, so warn about it. + */ + if (qdev_hotplug) { + if (new_granule != old_granule) { + error_setg(errp, + "virtio-iommu page mask 0x%"PRIx64 + " is incompatible with mask 0x%"PRIx64, + s->config.page_size_mask, page_size_mask); + return -1; + } else if (page_size_mask != s->config.page_size_mask) { + warn_report("virtio-iommu page mask 0x%"PRIx64 + " does not match 0x%"PRIx64, + s->config.page_size_mask, page_size_mask); + } + return 0; + } + + /* + * Disallow shrinking the page size. For example if an endpoint only + * supports 64kB pages, we can't globally enable 4kB pages. But that + * shouldn't happen, the host is unlikely to setup differing page granules. + * The other bits are only hints describing optimal block sizes. + */ + if (new_granule < old_granule) { + error_setg(errp, "memory region shrinks the virtio-iommu page granule"); + return -1; + } + + s->config.page_size_mask = page_size_mask; + return 0; +} + static void virtio_iommu_device_realize(DeviceState *dev, Error **errp) { VirtIODevice *vdev = VIRTIO_DEVICE(dev); @@ -1146,6 +1196,7 @@ static void virtio_iommu_memory_region_class_init(ObjectClass *klass, imrc->translate = virtio_iommu_translate; imrc->replay = virtio_iommu_replay; imrc->notify_flag_changed = virtio_iommu_notify_flag_changed; + imrc->iommu_set_page_size_mask = virtio_iommu_set_page_size_mask; } static const TypeInfo virtio_iommu_info = {