vfio-user: IOMMU support for remote device

Assign separate address space for each device in the remote processes.

Signed-off-by: Elena Ufimtseva <elena.ufimtseva@oracle.com>
Signed-off-by: John G Johnson <john.g.johnson@oracle.com>
Signed-off-by: Jagannathan Raman <jag.raman@oracle.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
Message-id: afe0b0a97582cdad42b5b25636a29c523265a10a.1655151679.git.jag.raman@oracle.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
This commit is contained in:
Jagannathan Raman 2022-06-13 16:26:30 -04:00 committed by Stefan Hajnoczi
parent 90072f29d6
commit 253007d147
5 changed files with 186 additions and 1 deletions

View File

@ -3644,6 +3644,8 @@ F: hw/remote/iohub.c
F: include/hw/remote/iohub.h
F: subprojects/libvfio-user
F: hw/remote/vfio-user-obj.c
F: hw/remote/iommu.c
F: include/hw/remote/iommu.h
EBPF:
M: Jason Wang <jasowang@redhat.com>

131
hw/remote/iommu.c Normal file
View File

@ -0,0 +1,131 @@
/**
* IOMMU for remote device
*
* Copyright © 2022 Oracle and/or its affiliates.
*
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
*
*/
#include "qemu/osdep.h"
#include "hw/remote/iommu.h"
#include "hw/pci/pci_bus.h"
#include "hw/pci/pci.h"
#include "exec/memory.h"
#include "exec/address-spaces.h"
#include "trace.h"
/**
* IOMMU for TYPE_REMOTE_MACHINE - manages DMA address space isolation
* for remote machine. It is used by TYPE_VFIO_USER_SERVER.
*
* - Each TYPE_VFIO_USER_SERVER instance handles one PCIDevice on a PCIBus.
* There is one RemoteIommu per PCIBus, so the RemoteIommu tracks multiple
* PCIDevices by maintaining a ->elem_by_devfn mapping.
*
* - memory_region_init_iommu() is not used because vfio-user MemoryRegions
* will be added to the elem->mr container instead. This is more natural
* than implementing the IOMMUMemoryRegionClass APIs since vfio-user
* provides something that is close to a full-fledged MemoryRegion and
* not like an IOMMU mapping.
*
* - When a device is hot unplugged, the elem->mr reference is dropped so
* all vfio-user MemoryRegions associated with this vfio-user server are
* destroyed.
*/
static AddressSpace *remote_iommu_find_add_as(PCIBus *pci_bus,
void *opaque, int devfn)
{
RemoteIommu *iommu = opaque;
RemoteIommuElem *elem = NULL;
qemu_mutex_lock(&iommu->lock);
elem = g_hash_table_lookup(iommu->elem_by_devfn, INT2VOIDP(devfn));
if (!elem) {
elem = g_malloc0(sizeof(RemoteIommuElem));
g_hash_table_insert(iommu->elem_by_devfn, INT2VOIDP(devfn), elem);
}
if (!elem->mr) {
elem->mr = MEMORY_REGION(object_new(TYPE_MEMORY_REGION));
memory_region_set_size(elem->mr, UINT64_MAX);
address_space_init(&elem->as, elem->mr, NULL);
}
qemu_mutex_unlock(&iommu->lock);
return &elem->as;
}
void remote_iommu_unplug_dev(PCIDevice *pci_dev)
{
AddressSpace *as = pci_device_iommu_address_space(pci_dev);
RemoteIommuElem *elem = NULL;
if (as == &address_space_memory) {
return;
}
elem = container_of(as, RemoteIommuElem, as);
address_space_destroy(&elem->as);
object_unref(elem->mr);
elem->mr = NULL;
}
static void remote_iommu_init(Object *obj)
{
RemoteIommu *iommu = REMOTE_IOMMU(obj);
iommu->elem_by_devfn = g_hash_table_new_full(NULL, NULL, NULL, g_free);
qemu_mutex_init(&iommu->lock);
}
static void remote_iommu_finalize(Object *obj)
{
RemoteIommu *iommu = REMOTE_IOMMU(obj);
qemu_mutex_destroy(&iommu->lock);
g_hash_table_destroy(iommu->elem_by_devfn);
iommu->elem_by_devfn = NULL;
}
void remote_iommu_setup(PCIBus *pci_bus)
{
RemoteIommu *iommu = NULL;
g_assert(pci_bus);
iommu = REMOTE_IOMMU(object_new(TYPE_REMOTE_IOMMU));
pci_setup_iommu(pci_bus, remote_iommu_find_add_as, iommu);
object_property_add_child(OBJECT(pci_bus), "remote-iommu", OBJECT(iommu));
object_unref(OBJECT(iommu));
}
static const TypeInfo remote_iommu_info = {
.name = TYPE_REMOTE_IOMMU,
.parent = TYPE_OBJECT,
.instance_size = sizeof(RemoteIommu),
.instance_init = remote_iommu_init,
.instance_finalize = remote_iommu_finalize,
};
static void remote_iommu_register_types(void)
{
type_register_static(&remote_iommu_info);
}
type_init(remote_iommu_register_types)

View File

@ -20,6 +20,7 @@
#include "qapi/error.h"
#include "hw/pci/pci_host.h"
#include "hw/remote/iohub.h"
#include "hw/remote/iommu.h"
#include "hw/qdev-core.h"
static void remote_machine_init(MachineState *machine)
@ -99,6 +100,16 @@ static void remote_machine_instance_init(Object *obj)
s->auto_shutdown = true;
}
static void remote_machine_dev_unplug_cb(HotplugHandler *hotplug_dev,
DeviceState *dev, Error **errp)
{
qdev_unrealize(dev);
if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
remote_iommu_unplug_dev(PCI_DEVICE(dev));
}
}
static void remote_machine_class_init(ObjectClass *oc, void *data)
{
MachineClass *mc = MACHINE_CLASS(oc);
@ -107,7 +118,7 @@ static void remote_machine_class_init(ObjectClass *oc, void *data)
mc->init = remote_machine_init;
mc->desc = "Experimental remote machine";
hc->unplug = qdev_simple_device_unplug_cb;
hc->unplug = remote_machine_dev_unplug_cb;
object_class_property_add_bool(oc, "vfio-user",
remote_machine_get_vfio_user,

View File

@ -6,6 +6,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c'))
remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c'))
remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c'))
remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c'))
remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iommu.c'))
remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: files('vfio-user-obj.c'))
remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: libvfio_user_dep)

40
include/hw/remote/iommu.h Normal file
View File

@ -0,0 +1,40 @@
/**
* Copyright © 2022 Oracle and/or its affiliates.
*
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
*
*/
#ifndef REMOTE_IOMMU_H
#define REMOTE_IOMMU_H
#include "hw/pci/pci_bus.h"
#include "hw/pci/pci.h"
#ifndef INT2VOIDP
#define INT2VOIDP(i) (void *)(uintptr_t)(i)
#endif
typedef struct RemoteIommuElem {
MemoryRegion *mr;
AddressSpace as;
} RemoteIommuElem;
#define TYPE_REMOTE_IOMMU "x-remote-iommu"
OBJECT_DECLARE_SIMPLE_TYPE(RemoteIommu, REMOTE_IOMMU)
struct RemoteIommu {
Object parent;
GHashTable *elem_by_devfn;
QemuMutex lock;
};
void remote_iommu_setup(PCIBus *pci_bus);
void remote_iommu_unplug_dev(PCIDevice *pci_dev);
#endif