spapr_pci_vfio: Add spapr-pci-vfio-host-bridge to support vfio
The patch adds a spapr-pci-vfio-host-bridge device type
which is a PCI Host Bridge with VFIO support. The new device
inherits from the spapr-pci-host-bridge device and adds an "iommu"
property which is an IOMMU id. This ID represents a minimal entity
for which IOMMU isolation can be guaranteed. In SPAPR architecture IOMMU
group is called a Partitionable Endpoint (PE).
Current implementation supports one IOMMU id per QEMU VFIO PHB. Since
SPAPR allows multiple PHB for no extra cost, this does not seem to
be a problem. This limitation may change in the future though.
Example of use:
Configure and Add 3 functions of a multifunctional device to QEMU:
(the NEC PCI USB card is used as an example here):
-device spapr-pci-vfio-host-bridge,id=USB,iommu=4,index=7 \
-device vfio-pci,host=4:0:1.0,addr=1.0,bus=USB,multifunction=true
-device vfio-pci,host=4:0:1.1,addr=1.1,bus=USB
-device vfio-pci,host=4:0:1.2,addr=1.2,bus=USB
where:
* index=7 is a QEMU PHB index (used as source for MMIO/MSI/IO windows
offset);
* iommu=4 is an IOMMU id which can be found in sysfs:
[aik@vpl2 ~]$ cd /sys/bus/pci/devices/0004:00:00.0/
[aik@vpl2 0004:00:00.0]$ ls -l iommu_group
lrwxrwxrwx 1 root root 0 Jun 5 12:49 iommu_group -> ../../../kernel/iommu_groups/4
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: Alexander Graf <agraf@suse.de>
2014-06-10 07:39:23 +02:00
|
|
|
/*
|
|
|
|
* QEMU sPAPR PCI host for VFIO
|
|
|
|
*
|
|
|
|
* Copyright (c) 2011-2014 Alexey Kardashevskiy, IBM Corporation.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation; either version 2 of the License,
|
|
|
|
* or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program; if not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
2016-01-26 19:16:58 +01:00
|
|
|
#include "qemu/osdep.h"
|
2016-06-22 19:11:19 +02:00
|
|
|
#include <linux/vfio.h>
|
2016-01-19 21:51:44 +01:00
|
|
|
#include "qemu-common.h"
|
|
|
|
#include "cpu.h"
|
spapr_pci_vfio: Add spapr-pci-vfio-host-bridge to support vfio
The patch adds a spapr-pci-vfio-host-bridge device type
which is a PCI Host Bridge with VFIO support. The new device
inherits from the spapr-pci-host-bridge device and adds an "iommu"
property which is an IOMMU id. This ID represents a minimal entity
for which IOMMU isolation can be guaranteed. In SPAPR architecture IOMMU
group is called a Partitionable Endpoint (PE).
Current implementation supports one IOMMU id per QEMU VFIO PHB. Since
SPAPR allows multiple PHB for no extra cost, this does not seem to
be a problem. This limitation may change in the future though.
Example of use:
Configure and Add 3 functions of a multifunctional device to QEMU:
(the NEC PCI USB card is used as an example here):
-device spapr-pci-vfio-host-bridge,id=USB,iommu=4,index=7 \
-device vfio-pci,host=4:0:1.0,addr=1.0,bus=USB,multifunction=true
-device vfio-pci,host=4:0:1.1,addr=1.1,bus=USB
-device vfio-pci,host=4:0:1.2,addr=1.2,bus=USB
where:
* index=7 is a QEMU PHB index (used as source for MMIO/MSI/IO windows
offset);
* iommu=4 is an IOMMU id which can be found in sysfs:
[aik@vpl2 ~]$ cd /sys/bus/pci/devices/0004:00:00.0/
[aik@vpl2 0004:00:00.0]$ ls -l iommu_group
lrwxrwxrwx 1 root root 0 Jun 5 12:49 iommu_group -> ../../../kernel/iommu_groups/4
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: Alexander Graf <agraf@suse.de>
2014-06-10 07:39:23 +02:00
|
|
|
#include "hw/ppc/spapr.h"
|
|
|
|
#include "hw/pci-host/spapr.h"
|
2015-07-02 08:23:28 +02:00
|
|
|
#include "hw/pci/msix.h"
|
2014-12-19 23:24:06 +01:00
|
|
|
#include "hw/vfio/vfio.h"
|
2016-02-29 07:19:50 +01:00
|
|
|
#include "qemu/error-report.h"
|
2016-06-14 19:23:03 +02:00
|
|
|
#include "sysemu/qtest.h"
|
spapr_pci_vfio: Add spapr-pci-vfio-host-bridge to support vfio
The patch adds a spapr-pci-vfio-host-bridge device type
which is a PCI Host Bridge with VFIO support. The new device
inherits from the spapr-pci-host-bridge device and adds an "iommu"
property which is an IOMMU id. This ID represents a minimal entity
for which IOMMU isolation can be guaranteed. In SPAPR architecture IOMMU
group is called a Partitionable Endpoint (PE).
Current implementation supports one IOMMU id per QEMU VFIO PHB. Since
SPAPR allows multiple PHB for no extra cost, this does not seem to
be a problem. This limitation may change in the future though.
Example of use:
Configure and Add 3 functions of a multifunctional device to QEMU:
(the NEC PCI USB card is used as an example here):
-device spapr-pci-vfio-host-bridge,id=USB,iommu=4,index=7 \
-device vfio-pci,host=4:0:1.0,addr=1.0,bus=USB,multifunction=true
-device vfio-pci,host=4:0:1.1,addr=1.1,bus=USB
-device vfio-pci,host=4:0:1.2,addr=1.2,bus=USB
where:
* index=7 is a QEMU PHB index (used as source for MMIO/MSI/IO windows
offset);
* iommu=4 is an IOMMU id which can be found in sysfs:
[aik@vpl2 ~]$ cd /sys/bus/pci/devices/0004:00:00.0/
[aik@vpl2 0004:00:00.0]$ ls -l iommu_group
lrwxrwxrwx 1 root root 0 Jun 5 12:49 iommu_group -> ../../../kernel/iommu_groups/4
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: Alexander Graf <agraf@suse.de>
2014-06-10 07:39:23 +02:00
|
|
|
|
2016-02-29 07:19:42 +01:00
|
|
|
bool spapr_phb_eeh_available(sPAPRPHBState *sphb)
|
|
|
|
{
|
|
|
|
return vfio_eeh_as_ok(&sphb->iommu_as);
|
|
|
|
}
|
|
|
|
|
2016-02-29 04:00:34 +01:00
|
|
|
static void spapr_phb_vfio_eeh_reenable(sPAPRPHBState *sphb)
|
2015-07-02 08:23:27 +02:00
|
|
|
{
|
2016-02-29 04:00:34 +01:00
|
|
|
vfio_eeh_as_op(&sphb->iommu_as, VFIO_EEH_PE_ENABLE);
|
2015-07-02 08:23:27 +02:00
|
|
|
}
|
|
|
|
|
2016-02-29 07:45:05 +01:00
|
|
|
void spapr_phb_vfio_reset(DeviceState *qdev)
|
spapr_pci_vfio: Add spapr-pci-vfio-host-bridge to support vfio
The patch adds a spapr-pci-vfio-host-bridge device type
which is a PCI Host Bridge with VFIO support. The new device
inherits from the spapr-pci-host-bridge device and adds an "iommu"
property which is an IOMMU id. This ID represents a minimal entity
for which IOMMU isolation can be guaranteed. In SPAPR architecture IOMMU
group is called a Partitionable Endpoint (PE).
Current implementation supports one IOMMU id per QEMU VFIO PHB. Since
SPAPR allows multiple PHB for no extra cost, this does not seem to
be a problem. This limitation may change in the future though.
Example of use:
Configure and Add 3 functions of a multifunctional device to QEMU:
(the NEC PCI USB card is used as an example here):
-device spapr-pci-vfio-host-bridge,id=USB,iommu=4,index=7 \
-device vfio-pci,host=4:0:1.0,addr=1.0,bus=USB,multifunction=true
-device vfio-pci,host=4:0:1.1,addr=1.1,bus=USB
-device vfio-pci,host=4:0:1.2,addr=1.2,bus=USB
where:
* index=7 is a QEMU PHB index (used as source for MMIO/MSI/IO windows
offset);
* iommu=4 is an IOMMU id which can be found in sysfs:
[aik@vpl2 ~]$ cd /sys/bus/pci/devices/0004:00:00.0/
[aik@vpl2 0004:00:00.0]$ ls -l iommu_group
lrwxrwxrwx 1 root root 0 Jun 5 12:49 iommu_group -> ../../../kernel/iommu_groups/4
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: Alexander Graf <agraf@suse.de>
2014-06-10 07:39:23 +02:00
|
|
|
{
|
2015-07-02 08:23:27 +02:00
|
|
|
/*
|
|
|
|
* The PE might be in frozen state. To reenable the EEH
|
|
|
|
* functionality on it will clean the frozen state, which
|
|
|
|
* ensures that the contained PCI devices will work properly
|
|
|
|
* after reboot.
|
|
|
|
*/
|
2016-02-29 04:00:34 +01:00
|
|
|
spapr_phb_vfio_eeh_reenable(SPAPR_PCI_HOST_BRIDGE(qdev));
|
spapr_pci_vfio: Add spapr-pci-vfio-host-bridge to support vfio
The patch adds a spapr-pci-vfio-host-bridge device type
which is a PCI Host Bridge with VFIO support. The new device
inherits from the spapr-pci-host-bridge device and adds an "iommu"
property which is an IOMMU id. This ID represents a minimal entity
for which IOMMU isolation can be guaranteed. In SPAPR architecture IOMMU
group is called a Partitionable Endpoint (PE).
Current implementation supports one IOMMU id per QEMU VFIO PHB. Since
SPAPR allows multiple PHB for no extra cost, this does not seem to
be a problem. This limitation may change in the future though.
Example of use:
Configure and Add 3 functions of a multifunctional device to QEMU:
(the NEC PCI USB card is used as an example here):
-device spapr-pci-vfio-host-bridge,id=USB,iommu=4,index=7 \
-device vfio-pci,host=4:0:1.0,addr=1.0,bus=USB,multifunction=true
-device vfio-pci,host=4:0:1.1,addr=1.1,bus=USB
-device vfio-pci,host=4:0:1.2,addr=1.2,bus=USB
where:
* index=7 is a QEMU PHB index (used as source for MMIO/MSI/IO windows
offset);
* iommu=4 is an IOMMU id which can be found in sysfs:
[aik@vpl2 ~]$ cd /sys/bus/pci/devices/0004:00:00.0/
[aik@vpl2 0004:00:00.0]$ ls -l iommu_group
lrwxrwxrwx 1 root root 0 Jun 5 12:49 iommu_group -> ../../../kernel/iommu_groups/4
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: Alexander Graf <agraf@suse.de>
2014-06-10 07:39:23 +02:00
|
|
|
}
|
|
|
|
|
2016-02-29 07:45:05 +01:00
|
|
|
int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
|
|
|
|
unsigned int addr, int option)
|
2015-02-20 05:58:53 +01:00
|
|
|
{
|
2016-02-29 04:00:34 +01:00
|
|
|
uint32_t op;
|
2015-02-20 05:58:53 +01:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
switch (option) {
|
|
|
|
case RTAS_EEH_DISABLE:
|
2016-02-29 04:00:34 +01:00
|
|
|
op = VFIO_EEH_PE_DISABLE;
|
2015-02-20 05:58:53 +01:00
|
|
|
break;
|
|
|
|
case RTAS_EEH_ENABLE: {
|
|
|
|
PCIHostState *phb;
|
|
|
|
PCIDevice *pdev;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The EEH functionality is enabled on basis of PCI device,
|
|
|
|
* instead of PE. We need check the validity of the PCI
|
|
|
|
* device address.
|
|
|
|
*/
|
|
|
|
phb = PCI_HOST_BRIDGE(sphb);
|
|
|
|
pdev = pci_find_device(phb->bus,
|
|
|
|
(addr >> 16) & 0xFF, (addr >> 8) & 0xFF);
|
2015-09-18 09:30:44 +02:00
|
|
|
if (!pdev || !object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
|
2015-02-20 05:58:53 +01:00
|
|
|
return RTAS_OUT_PARAM_ERROR;
|
|
|
|
}
|
|
|
|
|
2016-02-29 04:00:34 +01:00
|
|
|
op = VFIO_EEH_PE_ENABLE;
|
2015-02-20 05:58:53 +01:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
case RTAS_EEH_THAW_IO:
|
2016-02-29 04:00:34 +01:00
|
|
|
op = VFIO_EEH_PE_UNFREEZE_IO;
|
2015-02-20 05:58:53 +01:00
|
|
|
break;
|
|
|
|
case RTAS_EEH_THAW_DMA:
|
2016-02-29 04:00:34 +01:00
|
|
|
op = VFIO_EEH_PE_UNFREEZE_DMA;
|
2015-02-20 05:58:53 +01:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return RTAS_OUT_PARAM_ERROR;
|
|
|
|
}
|
|
|
|
|
2016-02-29 04:00:34 +01:00
|
|
|
ret = vfio_eeh_as_op(&sphb->iommu_as, op);
|
2015-02-20 05:58:53 +01:00
|
|
|
if (ret < 0) {
|
|
|
|
return RTAS_OUT_HW_ERROR;
|
|
|
|
}
|
|
|
|
|
|
|
|
return RTAS_OUT_SUCCESS;
|
|
|
|
}
|
|
|
|
|
2016-02-29 07:45:05 +01:00
|
|
|
int spapr_phb_vfio_eeh_get_state(sPAPRPHBState *sphb, int *state)
|
2015-02-20 05:58:53 +01:00
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
2016-02-29 04:00:34 +01:00
|
|
|
ret = vfio_eeh_as_op(&sphb->iommu_as, VFIO_EEH_PE_GET_STATE);
|
2015-02-20 05:58:53 +01:00
|
|
|
if (ret < 0) {
|
|
|
|
return RTAS_OUT_PARAM_ERROR;
|
|
|
|
}
|
|
|
|
|
|
|
|
*state = ret;
|
|
|
|
return RTAS_OUT_SUCCESS;
|
|
|
|
}
|
|
|
|
|
2015-07-02 08:23:28 +02:00
|
|
|
static void spapr_phb_vfio_eeh_clear_dev_msix(PCIBus *bus,
|
|
|
|
PCIDevice *pdev,
|
|
|
|
void *opaque)
|
|
|
|
{
|
|
|
|
/* Check if the device is VFIO PCI device */
|
|
|
|
if (!object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The MSIx table will be cleaned out by reset. We need
|
|
|
|
* disable it so that it can be reenabled properly. Also,
|
|
|
|
* the cached MSIx table should be cleared as it's not
|
|
|
|
* reflecting the contents in hardware.
|
|
|
|
*/
|
|
|
|
if (msix_enabled(pdev)) {
|
|
|
|
uint16_t flags;
|
|
|
|
|
|
|
|
flags = pci_host_config_read_common(pdev,
|
|
|
|
pdev->msix_cap + PCI_MSIX_FLAGS,
|
|
|
|
pci_config_size(pdev), 2);
|
|
|
|
flags &= ~PCI_MSIX_FLAGS_ENABLE;
|
|
|
|
pci_host_config_write_common(pdev,
|
|
|
|
pdev->msix_cap + PCI_MSIX_FLAGS,
|
|
|
|
pci_config_size(pdev), flags, 2);
|
|
|
|
}
|
|
|
|
|
|
|
|
msix_reset(pdev);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void spapr_phb_vfio_eeh_clear_bus_msix(PCIBus *bus, void *opaque)
|
|
|
|
{
|
|
|
|
pci_for_each_device(bus, pci_bus_num(bus),
|
|
|
|
spapr_phb_vfio_eeh_clear_dev_msix, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void spapr_phb_vfio_eeh_pre_reset(sPAPRPHBState *sphb)
|
|
|
|
{
|
|
|
|
PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
|
|
|
|
|
|
|
|
pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_clear_bus_msix, NULL);
|
|
|
|
}
|
|
|
|
|
2016-02-29 07:45:05 +01:00
|
|
|
int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, int option)
|
2015-02-20 05:58:53 +01:00
|
|
|
{
|
2016-02-29 04:00:34 +01:00
|
|
|
uint32_t op;
|
2015-02-20 05:58:53 +01:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
switch (option) {
|
|
|
|
case RTAS_SLOT_RESET_DEACTIVATE:
|
2016-02-29 04:00:34 +01:00
|
|
|
op = VFIO_EEH_PE_RESET_DEACTIVATE;
|
2015-02-20 05:58:53 +01:00
|
|
|
break;
|
|
|
|
case RTAS_SLOT_RESET_HOT:
|
2015-07-02 08:23:28 +02:00
|
|
|
spapr_phb_vfio_eeh_pre_reset(sphb);
|
2016-02-29 04:00:34 +01:00
|
|
|
op = VFIO_EEH_PE_RESET_HOT;
|
2015-02-20 05:58:53 +01:00
|
|
|
break;
|
|
|
|
case RTAS_SLOT_RESET_FUNDAMENTAL:
|
2015-07-02 08:23:28 +02:00
|
|
|
spapr_phb_vfio_eeh_pre_reset(sphb);
|
2016-02-29 04:00:34 +01:00
|
|
|
op = VFIO_EEH_PE_RESET_FUNDAMENTAL;
|
2015-02-20 05:58:53 +01:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return RTAS_OUT_PARAM_ERROR;
|
|
|
|
}
|
|
|
|
|
2016-02-29 04:00:34 +01:00
|
|
|
ret = vfio_eeh_as_op(&sphb->iommu_as, op);
|
2015-02-20 05:58:53 +01:00
|
|
|
if (ret < 0) {
|
|
|
|
return RTAS_OUT_HW_ERROR;
|
|
|
|
}
|
|
|
|
|
|
|
|
return RTAS_OUT_SUCCESS;
|
|
|
|
}
|
|
|
|
|
2016-02-29 07:45:05 +01:00
|
|
|
int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb)
|
2015-02-20 05:58:53 +01:00
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
2016-02-29 04:00:34 +01:00
|
|
|
ret = vfio_eeh_as_op(&sphb->iommu_as, VFIO_EEH_PE_CONFIGURE);
|
2015-02-20 05:58:53 +01:00
|
|
|
if (ret < 0) {
|
|
|
|
return RTAS_OUT_PARAM_ERROR;
|
|
|
|
}
|
|
|
|
|
|
|
|
return RTAS_OUT_SUCCESS;
|
|
|
|
}
|