qemu-e2k/hw/ppc/spapr.c

4667 lines
149 KiB
C
Raw Normal View History

/*
* QEMU PowerPC pSeries Logical Partition (aka sPAPR) hardware System Emulator
*
* Copyright (c) 2004-2007 Fabrice Bellard
* Copyright (c) 2007 Jocelyn Mayer
* Copyright (c) 2010 David Gibson, IBM Corporation.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*
*/
#include "qemu/osdep.h"
2016-03-14 09:01:28 +01:00
#include "qapi/error.h"
#include "qapi/visitor.h"
#include "sysemu/sysemu.h"
#include "sysemu/numa.h"
#include "hw/hw.h"
#include "qemu/log.h"
#include "hw/fw-path-provider.h"
#include "elf.h"
#include "net/net.h"
#include "sysemu/device_tree.h"
#include "sysemu/cpus.h"
#include "sysemu/hw_accel.h"
#include "kvm_ppc.h"
#include "migration/misc.h"
#include "migration/global_state.h"
#include "migration/register.h"
#include "mmu-hash64.h"
#include "mmu-book3s-v3.h"
target/ppc: Update setting of cpu features to account for compat modes The device tree nodes ibm,arch-vec-5-platform-support and ibm,pa-features are used to communicate features of the cpu to the guest operating system. The properties of each of these are determined based on the selected cpu model and the availability of hypervisor features. Currently the compatibility mode of the cpu is not taken into account. The ibm,arch-vec-5-platform-support node is used to communicate the level of support for various ISAv3 processor features to the guest before CAS to inform the guests' request. The available mmu mode should only be hash unless the cpu is a POWER9 which is not in a prePOWER9 compat mode, in which case the available modes depend on the accelerator and the hypervisor capabilities. The ibm,pa-featues node is used to communicate the level of cpu support for various features to the guest os. This should only contain features relevant to the operating mode of the processor, that is the selected cpu model taking into account any compat mode. This means that the compat mode should be taken into account when choosing the properties of ibm,pa-features and they should match the compat mode selected, or the cpu model selected if no compat mode. Update the setting of these cpu features in the device tree as described above to properly take into account any compat mode. We use the ppc_check_compat function which takes into account the current processor model and the cpu compat mode. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-11-17 06:39:00 +01:00
#include "cpu-models.h"
#include "qom/cpu.h"
#include "hw/boards.h"
#include "hw/ppc/ppc.h"
#include "hw/loader.h"
#include "hw/ppc/fdt.h"
#include "hw/ppc/spapr.h"
#include "hw/ppc/spapr_vio.h"
#include "hw/pci-host/spapr.h"
#include "hw/pci/msi.h"
#include "hw/pci/pci.h"
#include "hw/scsi/scsi.h"
#include "hw/virtio/virtio-scsi.h"
#include "hw/virtio/vhost-scsi-common.h"
#include "exec/address-spaces.h"
#include "exec/ram_addr.h"
#include "hw/usb.h"
#include "qemu/config-file.h"
#include "qemu/error-report.h"
spapr: Add ibm, client-architecture-support call The PAPR+ specification defines a ibm,client-architecture-support (CAS) RTAS call which purpose is to provide a negotiation mechanism for the guest and the hypervisor to work out the best compatibility parameters. During the negotiation process, the guest provides an array of various options and capabilities which it supports, the hypervisor adjusts the device tree and (optionally) reboots the guest. At the moment the Linux guest calls CAS method at early boot so SLOF gets called. SLOF allocates a memory buffer for the device tree changes and calls a custom KVMPPC_H_CAS hypercall. QEMU parses the options, composes a diff for the device tree, copies it to the buffer provided by SLOF and returns to SLOF. SLOF updates the device tree and returns control to the guest kernel. Only then the Linux guest parses the device tree so it is possible to avoid unnecessary reboot in most cases. The device tree diff is a header with an update format version (defined as 1 in this patch) followed by a device tree with the properties which require update. If QEMU detects that it has to reboot the guest, it silently does so as the guest expects reboot to happen because this is usual pHyp firmware behavior. This defines custom KVMPPC_H_CAS hypercall. The current SLOF already has support for it. This implements stub which returns very basic tree (root node, no properties) to the guest. As the return buffer does not contain any change, no change in behavior is expected. Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru> Signed-off-by: Alexander Graf <agraf@suse.de>
2014-05-23 04:26:54 +02:00
#include "trace.h"
#include "hw/nmi.h"
#include "hw/intc/intc.h"
#include "qemu/cutils.h"
#include "hw/ppc/spapr_cpu_core.h"
#include "hw/mem/memory-device.h"
#include <libfdt.h>
/* SLOF memory layout:
*
* SLOF raw image loaded at 0, copies its romfs right below the flat
* device-tree, then position SLOF itself 31M below that
*
* So we set FW_OVERHEAD to 40MB which should account for all of that
* and more
*
* We load our kernel at 4M, leaving space for SLOF initial image
*/
#define FDT_MAX_SIZE 0x100000
#define RTAS_MAX_SIZE 0x10000
#define RTAS_MAX_ADDR 0x80000000 /* RTAS must stay below that */
#define FW_MAX_SIZE 0x400000
#define FW_FILE_NAME "slof.bin"
#define FW_OVERHEAD 0x2800000
#define KERNEL_LOAD_ADDR FW_MAX_SIZE
#define MIN_RMA_SLOF 128UL
#define PHANDLE_INTC 0x00001111
/* These two functions implement the VCPU id numbering: one to compute them
* all and one to identify thread 0 of a VCORE. Any change to the first one
* is likely to have an impact on the second one, so let's keep them close.
*/
static int spapr_vcpu_id(sPAPRMachineState *spapr, int cpu_index)
{
assert(spapr->vsmt);
return
(cpu_index / smp_threads) * spapr->vsmt + cpu_index % smp_threads;
}
static bool spapr_is_thread0_in_vcore(sPAPRMachineState *spapr,
PowerPCCPU *cpu)
{
assert(spapr->vsmt);
return spapr_get_vcpu_id(cpu) % spapr->vsmt == 0;
}
static bool pre_2_10_vmstate_dummy_icp_needed(void *opaque)
{
/* Dummy entries correspond to unused ICPState objects in older QEMUs,
* and newer QEMUs don't even have them. In both cases, we don't want
* to send anything on the wire.
*/
return false;
}
static const VMStateDescription pre_2_10_vmstate_dummy_icp = {
.name = "icp/server",
.version_id = 1,
.minimum_version_id = 1,
.needed = pre_2_10_vmstate_dummy_icp_needed,
.fields = (VMStateField[]) {
VMSTATE_UNUSED(4), /* uint32_t xirr */
VMSTATE_UNUSED(1), /* uint8_t pending_priority */
VMSTATE_UNUSED(1), /* uint8_t mfrr */
VMSTATE_END_OF_LIST()
},
};
static void pre_2_10_vmstate_register_dummy_icp(int i)
{
vmstate_register(NULL, i, &pre_2_10_vmstate_dummy_icp,
(void *)(uintptr_t) i);
}
static void pre_2_10_vmstate_unregister_dummy_icp(int i)
{
vmstate_unregister(NULL, &pre_2_10_vmstate_dummy_icp,
(void *)(uintptr_t) i);
}
int spapr_max_server_number(sPAPRMachineState *spapr)
{
assert(spapr->vsmt);
spapr: use spapr->vsmt to compute VCPU ids Since the introduction of VSMT in 2.11, the spacing of VCPU ids between cores is controllable through a machine property instead of being only dictated by the SMT mode of the host: cpu->vcpu_id = (cc->core_id * spapr->vsmt / smp_threads) + i Until recently, the machine code would try to change the SMT mode of the host to be equal to VSMT or exit. This allowed the rest of the code to assume that kvmppc_smt_threads() == spapr->vsmt is always true. Recent commit "8904e5a75005 spapr: Adjust default VSMT value for better migration compatibility" relaxed the rule. If the VSMT mode cannot be set in KVM for some reasons, but the requested CPU topology is compatible with the current SMT mode, then we let the guest run with kvmppc_smt_threads() != spapr->vsmt. This breaks quite a few places in the code, in particular when calculating DRC indexes. This is what happens on a POWER host with subcores-per-core=2 (ie, supports up to SMT4) when passing the following topology: -smp threads=4,maxcpus=16 \ -device host-spapr-cpu-core,core-id=4,id=core1 \ -device host-spapr-cpu-core,core-id=8,id=core2 qemu-system-ppc64: warning: Failed to set KVM's VSMT mode to 8 (errno -22) This is expected since KVM is limited to SMT4, but the guest is started anyway because this topology can run on SMT4 even with a VSMT8 spacing. But when we look at the DT, things get nastier: cpus { ... ibm,drc-indexes = <0x4 0x10000000 0x10000004 0x10000008 0x1000000c>; This means that we have the following association: CPU core device | DRC | VCPU id -----------------+------------+--------- boot core | 0x10000000 | 0 core1 | 0x10000004 | 4 core2 | 0x10000008 | 8 core3 | 0x1000000c | 12 But since the spacing of VCPU ids is 8, the DRC for core1 points to a VCPU that doesn't exist, the DRC for core2 points to the first VCPU of core1 and and so on... ... PowerPC,POWER8@0 { ... ibm,my-drc-index = <0x10000000>; ... }; PowerPC,POWER8@8 { ... ibm,my-drc-index = <0x10000008>; ... }; PowerPC,POWER8@10 { ... No ibm,my-drc-index property for this core since 0x10000010 doesn't exist in ibm,drc-indexes above. ... }; }; ... interrupt-controller { ... ibm,interrupt-server-ranges = <0x0 0x10>; With a spacing of 8, the highest VCPU id for the given topology should be: 16 * 8 / 4 = 32 and not 16 ... linux,phandle = <0x7e7323b8>; interrupt-controller; }; And CPU hot-plug/unplug is broken: (qemu) device_del core1 pseries-hotplug-cpu: Cannot find CPU (drc index 10000004) to remove (qemu) device_del core2 cpu 4 (hwid 8) Ready to die... cpu 5 (hwid 9) Ready to die... cpu 6 (hwid 10) Ready to die... cpu 7 (hwid 11) Ready to die... These are the VCPU ids of core1 actually (qemu) device_add host-spapr-cpu-core,core-id=12,id=core3 (qemu) device_del core3 pseries-hotplug-cpu: Cannot find CPU (drc index 1000000c) to remove This patches all the code in hw/ppc/spapr.c to assume the VSMT spacing when manipulating VCPU ids. Fixes: 8904e5a75005 Signed-off-by: Greg Kurz <groug@kaod.org> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2018-02-14 20:40:26 +01:00
return DIV_ROUND_UP(max_cpus * spapr->vsmt, smp_threads);
}
static int spapr_fixup_cpu_smt_dt(void *fdt, int offset, PowerPCCPU *cpu,
int smt_threads)
{
int i, ret = 0;
uint32_t servers_prop[smt_threads];
uint32_t gservers_prop[smt_threads * 2];
int index = spapr_get_vcpu_id(cpu);
if (cpu->compat_pvr) {
ret = fdt_setprop_cell(fdt, offset, "cpu-version", cpu->compat_pvr);
if (ret < 0) {
return ret;
}
}
/* Build interrupt servers and gservers properties */
for (i = 0; i < smt_threads; i++) {
servers_prop[i] = cpu_to_be32(index + i);
/* Hack, direct the group queues back to cpu 0 */
gservers_prop[i*2] = cpu_to_be32(index + i);
gservers_prop[i*2 + 1] = 0;
}
ret = fdt_setprop(fdt, offset, "ibm,ppc-interrupt-server#s",
servers_prop, sizeof(servers_prop));
if (ret < 0) {
return ret;
}
ret = fdt_setprop(fdt, offset, "ibm,ppc-interrupt-gserver#s",
gservers_prop, sizeof(gservers_prop));
return ret;
}
static int spapr_fixup_cpu_numa_dt(void *fdt, int offset, PowerPCCPU *cpu)
{
int index = spapr_get_vcpu_id(cpu);
uint32_t associativity[] = {cpu_to_be32(0x5),
cpu_to_be32(0x0),
cpu_to_be32(0x0),
cpu_to_be32(0x0),
cpu_to_be32(cpu->node_id),
cpu_to_be32(index)};
/* Advertise NUMA via ibm,associativity */
return fdt_setprop(fdt, offset, "ibm,associativity", associativity,
sizeof(associativity));
}
/* Populate the "ibm,pa-features" property */
static void spapr_populate_pa_features(sPAPRMachineState *spapr,
PowerPCCPU *cpu,
void *fdt, int offset,
target/ppc: Update setting of cpu features to account for compat modes The device tree nodes ibm,arch-vec-5-platform-support and ibm,pa-features are used to communicate features of the cpu to the guest operating system. The properties of each of these are determined based on the selected cpu model and the availability of hypervisor features. Currently the compatibility mode of the cpu is not taken into account. The ibm,arch-vec-5-platform-support node is used to communicate the level of support for various ISAv3 processor features to the guest before CAS to inform the guests' request. The available mmu mode should only be hash unless the cpu is a POWER9 which is not in a prePOWER9 compat mode, in which case the available modes depend on the accelerator and the hypervisor capabilities. The ibm,pa-featues node is used to communicate the level of cpu support for various features to the guest os. This should only contain features relevant to the operating mode of the processor, that is the selected cpu model taking into account any compat mode. This means that the compat mode should be taken into account when choosing the properties of ibm,pa-features and they should match the compat mode selected, or the cpu model selected if no compat mode. Update the setting of these cpu features in the device tree as described above to properly take into account any compat mode. We use the ppc_check_compat function which takes into account the current processor model and the cpu compat mode. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-11-17 06:39:00 +01:00
bool legacy_guest)
{
uint8_t pa_features_206[] = { 6, 0,
0xf6, 0x1f, 0xc7, 0x00, 0x80, 0xc0 };
uint8_t pa_features_207[] = { 24, 0,
0xf6, 0x1f, 0xc7, 0xc0, 0x80, 0xf0,
0x80, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x80, 0x00,
0x80, 0x00, 0x80, 0x00, 0x00, 0x00 };
uint8_t pa_features_300[] = { 66, 0,
/* 0: MMU|FPU|SLB|RUN|DABR|NX, 1: fri[nzpm]|DABRX|SPRG3|SLB0|PP110 */
/* 2: VPM|DS205|PPR|DS202|DS206, 3: LSD|URG, SSO, 5: LE|CFAR|EB|LSQ */
0xf6, 0x1f, 0xc7, 0xc0, 0x80, 0xf0, /* 0 - 5 */
/* 6: DS207 */
0x80, 0x00, 0x00, 0x00, 0x00, 0x00, /* 6 - 11 */
/* 16: Vector */
0x00, 0x00, 0x00, 0x00, 0x80, 0x00, /* 12 - 17 */
/* 18: Vec. Scalar, 20: Vec. XOR, 22: HTM */
0x80, 0x00, 0x80, 0x00, 0x00, 0x00, /* 18 - 23 */
/* 24: Ext. Dec, 26: 64 bit ftrs, 28: PM ftrs */
0x80, 0x00, 0x80, 0x00, 0x80, 0x00, /* 24 - 29 */
/* 30: MMR, 32: LE atomic, 34: EBB + ext EBB */
0x80, 0x00, 0x80, 0x00, 0xC0, 0x00, /* 30 - 35 */
/* 36: SPR SO, 38: Copy/Paste, 40: Radix MMU */
0x80, 0x00, 0x80, 0x00, 0x80, 0x00, /* 36 - 41 */
/* 42: PM, 44: PC RA, 46: SC vec'd */
0x80, 0x00, 0x80, 0x00, 0x80, 0x00, /* 42 - 47 */
/* 48: SIMD, 50: QP BFP, 52: String */
0x80, 0x00, 0x80, 0x00, 0x80, 0x00, /* 48 - 53 */
/* 54: DecFP, 56: DecI, 58: SHA */
0x80, 0x00, 0x80, 0x00, 0x80, 0x00, /* 54 - 59 */
/* 60: NM atomic, 62: RNG */
0x80, 0x00, 0x80, 0x00, 0x00, 0x00, /* 60 - 65 */
};
target/ppc: Update setting of cpu features to account for compat modes The device tree nodes ibm,arch-vec-5-platform-support and ibm,pa-features are used to communicate features of the cpu to the guest operating system. The properties of each of these are determined based on the selected cpu model and the availability of hypervisor features. Currently the compatibility mode of the cpu is not taken into account. The ibm,arch-vec-5-platform-support node is used to communicate the level of support for various ISAv3 processor features to the guest before CAS to inform the guests' request. The available mmu mode should only be hash unless the cpu is a POWER9 which is not in a prePOWER9 compat mode, in which case the available modes depend on the accelerator and the hypervisor capabilities. The ibm,pa-featues node is used to communicate the level of cpu support for various features to the guest os. This should only contain features relevant to the operating mode of the processor, that is the selected cpu model taking into account any compat mode. This means that the compat mode should be taken into account when choosing the properties of ibm,pa-features and they should match the compat mode selected, or the cpu model selected if no compat mode. Update the setting of these cpu features in the device tree as described above to properly take into account any compat mode. We use the ppc_check_compat function which takes into account the current processor model and the cpu compat mode. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-11-17 06:39:00 +01:00
uint8_t *pa_features = NULL;
size_t pa_size;
target/ppc: Update setting of cpu features to account for compat modes The device tree nodes ibm,arch-vec-5-platform-support and ibm,pa-features are used to communicate features of the cpu to the guest operating system. The properties of each of these are determined based on the selected cpu model and the availability of hypervisor features. Currently the compatibility mode of the cpu is not taken into account. The ibm,arch-vec-5-platform-support node is used to communicate the level of support for various ISAv3 processor features to the guest before CAS to inform the guests' request. The available mmu mode should only be hash unless the cpu is a POWER9 which is not in a prePOWER9 compat mode, in which case the available modes depend on the accelerator and the hypervisor capabilities. The ibm,pa-featues node is used to communicate the level of cpu support for various features to the guest os. This should only contain features relevant to the operating mode of the processor, that is the selected cpu model taking into account any compat mode. This means that the compat mode should be taken into account when choosing the properties of ibm,pa-features and they should match the compat mode selected, or the cpu model selected if no compat mode. Update the setting of these cpu features in the device tree as described above to properly take into account any compat mode. We use the ppc_check_compat function which takes into account the current processor model and the cpu compat mode. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-11-17 06:39:00 +01:00
if (ppc_check_compat(cpu, CPU_POWERPC_LOGICAL_2_06, 0, cpu->compat_pvr)) {
pa_features = pa_features_206;
pa_size = sizeof(pa_features_206);
target/ppc: Update setting of cpu features to account for compat modes The device tree nodes ibm,arch-vec-5-platform-support and ibm,pa-features are used to communicate features of the cpu to the guest operating system. The properties of each of these are determined based on the selected cpu model and the availability of hypervisor features. Currently the compatibility mode of the cpu is not taken into account. The ibm,arch-vec-5-platform-support node is used to communicate the level of support for various ISAv3 processor features to the guest before CAS to inform the guests' request. The available mmu mode should only be hash unless the cpu is a POWER9 which is not in a prePOWER9 compat mode, in which case the available modes depend on the accelerator and the hypervisor capabilities. The ibm,pa-featues node is used to communicate the level of cpu support for various features to the guest os. This should only contain features relevant to the operating mode of the processor, that is the selected cpu model taking into account any compat mode. This means that the compat mode should be taken into account when choosing the properties of ibm,pa-features and they should match the compat mode selected, or the cpu model selected if no compat mode. Update the setting of these cpu features in the device tree as described above to properly take into account any compat mode. We use the ppc_check_compat function which takes into account the current processor model and the cpu compat mode. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-11-17 06:39:00 +01:00
}
if (ppc_check_compat(cpu, CPU_POWERPC_LOGICAL_2_07, 0, cpu->compat_pvr)) {
pa_features = pa_features_207;
pa_size = sizeof(pa_features_207);
target/ppc: Update setting of cpu features to account for compat modes The device tree nodes ibm,arch-vec-5-platform-support and ibm,pa-features are used to communicate features of the cpu to the guest operating system. The properties of each of these are determined based on the selected cpu model and the availability of hypervisor features. Currently the compatibility mode of the cpu is not taken into account. The ibm,arch-vec-5-platform-support node is used to communicate the level of support for various ISAv3 processor features to the guest before CAS to inform the guests' request. The available mmu mode should only be hash unless the cpu is a POWER9 which is not in a prePOWER9 compat mode, in which case the available modes depend on the accelerator and the hypervisor capabilities. The ibm,pa-featues node is used to communicate the level of cpu support for various features to the guest os. This should only contain features relevant to the operating mode of the processor, that is the selected cpu model taking into account any compat mode. This means that the compat mode should be taken into account when choosing the properties of ibm,pa-features and they should match the compat mode selected, or the cpu model selected if no compat mode. Update the setting of these cpu features in the device tree as described above to properly take into account any compat mode. We use the ppc_check_compat function which takes into account the current processor model and the cpu compat mode. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-11-17 06:39:00 +01:00
}
if (ppc_check_compat(cpu, CPU_POWERPC_LOGICAL_3_00, 0, cpu->compat_pvr)) {
pa_features = pa_features_300;
pa_size = sizeof(pa_features_300);
target/ppc: Update setting of cpu features to account for compat modes The device tree nodes ibm,arch-vec-5-platform-support and ibm,pa-features are used to communicate features of the cpu to the guest operating system. The properties of each of these are determined based on the selected cpu model and the availability of hypervisor features. Currently the compatibility mode of the cpu is not taken into account. The ibm,arch-vec-5-platform-support node is used to communicate the level of support for various ISAv3 processor features to the guest before CAS to inform the guests' request. The available mmu mode should only be hash unless the cpu is a POWER9 which is not in a prePOWER9 compat mode, in which case the available modes depend on the accelerator and the hypervisor capabilities. The ibm,pa-featues node is used to communicate the level of cpu support for various features to the guest os. This should only contain features relevant to the operating mode of the processor, that is the selected cpu model taking into account any compat mode. This means that the compat mode should be taken into account when choosing the properties of ibm,pa-features and they should match the compat mode selected, or the cpu model selected if no compat mode. Update the setting of these cpu features in the device tree as described above to properly take into account any compat mode. We use the ppc_check_compat function which takes into account the current processor model and the cpu compat mode. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-11-17 06:39:00 +01:00
}
if (!pa_features) {
return;
}
if (ppc_hash64_has(cpu, PPC_HASH64_CI_LARGEPAGE)) {
/*
* Note: we keep CI large pages off by default because a 64K capable
* guest provisioned with large pages might otherwise try to map a qemu
* framebuffer (or other kind of memory mapped PCI BAR) using 64K pages
* even if that qemu runs on a 4k host.
* We dd this bit back here if we are confident this is not an issue
*/
pa_features[3] |= 0x20;
}
if ((spapr_get_cap(spapr, SPAPR_CAP_HTM) != 0) && pa_size > 24) {
pa_features[24] |= 0x80; /* Transactional memory support */
}
if (legacy_guest && pa_size > 40) {
/* Workaround for broken kernels that attempt (guest) radix
* mode when they can't handle it, if they see the radix bit set
* in pa-features. So hide it from them. */
pa_features[40 + 2] &= ~0x80; /* Radix MMU */
}
_FDT((fdt_setprop(fdt, offset, "ibm,pa-features", pa_features, pa_size)));
}
static int spapr_fixup_cpu_dt(void *fdt, sPAPRMachineState *spapr)
{
int ret = 0, offset, cpus_offset;
CPUState *cs;
char cpu_model[32];
uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)};
CPU_FOREACH(cs) {
PowerPCCPU *cpu = POWERPC_CPU(cs);
DeviceClass *dc = DEVICE_GET_CLASS(cs);
int index = spapr_get_vcpu_id(cpu);
int compat_smt = MIN(smp_threads, ppc_compat_max_vthreads(cpu));
if (!spapr_is_thread0_in_vcore(spapr, cpu)) {
continue;
}
snprintf(cpu_model, 32, "%s@%x", dc->fw_name, index);
cpus_offset = fdt_path_offset(fdt, "/cpus");
if (cpus_offset < 0) {
cpus_offset = fdt_add_subnode(fdt, 0, "cpus");
if (cpus_offset < 0) {
return cpus_offset;
}
}
offset = fdt_subnode_offset(fdt, cpus_offset, cpu_model);
if (offset < 0) {
offset = fdt_add_subnode(fdt, cpus_offset, cpu_model);
if (offset < 0) {
return offset;
}
}
ret = fdt_setprop(fdt, offset, "ibm,pft-size",
pft_size_prop, sizeof(pft_size_prop));
if (ret < 0) {
return ret;
}
if (nb_numa_nodes > 1) {
ret = spapr_fixup_cpu_numa_dt(fdt, offset, cpu);
if (ret < 0) {
return ret;
}
}
ret = spapr_fixup_cpu_smt_dt(fdt, offset, cpu, compat_smt);
if (ret < 0) {
return ret;
}
spapr_populate_pa_features(spapr, cpu, fdt, offset,
spapr->cas_legacy_guest_workaround);
}
return ret;
}
static hwaddr spapr_node0_size(MachineState *machine)
{
if (nb_numa_nodes) {
int i;
for (i = 0; i < nb_numa_nodes; ++i) {
if (numa_info[i].node_mem) {
return MIN(pow2floor(numa_info[i].node_mem),
machine->ram_size);
}
}
}
return machine->ram_size;
}
static void add_str(GString *s, const gchar *s1)
{
g_string_append_len(s, s1, strlen(s1) + 1);
}
static int spapr_populate_memory_node(void *fdt, int nodeid, hwaddr start,
hwaddr size)
{
uint32_t associativity[] = {
cpu_to_be32(0x4), /* length */
cpu_to_be32(0x0), cpu_to_be32(0x0),
cpu_to_be32(0x0), cpu_to_be32(nodeid)
};
char mem_name[32];
uint64_t mem_reg_property[2];
int off;
mem_reg_property[0] = cpu_to_be64(start);
mem_reg_property[1] = cpu_to_be64(size);
sprintf(mem_name, "memory@" TARGET_FMT_lx, start);
off = fdt_add_subnode(fdt, 0, mem_name);
_FDT(off);
_FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
_FDT((fdt_setprop(fdt, off, "reg", mem_reg_property,
sizeof(mem_reg_property))));
_FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
sizeof(associativity))));
return off;
}
static int spapr_populate_memory(sPAPRMachineState *spapr, void *fdt)
{
MachineState *machine = MACHINE(spapr);
hwaddr mem_start, node_size;
int i, nb_nodes = nb_numa_nodes;
NodeInfo *nodes = numa_info;
NodeInfo ramnode;
/* No NUMA nodes, assume there is just one node with whole RAM */
if (!nb_numa_nodes) {
nb_nodes = 1;
ramnode.node_mem = machine->ram_size;
nodes = &ramnode;
}
for (i = 0, mem_start = 0; i < nb_nodes; ++i) {
if (!nodes[i].node_mem) {
continue;
}
if (mem_start >= machine->ram_size) {
node_size = 0;
} else {
node_size = nodes[i].node_mem;
if (node_size > machine->ram_size - mem_start) {
node_size = machine->ram_size - mem_start;
}
}
if (!mem_start) {
/* spapr_machine_init() checks for rma_size <= node0_size
* already */
spapr_populate_memory_node(fdt, i, 0, spapr->rma_size);
mem_start += spapr->rma_size;
node_size -= spapr->rma_size;
}
for ( ; node_size; ) {
hwaddr sizetmp = pow2floor(node_size);
/* mem_start != 0 here */
if (ctzl(mem_start) < ctzl(sizetmp)) {
sizetmp = 1ULL << ctzl(mem_start);
}
spapr_populate_memory_node(fdt, i, mem_start, sizetmp);
node_size -= sizetmp;
mem_start += sizetmp;
}
}
return 0;
}
static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset,
sPAPRMachineState *spapr)
{
PowerPCCPU *cpu = POWERPC_CPU(cs);
CPUPPCState *env = &cpu->env;
PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cs);
int index = spapr_get_vcpu_id(cpu);
uint32_t segs[] = {cpu_to_be32(28), cpu_to_be32(40),
0xffffffff, 0xffffffff};
uint32_t tbfreq = kvm_enabled() ? kvmppc_get_tbfreq()
: SPAPR_TIMEBASE_FREQ;
uint32_t cpufreq = kvm_enabled() ? kvmppc_get_clockfreq() : 1000000000;
uint32_t page_sizes_prop[64];
size_t page_sizes_prop_size;
uint32_t vcpus_per_socket = smp_threads * smp_cores;
uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)};
int compat_smt = MIN(smp_threads, ppc_compat_max_vthreads(cpu));
sPAPRDRConnector *drc;
int drc_index;
uint32_t radix_AP_encodings[PPC_PAGE_SIZES_MAX_SZ];
int i;
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_CPU, index);
if (drc) {
drc_index = spapr_drc_index(drc);
_FDT((fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_index)));
}
_FDT((fdt_setprop_cell(fdt, offset, "reg", index)));
_FDT((fdt_setprop_string(fdt, offset, "device_type", "cpu")));
_FDT((fdt_setprop_cell(fdt, offset, "cpu-version", env->spr[SPR_PVR])));
_FDT((fdt_setprop_cell(fdt, offset, "d-cache-block-size",
env->dcache_line_size)));
_FDT((fdt_setprop_cell(fdt, offset, "d-cache-line-size",
env->dcache_line_size)));
_FDT((fdt_setprop_cell(fdt, offset, "i-cache-block-size",
env->icache_line_size)));
_FDT((fdt_setprop_cell(fdt, offset, "i-cache-line-size",
env->icache_line_size)));
if (pcc->l1_dcache_size) {
_FDT((fdt_setprop_cell(fdt, offset, "d-cache-size",
pcc->l1_dcache_size)));
} else {
Convert error_report() to warn_report() Convert all uses of error_report("warning:"... to use warn_report() instead. This helps standardise on a single method of printing warnings to the user. All of the warnings were changed using these two commands: find ./* -type f -exec sed -i \ 's|error_report(".*warning[,:] |warn_report("|Ig' {} + Indentation fixed up manually afterwards. The test-qdev-global-props test case was manually updated to ensure that this patch passes make check (as the test cases are case sensitive). Signed-off-by: Alistair Francis <alistair.francis@xilinx.com> Suggested-by: Thomas Huth <thuth@redhat.com> Cc: Jeff Cody <jcody@redhat.com> Cc: Kevin Wolf <kwolf@redhat.com> Cc: Max Reitz <mreitz@redhat.com> Cc: Ronnie Sahlberg <ronniesahlberg@gmail.com> Cc: Paolo Bonzini <pbonzini@redhat.com> Cc: Peter Lieven <pl@kamp.de> Cc: Josh Durgin <jdurgin@redhat.com> Cc: "Richard W.M. Jones" <rjones@redhat.com> Cc: Markus Armbruster <armbru@redhat.com> Cc: Peter Crosthwaite <crosthwaite.peter@gmail.com> Cc: Richard Henderson <rth@twiddle.net> Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com> Cc: Greg Kurz <groug@kaod.org> Cc: Rob Herring <robh@kernel.org> Cc: Peter Maydell <peter.maydell@linaro.org> Cc: Peter Chubb <peter.chubb@nicta.com.au> Cc: Eduardo Habkost <ehabkost@redhat.com> Cc: Marcel Apfelbaum <marcel@redhat.com> Cc: "Michael S. Tsirkin" <mst@redhat.com> Cc: Igor Mammedov <imammedo@redhat.com> Cc: David Gibson <david@gibson.dropbear.id.au> Cc: Alexander Graf <agraf@suse.de> Cc: Gerd Hoffmann <kraxel@redhat.com> Cc: Jason Wang <jasowang@redhat.com> Cc: Marcelo Tosatti <mtosatti@redhat.com> Cc: Christian Borntraeger <borntraeger@de.ibm.com> Cc: Cornelia Huck <cohuck@redhat.com> Cc: Stefan Hajnoczi <stefanha@redhat.com> Acked-by: David Gibson <david@gibson.dropbear.id.au> Acked-by: Greg Kurz <groug@kaod.org> Acked-by: Cornelia Huck <cohuck@redhat.com> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed by: Peter Chubb <peter.chubb@data61.csiro.au> Acked-by: Max Reitz <mreitz@redhat.com> Acked-by: Marcel Apfelbaum <marcel@redhat.com> Message-Id: <e1cfa2cd47087c248dd24caca9c33d9af0c499b0.1499866456.git.alistair.francis@xilinx.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-07-12 15:57:41 +02:00
warn_report("Unknown L1 dcache size for cpu");
}
if (pcc->l1_icache_size) {
_FDT((fdt_setprop_cell(fdt, offset, "i-cache-size",
pcc->l1_icache_size)));
} else {
Convert error_report() to warn_report() Convert all uses of error_report("warning:"... to use warn_report() instead. This helps standardise on a single method of printing warnings to the user. All of the warnings were changed using these two commands: find ./* -type f -exec sed -i \ 's|error_report(".*warning[,:] |warn_report("|Ig' {} + Indentation fixed up manually afterwards. The test-qdev-global-props test case was manually updated to ensure that this patch passes make check (as the test cases are case sensitive). Signed-off-by: Alistair Francis <alistair.francis@xilinx.com> Suggested-by: Thomas Huth <thuth@redhat.com> Cc: Jeff Cody <jcody@redhat.com> Cc: Kevin Wolf <kwolf@redhat.com> Cc: Max Reitz <mreitz@redhat.com> Cc: Ronnie Sahlberg <ronniesahlberg@gmail.com> Cc: Paolo Bonzini <pbonzini@redhat.com> Cc: Peter Lieven <pl@kamp.de> Cc: Josh Durgin <jdurgin@redhat.com> Cc: "Richard W.M. Jones" <rjones@redhat.com> Cc: Markus Armbruster <armbru@redhat.com> Cc: Peter Crosthwaite <crosthwaite.peter@gmail.com> Cc: Richard Henderson <rth@twiddle.net> Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com> Cc: Greg Kurz <groug@kaod.org> Cc: Rob Herring <robh@kernel.org> Cc: Peter Maydell <peter.maydell@linaro.org> Cc: Peter Chubb <peter.chubb@nicta.com.au> Cc: Eduardo Habkost <ehabkost@redhat.com> Cc: Marcel Apfelbaum <marcel@redhat.com> Cc: "Michael S. Tsirkin" <mst@redhat.com> Cc: Igor Mammedov <imammedo@redhat.com> Cc: David Gibson <david@gibson.dropbear.id.au> Cc: Alexander Graf <agraf@suse.de> Cc: Gerd Hoffmann <kraxel@redhat.com> Cc: Jason Wang <jasowang@redhat.com> Cc: Marcelo Tosatti <mtosatti@redhat.com> Cc: Christian Borntraeger <borntraeger@de.ibm.com> Cc: Cornelia Huck <cohuck@redhat.com> Cc: Stefan Hajnoczi <stefanha@redhat.com> Acked-by: David Gibson <david@gibson.dropbear.id.au> Acked-by: Greg Kurz <groug@kaod.org> Acked-by: Cornelia Huck <cohuck@redhat.com> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed by: Peter Chubb <peter.chubb@data61.csiro.au> Acked-by: Max Reitz <mreitz@redhat.com> Acked-by: Marcel Apfelbaum <marcel@redhat.com> Message-Id: <e1cfa2cd47087c248dd24caca9c33d9af0c499b0.1499866456.git.alistair.francis@xilinx.com> Reviewed-by: Markus Armbruster <armbru@redhat.com> Signed-off-by: Markus Armbruster <armbru@redhat.com>
2017-07-12 15:57:41 +02:00
warn_report("Unknown L1 icache size for cpu");
}
_FDT((fdt_setprop_cell(fdt, offset, "timebase-frequency", tbfreq)));
_FDT((fdt_setprop_cell(fdt, offset, "clock-frequency", cpufreq)));
_FDT((fdt_setprop_cell(fdt, offset, "slb-size", cpu->hash64_opts->slb_size)));
_FDT((fdt_setprop_cell(fdt, offset, "ibm,slb-size", cpu->hash64_opts->slb_size)));
_FDT((fdt_setprop_string(fdt, offset, "status", "okay")));
_FDT((fdt_setprop(fdt, offset, "64-bit", NULL, 0)));
if (env->spr_cb[SPR_PURR].oea_read) {
_FDT((fdt_setprop(fdt, offset, "ibm,purr", NULL, 0)));
}
if (ppc_hash64_has(cpu, PPC_HASH64_1TSEG)) {
_FDT((fdt_setprop(fdt, offset, "ibm,processor-segment-sizes",
segs, sizeof(segs))));
}
/* Advertise VSX (vector extensions) if available
* 1 == VMX / Altivec available
* 2 == VSX available
*
* Only CPUs for which we create core types in spapr_cpu_core.c
* are possible, and all of those have VMX */
if (spapr_get_cap(spapr, SPAPR_CAP_VSX) != 0) {
_FDT((fdt_setprop_cell(fdt, offset, "ibm,vmx", 2)));
} else {
_FDT((fdt_setprop_cell(fdt, offset, "ibm,vmx", 1)));
}
/* Advertise DFP (Decimal Floating Point) if available
* 0 / no property == no DFP
* 1 == DFP available */
if (spapr_get_cap(spapr, SPAPR_CAP_DFP) != 0) {
_FDT((fdt_setprop_cell(fdt, offset, "ibm,dfp", 1)));
}
page_sizes_prop_size = ppc_create_page_sizes_prop(cpu, page_sizes_prop,
sizeof(page_sizes_prop));
if (page_sizes_prop_size) {
_FDT((fdt_setprop(fdt, offset, "ibm,segment-page-sizes",
page_sizes_prop, page_sizes_prop_size)));
}
spapr_populate_pa_features(spapr, cpu, fdt, offset, false);
_FDT((fdt_setprop_cell(fdt, offset, "ibm,chip-id",
cs->cpu_index / vcpus_per_socket)));
_FDT((fdt_setprop(fdt, offset, "ibm,pft-size",
pft_size_prop, sizeof(pft_size_prop))));
if (nb_numa_nodes > 1) {
_FDT(spapr_fixup_cpu_numa_dt(fdt, offset, cpu));
}
_FDT(spapr_fixup_cpu_smt_dt(fdt, offset, cpu, compat_smt));
if (pcc->radix_page_info) {
for (i = 0; i < pcc->radix_page_info->count; i++) {
radix_AP_encodings[i] =
cpu_to_be32(pcc->radix_page_info->entries[i]);
}
_FDT((fdt_setprop(fdt, offset, "ibm,processor-radix-AP-encodings",
radix_AP_encodings,
pcc->radix_page_info->count *
sizeof(radix_AP_encodings[0]))));
}
}
static void spapr_populate_cpus_dt_node(void *fdt, sPAPRMachineState *spapr)
{
CPUState **rev;
CPUState *cs;
int n_cpus;
int cpus_offset;
char *nodename;
int i;
cpus_offset = fdt_add_subnode(fdt, 0, "cpus");
_FDT(cpus_offset);
_FDT((fdt_setprop_cell(fdt, cpus_offset, "#address-cells", 0x1)));
_FDT((fdt_setprop_cell(fdt, cpus_offset, "#size-cells", 0x0)));
/*
* We walk the CPUs in reverse order to ensure that CPU DT nodes
* created by fdt_add_subnode() end up in the right order in FDT
* for the guest kernel the enumerate the CPUs correctly.
*
* The CPU list cannot be traversed in reverse order, so we need
* to do extra work.
*/
n_cpus = 0;
rev = NULL;
CPU_FOREACH(cs) {
rev = g_renew(CPUState *, rev, n_cpus + 1);
rev[n_cpus++] = cs;
}
for (i = n_cpus - 1; i >= 0; i--) {
CPUState *cs = rev[i];
PowerPCCPU *cpu = POWERPC_CPU(cs);
int index = spapr_get_vcpu_id(cpu);
DeviceClass *dc = DEVICE_GET_CLASS(cs);
int offset;
if (!spapr_is_thread0_in_vcore(spapr, cpu)) {
continue;
}
nodename = g_strdup_printf("%s@%x", dc->fw_name, index);
offset = fdt_add_subnode(fdt, cpus_offset, nodename);
g_free(nodename);
_FDT(offset);
spapr_populate_cpu_dt(cs, fdt, offset, spapr);
}
g_free(rev);
}
static int spapr_rng_populate_dt(void *fdt)
{
int node;
int ret;
node = qemu_fdt_add_subnode(fdt, "/ibm,platform-facilities");
if (node <= 0) {
return -1;
}
ret = fdt_setprop_string(fdt, node, "device_type",
"ibm,platform-facilities");
ret |= fdt_setprop_cell(fdt, node, "#address-cells", 0x1);
ret |= fdt_setprop_cell(fdt, node, "#size-cells", 0x0);
node = fdt_add_subnode(fdt, node, "ibm,random-v1");
if (node <= 0) {
return -1;
}
ret |= fdt_setprop_string(fdt, node, "compatible", "ibm,random");
return ret ? -1 : 0;
}
static uint32_t spapr_pc_dimm_node(MemoryDeviceInfoList *list, ram_addr_t addr)
{
MemoryDeviceInfoList *info;
for (info = list; info; info = info->next) {
MemoryDeviceInfo *value = info->value;
if (value && value->type == MEMORY_DEVICE_INFO_KIND_DIMM) {
PCDIMMDeviceInfo *pcdimm_info = value->u.dimm.data;
if (addr >= pcdimm_info->addr &&
addr < (pcdimm_info->addr + pcdimm_info->size)) {
return pcdimm_info->node;
}
}
}
return -1;
}
struct sPAPRDrconfCellV2 {
uint32_t seq_lmbs;
uint64_t base_addr;
uint32_t drc_index;
uint32_t aa_index;
uint32_t flags;
} QEMU_PACKED;
typedef struct DrconfCellQueue {
struct sPAPRDrconfCellV2 cell;
QSIMPLEQ_ENTRY(DrconfCellQueue) entry;
} DrconfCellQueue;
static DrconfCellQueue *
spapr_get_drconf_cell(uint32_t seq_lmbs, uint64_t base_addr,
uint32_t drc_index, uint32_t aa_index,
uint32_t flags)
{
DrconfCellQueue *elem;
elem = g_malloc0(sizeof(*elem));
elem->cell.seq_lmbs = cpu_to_be32(seq_lmbs);
elem->cell.base_addr = cpu_to_be64(base_addr);
elem->cell.drc_index = cpu_to_be32(drc_index);
elem->cell.aa_index = cpu_to_be32(aa_index);
elem->cell.flags = cpu_to_be32(flags);
return elem;
}
/* ibm,dynamic-memory-v2 */
static int spapr_populate_drmem_v2(sPAPRMachineState *spapr, void *fdt,
int offset, MemoryDeviceInfoList *dimms)
{
MachineState *machine = MACHINE(spapr);
uint8_t *int_buf, *cur_index;
int ret;
uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
uint64_t addr, cur_addr, size;
uint32_t nr_boot_lmbs = (machine->device_memory->base / lmb_size);
uint64_t mem_end = machine->device_memory->base +
memory_region_size(&machine->device_memory->mr);
uint32_t node, buf_len, nr_entries = 0;
sPAPRDRConnector *drc;
DrconfCellQueue *elem, *next;
MemoryDeviceInfoList *info;
QSIMPLEQ_HEAD(, DrconfCellQueue) drconf_queue
= QSIMPLEQ_HEAD_INITIALIZER(drconf_queue);
/* Entry to cover RAM and the gap area */
elem = spapr_get_drconf_cell(nr_boot_lmbs, 0, 0, -1,
SPAPR_LMB_FLAGS_RESERVED |
SPAPR_LMB_FLAGS_DRC_INVALID);
QSIMPLEQ_INSERT_TAIL(&drconf_queue, elem, entry);
nr_entries++;
cur_addr = machine->device_memory->base;
for (info = dimms; info; info = info->next) {
PCDIMMDeviceInfo *di = info->value->u.dimm.data;
addr = di->addr;
size = di->size;
node = di->node;
/* Entry for hot-pluggable area */
if (cur_addr < addr) {
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB, cur_addr / lmb_size);
g_assert(drc);
elem = spapr_get_drconf_cell((addr - cur_addr) / lmb_size,
cur_addr, spapr_drc_index(drc), -1, 0);
QSIMPLEQ_INSERT_TAIL(&drconf_queue, elem, entry);
nr_entries++;
}
/* Entry for DIMM */
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB, addr / lmb_size);
g_assert(drc);
elem = spapr_get_drconf_cell(size / lmb_size, addr,
spapr_drc_index(drc), node,
SPAPR_LMB_FLAGS_ASSIGNED);
QSIMPLEQ_INSERT_TAIL(&drconf_queue, elem, entry);
nr_entries++;
cur_addr = addr + size;
}
/* Entry for remaining hotpluggable area */
if (cur_addr < mem_end) {
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB, cur_addr / lmb_size);
g_assert(drc);
elem = spapr_get_drconf_cell((mem_end - cur_addr) / lmb_size,
cur_addr, spapr_drc_index(drc), -1, 0);
QSIMPLEQ_INSERT_TAIL(&drconf_queue, elem, entry);
nr_entries++;
}
buf_len = nr_entries * sizeof(struct sPAPRDrconfCellV2) + sizeof(uint32_t);
int_buf = cur_index = g_malloc0(buf_len);
*(uint32_t *)int_buf = cpu_to_be32(nr_entries);
cur_index += sizeof(nr_entries);
QSIMPLEQ_FOREACH_SAFE(elem, &drconf_queue, entry, next) {
memcpy(cur_index, &elem->cell, sizeof(elem->cell));
cur_index += sizeof(elem->cell);
QSIMPLEQ_REMOVE(&drconf_queue, elem, DrconfCellQueue, entry);
g_free(elem);
}
ret = fdt_setprop(fdt, offset, "ibm,dynamic-memory-v2", int_buf, buf_len);
g_free(int_buf);
if (ret < 0) {
return -1;
}
return 0;
}
/* ibm,dynamic-memory */
static int spapr_populate_drmem_v1(sPAPRMachineState *spapr, void *fdt,
int offset, MemoryDeviceInfoList *dimms)
{
MachineState *machine = MACHINE(spapr);
int i, ret;
uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
uint32_t device_lmb_start = machine->device_memory->base / lmb_size;
uint32_t nr_lmbs = (machine->device_memory->base +
memory_region_size(&machine->device_memory->mr)) /
spapr: Ensure all LMBs are represented in ibm,dynamic-memory Memory hotplug can fail for some combinations of RAM and maxmem when DDW is enabled in the presence of devices like nec-usb-xhci. DDW depends on maximum addressable memory returned by guest and this value is currently being calculated wrongly by the guest kernel routine memory_hotplug_max(). While there is an attempt to fix the guest kernel, this patch works around the problem within QEMU itself. memory_hotplug_max() routine in the guest kernel arrives at max addressable memory by multiplying lmb-size with the lmb-count obtained from ibm,dynamic-memory property. There are two assumptions here: - All LMBs are part of ibm,dynamic memory: This is not true for PowerKVM where only hot-pluggable LMBs are present in this property. - The memory area comprising of RAM and hotplug region is contiguous: This needn't be true always for PowerKVM as there can be gap between boot time RAM and hotplug region. To work around this guest kernel bug, ensure that ibm,dynamic-memory has information about all the LMBs (RMA, boot-time LMBs, future hotpluggable LMBs, and dummy LMBs to cover the gap between RAM and hotpluggable region). RMA is represented separately by memory@0 node. Hence mark RMA LMBs and also the LMBs for the gap b/n RAM and hotpluggable region as reserved and as having no valid DRC so that these LMBs are not considered by the guest. Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> Reviewed-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: Nathan Fontenot <nfont@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-06-10 07:14:48 +02:00
lmb_size;
uint32_t *int_buf, *cur_index, buf_len;
/*
* Allocate enough buffer size to fit in ibm,dynamic-memory
*/
buf_len = (nr_lmbs * SPAPR_DR_LMB_LIST_ENTRY_SIZE + 1) * sizeof(uint32_t);
cur_index = int_buf = g_malloc0(buf_len);
int_buf[0] = cpu_to_be32(nr_lmbs);
cur_index++;
for (i = 0; i < nr_lmbs; i++) {
spapr: Ensure all LMBs are represented in ibm,dynamic-memory Memory hotplug can fail for some combinations of RAM and maxmem when DDW is enabled in the presence of devices like nec-usb-xhci. DDW depends on maximum addressable memory returned by guest and this value is currently being calculated wrongly by the guest kernel routine memory_hotplug_max(). While there is an attempt to fix the guest kernel, this patch works around the problem within QEMU itself. memory_hotplug_max() routine in the guest kernel arrives at max addressable memory by multiplying lmb-size with the lmb-count obtained from ibm,dynamic-memory property. There are two assumptions here: - All LMBs are part of ibm,dynamic memory: This is not true for PowerKVM where only hot-pluggable LMBs are present in this property. - The memory area comprising of RAM and hotplug region is contiguous: This needn't be true always for PowerKVM as there can be gap between boot time RAM and hotplug region. To work around this guest kernel bug, ensure that ibm,dynamic-memory has information about all the LMBs (RMA, boot-time LMBs, future hotpluggable LMBs, and dummy LMBs to cover the gap between RAM and hotpluggable region). RMA is represented separately by memory@0 node. Hence mark RMA LMBs and also the LMBs for the gap b/n RAM and hotpluggable region as reserved and as having no valid DRC so that these LMBs are not considered by the guest. Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> Reviewed-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: Nathan Fontenot <nfont@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-06-10 07:14:48 +02:00
uint64_t addr = i * lmb_size;
uint32_t *dynamic_memory = cur_index;
if (i >= device_lmb_start) {
spapr: Ensure all LMBs are represented in ibm,dynamic-memory Memory hotplug can fail for some combinations of RAM and maxmem when DDW is enabled in the presence of devices like nec-usb-xhci. DDW depends on maximum addressable memory returned by guest and this value is currently being calculated wrongly by the guest kernel routine memory_hotplug_max(). While there is an attempt to fix the guest kernel, this patch works around the problem within QEMU itself. memory_hotplug_max() routine in the guest kernel arrives at max addressable memory by multiplying lmb-size with the lmb-count obtained from ibm,dynamic-memory property. There are two assumptions here: - All LMBs are part of ibm,dynamic memory: This is not true for PowerKVM where only hot-pluggable LMBs are present in this property. - The memory area comprising of RAM and hotplug region is contiguous: This needn't be true always for PowerKVM as there can be gap between boot time RAM and hotplug region. To work around this guest kernel bug, ensure that ibm,dynamic-memory has information about all the LMBs (RMA, boot-time LMBs, future hotpluggable LMBs, and dummy LMBs to cover the gap between RAM and hotpluggable region). RMA is represented separately by memory@0 node. Hence mark RMA LMBs and also the LMBs for the gap b/n RAM and hotpluggable region as reserved and as having no valid DRC so that these LMBs are not considered by the guest. Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> Reviewed-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: Nathan Fontenot <nfont@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-06-10 07:14:48 +02:00
sPAPRDRConnector *drc;
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB, i);
spapr: Ensure all LMBs are represented in ibm,dynamic-memory Memory hotplug can fail for some combinations of RAM and maxmem when DDW is enabled in the presence of devices like nec-usb-xhci. DDW depends on maximum addressable memory returned by guest and this value is currently being calculated wrongly by the guest kernel routine memory_hotplug_max(). While there is an attempt to fix the guest kernel, this patch works around the problem within QEMU itself. memory_hotplug_max() routine in the guest kernel arrives at max addressable memory by multiplying lmb-size with the lmb-count obtained from ibm,dynamic-memory property. There are two assumptions here: - All LMBs are part of ibm,dynamic memory: This is not true for PowerKVM where only hot-pluggable LMBs are present in this property. - The memory area comprising of RAM and hotplug region is contiguous: This needn't be true always for PowerKVM as there can be gap between boot time RAM and hotplug region. To work around this guest kernel bug, ensure that ibm,dynamic-memory has information about all the LMBs (RMA, boot-time LMBs, future hotpluggable LMBs, and dummy LMBs to cover the gap between RAM and hotpluggable region). RMA is represented separately by memory@0 node. Hence mark RMA LMBs and also the LMBs for the gap b/n RAM and hotpluggable region as reserved and as having no valid DRC so that these LMBs are not considered by the guest. Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> Reviewed-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: Nathan Fontenot <nfont@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-06-10 07:14:48 +02:00
g_assert(drc);
dynamic_memory[0] = cpu_to_be32(addr >> 32);
dynamic_memory[1] = cpu_to_be32(addr & 0xffffffff);
dynamic_memory[2] = cpu_to_be32(spapr_drc_index(drc));
spapr: Ensure all LMBs are represented in ibm,dynamic-memory Memory hotplug can fail for some combinations of RAM and maxmem when DDW is enabled in the presence of devices like nec-usb-xhci. DDW depends on maximum addressable memory returned by guest and this value is currently being calculated wrongly by the guest kernel routine memory_hotplug_max(). While there is an attempt to fix the guest kernel, this patch works around the problem within QEMU itself. memory_hotplug_max() routine in the guest kernel arrives at max addressable memory by multiplying lmb-size with the lmb-count obtained from ibm,dynamic-memory property. There are two assumptions here: - All LMBs are part of ibm,dynamic memory: This is not true for PowerKVM where only hot-pluggable LMBs are present in this property. - The memory area comprising of RAM and hotplug region is contiguous: This needn't be true always for PowerKVM as there can be gap between boot time RAM and hotplug region. To work around this guest kernel bug, ensure that ibm,dynamic-memory has information about all the LMBs (RMA, boot-time LMBs, future hotpluggable LMBs, and dummy LMBs to cover the gap between RAM and hotpluggable region). RMA is represented separately by memory@0 node. Hence mark RMA LMBs and also the LMBs for the gap b/n RAM and hotpluggable region as reserved and as having no valid DRC so that these LMBs are not considered by the guest. Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> Reviewed-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: Nathan Fontenot <nfont@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-06-10 07:14:48 +02:00
dynamic_memory[3] = cpu_to_be32(0); /* reserved */
dynamic_memory[4] = cpu_to_be32(spapr_pc_dimm_node(dimms, addr));
spapr: Ensure all LMBs are represented in ibm,dynamic-memory Memory hotplug can fail for some combinations of RAM and maxmem when DDW is enabled in the presence of devices like nec-usb-xhci. DDW depends on maximum addressable memory returned by guest and this value is currently being calculated wrongly by the guest kernel routine memory_hotplug_max(). While there is an attempt to fix the guest kernel, this patch works around the problem within QEMU itself. memory_hotplug_max() routine in the guest kernel arrives at max addressable memory by multiplying lmb-size with the lmb-count obtained from ibm,dynamic-memory property. There are two assumptions here: - All LMBs are part of ibm,dynamic memory: This is not true for PowerKVM where only hot-pluggable LMBs are present in this property. - The memory area comprising of RAM and hotplug region is contiguous: This needn't be true always for PowerKVM as there can be gap between boot time RAM and hotplug region. To work around this guest kernel bug, ensure that ibm,dynamic-memory has information about all the LMBs (RMA, boot-time LMBs, future hotpluggable LMBs, and dummy LMBs to cover the gap between RAM and hotpluggable region). RMA is represented separately by memory@0 node. Hence mark RMA LMBs and also the LMBs for the gap b/n RAM and hotpluggable region as reserved and as having no valid DRC so that these LMBs are not considered by the guest. Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> Reviewed-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: Nathan Fontenot <nfont@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-06-10 07:14:48 +02:00
if (memory_region_present(get_system_memory(), addr)) {
dynamic_memory[5] = cpu_to_be32(SPAPR_LMB_FLAGS_ASSIGNED);
} else {
dynamic_memory[5] = cpu_to_be32(0);
}
} else {
spapr: Ensure all LMBs are represented in ibm,dynamic-memory Memory hotplug can fail for some combinations of RAM and maxmem when DDW is enabled in the presence of devices like nec-usb-xhci. DDW depends on maximum addressable memory returned by guest and this value is currently being calculated wrongly by the guest kernel routine memory_hotplug_max(). While there is an attempt to fix the guest kernel, this patch works around the problem within QEMU itself. memory_hotplug_max() routine in the guest kernel arrives at max addressable memory by multiplying lmb-size with the lmb-count obtained from ibm,dynamic-memory property. There are two assumptions here: - All LMBs are part of ibm,dynamic memory: This is not true for PowerKVM where only hot-pluggable LMBs are present in this property. - The memory area comprising of RAM and hotplug region is contiguous: This needn't be true always for PowerKVM as there can be gap between boot time RAM and hotplug region. To work around this guest kernel bug, ensure that ibm,dynamic-memory has information about all the LMBs (RMA, boot-time LMBs, future hotpluggable LMBs, and dummy LMBs to cover the gap between RAM and hotpluggable region). RMA is represented separately by memory@0 node. Hence mark RMA LMBs and also the LMBs for the gap b/n RAM and hotpluggable region as reserved and as having no valid DRC so that these LMBs are not considered by the guest. Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> Reviewed-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: Nathan Fontenot <nfont@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-06-10 07:14:48 +02:00
/*
* LMB information for RMA, boot time RAM and gap b/n RAM and
* device memory region -- all these are marked as reserved
spapr: Ensure all LMBs are represented in ibm,dynamic-memory Memory hotplug can fail for some combinations of RAM and maxmem when DDW is enabled in the presence of devices like nec-usb-xhci. DDW depends on maximum addressable memory returned by guest and this value is currently being calculated wrongly by the guest kernel routine memory_hotplug_max(). While there is an attempt to fix the guest kernel, this patch works around the problem within QEMU itself. memory_hotplug_max() routine in the guest kernel arrives at max addressable memory by multiplying lmb-size with the lmb-count obtained from ibm,dynamic-memory property. There are two assumptions here: - All LMBs are part of ibm,dynamic memory: This is not true for PowerKVM where only hot-pluggable LMBs are present in this property. - The memory area comprising of RAM and hotplug region is contiguous: This needn't be true always for PowerKVM as there can be gap between boot time RAM and hotplug region. To work around this guest kernel bug, ensure that ibm,dynamic-memory has information about all the LMBs (RMA, boot-time LMBs, future hotpluggable LMBs, and dummy LMBs to cover the gap between RAM and hotpluggable region). RMA is represented separately by memory@0 node. Hence mark RMA LMBs and also the LMBs for the gap b/n RAM and hotpluggable region as reserved and as having no valid DRC so that these LMBs are not considered by the guest. Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> Reviewed-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: Nathan Fontenot <nfont@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-06-10 07:14:48 +02:00
* and as having no valid DRC.
*/
dynamic_memory[0] = cpu_to_be32(addr >> 32);
dynamic_memory[1] = cpu_to_be32(addr & 0xffffffff);
dynamic_memory[2] = cpu_to_be32(0);
dynamic_memory[3] = cpu_to_be32(0); /* reserved */
dynamic_memory[4] = cpu_to_be32(-1);
dynamic_memory[5] = cpu_to_be32(SPAPR_LMB_FLAGS_RESERVED |
SPAPR_LMB_FLAGS_DRC_INVALID);
}
cur_index += SPAPR_DR_LMB_LIST_ENTRY_SIZE;
}
ret = fdt_setprop(fdt, offset, "ibm,dynamic-memory", int_buf, buf_len);
g_free(int_buf);
if (ret < 0) {
return -1;
}
return 0;
}
/*
* Adds ibm,dynamic-reconfiguration-memory node.
* Refer to docs/specs/ppc-spapr-hotplug.txt for the documentation
* of this device tree node.
*/
static int spapr_populate_drconf_memory(sPAPRMachineState *spapr, void *fdt)
{
MachineState *machine = MACHINE(spapr);
int ret, i, offset;
uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
uint32_t prop_lmb_size[] = {0, cpu_to_be32(lmb_size)};
uint32_t *int_buf, *cur_index, buf_len;
int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
MemoryDeviceInfoList *dimms = NULL;
/*
* Don't create the node if there is no device memory
*/
if (machine->ram_size == machine->maxram_size) {
return 0;
}
offset = fdt_add_subnode(fdt, 0, "ibm,dynamic-reconfiguration-memory");
ret = fdt_setprop(fdt, offset, "ibm,lmb-size", prop_lmb_size,
sizeof(prop_lmb_size));
if (ret < 0) {
return ret;
}
ret = fdt_setprop_cell(fdt, offset, "ibm,memory-flags-mask", 0xff);
if (ret < 0) {
return ret;
}
ret = fdt_setprop_cell(fdt, offset, "ibm,memory-preservation-time", 0x0);
if (ret < 0) {
return ret;
}
/* ibm,dynamic-memory or ibm,dynamic-memory-v2 */
dimms = qmp_memory_device_list();
if (spapr_ovec_test(spapr->ov5_cas, OV5_DRMEM_V2)) {
ret = spapr_populate_drmem_v2(spapr, fdt, offset, dimms);
} else {
ret = spapr_populate_drmem_v1(spapr, fdt, offset, dimms);
}
qapi_free_MemoryDeviceInfoList(dimms);
if (ret < 0) {
return ret;
}
/* ibm,associativity-lookup-arrays */
buf_len = (nr_nodes * 4 + 2) * sizeof(uint32_t);
cur_index = int_buf = g_malloc0(buf_len);
int_buf[0] = cpu_to_be32(nr_nodes);
int_buf[1] = cpu_to_be32(4); /* Number of entries per associativity list */
cur_index += 2;
for (i = 0; i < nr_nodes; i++) {
uint32_t associativity[] = {
cpu_to_be32(0x0),
cpu_to_be32(0x0),
cpu_to_be32(0x0),
cpu_to_be32(i)
};
memcpy(cur_index, associativity, sizeof(associativity));
cur_index += 4;
}
ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
(cur_index - int_buf) * sizeof(uint32_t));
g_free(int_buf);
return ret;
}
spapr: add option vector handling in CAS-generated resets In some cases, ibm,client-architecture-support calls can fail. This could happen in the current code for situations where the modified device tree segment exceeds the buffer size provided by the guest via the call parameters. In these cases, QEMU will reset, allowing an opportunity to regenerate the device tree from scratch via boot-time handling. There are potentially other scenarios as well, not currently reachable in the current code, but possible in theory, such as cases where device-tree properties or nodes need to be removed. We currently don't handle either of these properly for option vector capabilities however. Instead of carrying the negotiated capability beyond the reset and creating the boot-time device tree accordingly, we start from scratch, generating the same boot-time device tree as we did prior to the CAS-generated and the same device tree updates as we did before. This could (in theory) cause us to get stuck in a reset loop. This hasn't been observed, but depending on the extensiveness of CAS-induced device tree updates in the future, could eventually become an issue. Address this by pulling capability-related device tree updates resulting from CAS calls into a common routine, spapr_dt_cas_updates(), and adding an sPAPROptionVector* parameter that allows us to test for newly-negotiated capabilities. We invoke it as follows: 1) When ibm,client-architecture-support gets called, we call spapr_dt_cas_updates() with the set of capabilities added since the previous call to ibm,client-architecture-support. For the initial boot, or a system reset generated by something other than the CAS call itself, this set will consist of *all* options supported both the platform and the guest. For calls to ibm,client-architecture-support immediately after a CAS-induced reset, we call spapr_dt_cas_updates() with only the set of capabilities added since the previous call, since the other capabilities will have already been addressed by the boot-time device-tree this time around. In the unlikely event that capabilities are *removed* since the previous CAS, we will generate a CAS-induced reset. In the unlikely event that we cannot fit the device-tree updates into the buffer provided by the guest, well generate a CAS-induced reset. 2) When a CAS update results in the need to reset the machine and include the updates in the boot-time device tree, we call the spapr_dt_cas_updates() using the full set of negotiated capabilities as part of the reset path. At initial boot, or after a reset generated by something other than the CAS call itself, this set will be empty, resulting in what should be the same boot-time device-tree as we generated prior to this patch. For CAS-induced reset, this routine will be called with the full set of capabilities negotiated by the platform/guest in the previous CAS call, which should result in CAS updates from previous call being accounted for in the initial boot-time device tree. Signed-off-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> [dwg: Changed an int -> bool conversion to be more explicit] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-10-25 06:47:29 +02:00
static int spapr_dt_cas_updates(sPAPRMachineState *spapr, void *fdt,
sPAPROptionVector *ov5_updates)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
int ret = 0, offset;
spapr: add option vector handling in CAS-generated resets In some cases, ibm,client-architecture-support calls can fail. This could happen in the current code for situations where the modified device tree segment exceeds the buffer size provided by the guest via the call parameters. In these cases, QEMU will reset, allowing an opportunity to regenerate the device tree from scratch via boot-time handling. There are potentially other scenarios as well, not currently reachable in the current code, but possible in theory, such as cases where device-tree properties or nodes need to be removed. We currently don't handle either of these properly for option vector capabilities however. Instead of carrying the negotiated capability beyond the reset and creating the boot-time device tree accordingly, we start from scratch, generating the same boot-time device tree as we did prior to the CAS-generated and the same device tree updates as we did before. This could (in theory) cause us to get stuck in a reset loop. This hasn't been observed, but depending on the extensiveness of CAS-induced device tree updates in the future, could eventually become an issue. Address this by pulling capability-related device tree updates resulting from CAS calls into a common routine, spapr_dt_cas_updates(), and adding an sPAPROptionVector* parameter that allows us to test for newly-negotiated capabilities. We invoke it as follows: 1) When ibm,client-architecture-support gets called, we call spapr_dt_cas_updates() with the set of capabilities added since the previous call to ibm,client-architecture-support. For the initial boot, or a system reset generated by something other than the CAS call itself, this set will consist of *all* options supported both the platform and the guest. For calls to ibm,client-architecture-support immediately after a CAS-induced reset, we call spapr_dt_cas_updates() with only the set of capabilities added since the previous call, since the other capabilities will have already been addressed by the boot-time device-tree this time around. In the unlikely event that capabilities are *removed* since the previous CAS, we will generate a CAS-induced reset. In the unlikely event that we cannot fit the device-tree updates into the buffer provided by the guest, well generate a CAS-induced reset. 2) When a CAS update results in the need to reset the machine and include the updates in the boot-time device tree, we call the spapr_dt_cas_updates() using the full set of negotiated capabilities as part of the reset path. At initial boot, or after a reset generated by something other than the CAS call itself, this set will be empty, resulting in what should be the same boot-time device-tree as we generated prior to this patch. For CAS-induced reset, this routine will be called with the full set of capabilities negotiated by the platform/guest in the previous CAS call, which should result in CAS updates from previous call being accounted for in the initial boot-time device tree. Signed-off-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> [dwg: Changed an int -> bool conversion to be more explicit] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-10-25 06:47:29 +02:00
/* Generate ibm,dynamic-reconfiguration-memory node if required */
if (spapr_ovec_test(ov5_updates, OV5_DRCONF_MEMORY)) {
g_assert(smc->dr_lmb_enabled);
ret = spapr_populate_drconf_memory(spapr, fdt);
if (ret) {
goto out;
}
spapr: add option vector handling in CAS-generated resets In some cases, ibm,client-architecture-support calls can fail. This could happen in the current code for situations where the modified device tree segment exceeds the buffer size provided by the guest via the call parameters. In these cases, QEMU will reset, allowing an opportunity to regenerate the device tree from scratch via boot-time handling. There are potentially other scenarios as well, not currently reachable in the current code, but possible in theory, such as cases where device-tree properties or nodes need to be removed. We currently don't handle either of these properly for option vector capabilities however. Instead of carrying the negotiated capability beyond the reset and creating the boot-time device tree accordingly, we start from scratch, generating the same boot-time device tree as we did prior to the CAS-generated and the same device tree updates as we did before. This could (in theory) cause us to get stuck in a reset loop. This hasn't been observed, but depending on the extensiveness of CAS-induced device tree updates in the future, could eventually become an issue. Address this by pulling capability-related device tree updates resulting from CAS calls into a common routine, spapr_dt_cas_updates(), and adding an sPAPROptionVector* parameter that allows us to test for newly-negotiated capabilities. We invoke it as follows: 1) When ibm,client-architecture-support gets called, we call spapr_dt_cas_updates() with the set of capabilities added since the previous call to ibm,client-architecture-support. For the initial boot, or a system reset generated by something other than the CAS call itself, this set will consist of *all* options supported both the platform and the guest. For calls to ibm,client-architecture-support immediately after a CAS-induced reset, we call spapr_dt_cas_updates() with only the set of capabilities added since the previous call, since the other capabilities will have already been addressed by the boot-time device-tree this time around. In the unlikely event that capabilities are *removed* since the previous CAS, we will generate a CAS-induced reset. In the unlikely event that we cannot fit the device-tree updates into the buffer provided by the guest, well generate a CAS-induced reset. 2) When a CAS update results in the need to reset the machine and include the updates in the boot-time device tree, we call the spapr_dt_cas_updates() using the full set of negotiated capabilities as part of the reset path. At initial boot, or after a reset generated by something other than the CAS call itself, this set will be empty, resulting in what should be the same boot-time device-tree as we generated prior to this patch. For CAS-induced reset, this routine will be called with the full set of capabilities negotiated by the platform/guest in the previous CAS call, which should result in CAS updates from previous call being accounted for in the initial boot-time device tree. Signed-off-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> [dwg: Changed an int -> bool conversion to be more explicit] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-10-25 06:47:29 +02:00
}
offset = fdt_path_offset(fdt, "/chosen");
if (offset < 0) {
offset = fdt_add_subnode(fdt, 0, "chosen");
if (offset < 0) {
return offset;
}
}
ret = spapr_ovec_populate_dt(fdt, offset, spapr->ov5_cas,
"ibm,architecture-vec-5");
out:
spapr: add option vector handling in CAS-generated resets In some cases, ibm,client-architecture-support calls can fail. This could happen in the current code for situations where the modified device tree segment exceeds the buffer size provided by the guest via the call parameters. In these cases, QEMU will reset, allowing an opportunity to regenerate the device tree from scratch via boot-time handling. There are potentially other scenarios as well, not currently reachable in the current code, but possible in theory, such as cases where device-tree properties or nodes need to be removed. We currently don't handle either of these properly for option vector capabilities however. Instead of carrying the negotiated capability beyond the reset and creating the boot-time device tree accordingly, we start from scratch, generating the same boot-time device tree as we did prior to the CAS-generated and the same device tree updates as we did before. This could (in theory) cause us to get stuck in a reset loop. This hasn't been observed, but depending on the extensiveness of CAS-induced device tree updates in the future, could eventually become an issue. Address this by pulling capability-related device tree updates resulting from CAS calls into a common routine, spapr_dt_cas_updates(), and adding an sPAPROptionVector* parameter that allows us to test for newly-negotiated capabilities. We invoke it as follows: 1) When ibm,client-architecture-support gets called, we call spapr_dt_cas_updates() with the set of capabilities added since the previous call to ibm,client-architecture-support. For the initial boot, or a system reset generated by something other than the CAS call itself, this set will consist of *all* options supported both the platform and the guest. For calls to ibm,client-architecture-support immediately after a CAS-induced reset, we call spapr_dt_cas_updates() with only the set of capabilities added since the previous call, since the other capabilities will have already been addressed by the boot-time device-tree this time around. In the unlikely event that capabilities are *removed* since the previous CAS, we will generate a CAS-induced reset. In the unlikely event that we cannot fit the device-tree updates into the buffer provided by the guest, well generate a CAS-induced reset. 2) When a CAS update results in the need to reset the machine and include the updates in the boot-time device tree, we call the spapr_dt_cas_updates() using the full set of negotiated capabilities as part of the reset path. At initial boot, or after a reset generated by something other than the CAS call itself, this set will be empty, resulting in what should be the same boot-time device-tree as we generated prior to this patch. For CAS-induced reset, this routine will be called with the full set of capabilities negotiated by the platform/guest in the previous CAS call, which should result in CAS updates from previous call being accounted for in the initial boot-time device tree. Signed-off-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> [dwg: Changed an int -> bool conversion to be more explicit] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-10-25 06:47:29 +02:00
return ret;
}
hw/ppc: CAS reset on early device hotplug This patch is a follow up on the discussions made in patch "hw/ppc: disable hotplug before CAS is completed" that can be found at [1]. At this moment, we do not support CPU/memory hotplug in early boot stages, before CAS. When a hotplug occurs, the event is logged in an internal RTAS event log queue and an IRQ pulse is fired. In regular conditions, the guest handles the interrupt by executing check_exception, fetching the generated hotplug event and enabling the device for use. In early boot, this IRQ isn't caught (SLOF does not handle hotplug events), leaving the event in the rtas event log queue. If the guest executes check_exception due to another hotplug event, the re-assertion of the IRQ ends up de-queuing the first hotplug event as well. In short, a device hotplugged before CAS is considered coldplugged by SLOF. This leads to device misbehavior and, in some cases, guest kernel Ooops when trying to unplug the device. A proper fix would be to turn every device hotplugged before CAS as a colplugged device. This is not trivial to do with the current code base though - the FDT is written in the guest memory at ppc_spapr_reset and can't be retrieved without adding extra state (fdt_size for example) that will need to managed and migrated. Adding the hotplugged DT in the middle of CAS negotiation via the updated DT tree works with CPU devs, but panics the guest kernel at boot. Additional analysis would be necessary for LMBs and PCI devices. There are questions to be made in QEMU/SLOF/kernel level about how we can make this change in a sustainable way. With Linux guests, a fix would be the kernel executing check_exception at boot time, de-queueing the events that happened in early boot and processing them. However, even if/when the newer kernels start fetching these events at boot time, we need to take care of older kernels that won't be doing that. This patch works around the situation by issuing a CAS reset if a hotplugged device is detected during CAS: - the DRC conditions that warrant a CAS reset is the same as those that triggers a DRC migration - the DRC must have a device attached and the DRC state is not equal to its ready_state. With that in mind, this patch makes use of 'spapr_drc_needed' to determine if a CAS reset is needed. - In the middle of CAS negotiations, the function 'spapr_hotplugged_dev_before_cas' goes through all the DRCs to see if there are any DRC that requires a reset, using spapr_drc_needed. If that happens, returns '1' in 'spapr_h_cas_compose_response' which will set spapr->cas_reboot to true, causing the machine to reboot. No changes are made for coldplug devices. [1] http://lists.nongnu.org/archive/html/qemu-devel/2017-08/msg02855.html Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-08-30 20:21:41 +02:00
static bool spapr_hotplugged_dev_before_cas(void)
{
Object *drc_container, *obj;
ObjectProperty *prop;
ObjectPropertyIterator iter;
drc_container = container_get(object_get_root(), "/dr-connector");
object_property_iter_init(&iter, drc_container);
while ((prop = object_property_iter_next(&iter))) {
if (!strstart(prop->type, "link<", NULL)) {
continue;
}
obj = object_property_get_link(drc_container, prop->name, NULL);
if (spapr_drc_needed(obj)) {
return true;
}
}
return false;
}
int spapr_h_cas_compose_response(sPAPRMachineState *spapr,
target_ulong addr, target_ulong size,
spapr: add option vector handling in CAS-generated resets In some cases, ibm,client-architecture-support calls can fail. This could happen in the current code for situations where the modified device tree segment exceeds the buffer size provided by the guest via the call parameters. In these cases, QEMU will reset, allowing an opportunity to regenerate the device tree from scratch via boot-time handling. There are potentially other scenarios as well, not currently reachable in the current code, but possible in theory, such as cases where device-tree properties or nodes need to be removed. We currently don't handle either of these properly for option vector capabilities however. Instead of carrying the negotiated capability beyond the reset and creating the boot-time device tree accordingly, we start from scratch, generating the same boot-time device tree as we did prior to the CAS-generated and the same device tree updates as we did before. This could (in theory) cause us to get stuck in a reset loop. This hasn't been observed, but depending on the extensiveness of CAS-induced device tree updates in the future, could eventually become an issue. Address this by pulling capability-related device tree updates resulting from CAS calls into a common routine, spapr_dt_cas_updates(), and adding an sPAPROptionVector* parameter that allows us to test for newly-negotiated capabilities. We invoke it as follows: 1) When ibm,client-architecture-support gets called, we call spapr_dt_cas_updates() with the set of capabilities added since the previous call to ibm,client-architecture-support. For the initial boot, or a system reset generated by something other than the CAS call itself, this set will consist of *all* options supported both the platform and the guest. For calls to ibm,client-architecture-support immediately after a CAS-induced reset, we call spapr_dt_cas_updates() with only the set of capabilities added since the previous call, since the other capabilities will have already been addressed by the boot-time device-tree this time around. In the unlikely event that capabilities are *removed* since the previous CAS, we will generate a CAS-induced reset. In the unlikely event that we cannot fit the device-tree updates into the buffer provided by the guest, well generate a CAS-induced reset. 2) When a CAS update results in the need to reset the machine and include the updates in the boot-time device tree, we call the spapr_dt_cas_updates() using the full set of negotiated capabilities as part of the reset path. At initial boot, or after a reset generated by something other than the CAS call itself, this set will be empty, resulting in what should be the same boot-time device-tree as we generated prior to this patch. For CAS-induced reset, this routine will be called with the full set of capabilities negotiated by the platform/guest in the previous CAS call, which should result in CAS updates from previous call being accounted for in the initial boot-time device tree. Signed-off-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> [dwg: Changed an int -> bool conversion to be more explicit] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-10-25 06:47:29 +02:00
sPAPROptionVector *ov5_updates)
{
void *fdt, *fdt_skel;
sPAPRDeviceTreeUpdateHeader hdr = { .version_id = 1 };
hw/ppc: CAS reset on early device hotplug This patch is a follow up on the discussions made in patch "hw/ppc: disable hotplug before CAS is completed" that can be found at [1]. At this moment, we do not support CPU/memory hotplug in early boot stages, before CAS. When a hotplug occurs, the event is logged in an internal RTAS event log queue and an IRQ pulse is fired. In regular conditions, the guest handles the interrupt by executing check_exception, fetching the generated hotplug event and enabling the device for use. In early boot, this IRQ isn't caught (SLOF does not handle hotplug events), leaving the event in the rtas event log queue. If the guest executes check_exception due to another hotplug event, the re-assertion of the IRQ ends up de-queuing the first hotplug event as well. In short, a device hotplugged before CAS is considered coldplugged by SLOF. This leads to device misbehavior and, in some cases, guest kernel Ooops when trying to unplug the device. A proper fix would be to turn every device hotplugged before CAS as a colplugged device. This is not trivial to do with the current code base though - the FDT is written in the guest memory at ppc_spapr_reset and can't be retrieved without adding extra state (fdt_size for example) that will need to managed and migrated. Adding the hotplugged DT in the middle of CAS negotiation via the updated DT tree works with CPU devs, but panics the guest kernel at boot. Additional analysis would be necessary for LMBs and PCI devices. There are questions to be made in QEMU/SLOF/kernel level about how we can make this change in a sustainable way. With Linux guests, a fix would be the kernel executing check_exception at boot time, de-queueing the events that happened in early boot and processing them. However, even if/when the newer kernels start fetching these events at boot time, we need to take care of older kernels that won't be doing that. This patch works around the situation by issuing a CAS reset if a hotplugged device is detected during CAS: - the DRC conditions that warrant a CAS reset is the same as those that triggers a DRC migration - the DRC must have a device attached and the DRC state is not equal to its ready_state. With that in mind, this patch makes use of 'spapr_drc_needed' to determine if a CAS reset is needed. - In the middle of CAS negotiations, the function 'spapr_hotplugged_dev_before_cas' goes through all the DRCs to see if there are any DRC that requires a reset, using spapr_drc_needed. If that happens, returns '1' in 'spapr_h_cas_compose_response' which will set spapr->cas_reboot to true, causing the machine to reboot. No changes are made for coldplug devices. [1] http://lists.nongnu.org/archive/html/qemu-devel/2017-08/msg02855.html Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-08-30 20:21:41 +02:00
if (spapr_hotplugged_dev_before_cas()) {
return 1;
}
if (size < sizeof(hdr) || size > FW_MAX_SIZE) {
error_report("SLOF provided an unexpected CAS buffer size "
TARGET_FMT_lu " (min: %zu, max: %u)",
size, sizeof(hdr), FW_MAX_SIZE);
exit(EXIT_FAILURE);
}
size -= sizeof(hdr);
hw/ppc: CAS reset on early device hotplug This patch is a follow up on the discussions made in patch "hw/ppc: disable hotplug before CAS is completed" that can be found at [1]. At this moment, we do not support CPU/memory hotplug in early boot stages, before CAS. When a hotplug occurs, the event is logged in an internal RTAS event log queue and an IRQ pulse is fired. In regular conditions, the guest handles the interrupt by executing check_exception, fetching the generated hotplug event and enabling the device for use. In early boot, this IRQ isn't caught (SLOF does not handle hotplug events), leaving the event in the rtas event log queue. If the guest executes check_exception due to another hotplug event, the re-assertion of the IRQ ends up de-queuing the first hotplug event as well. In short, a device hotplugged before CAS is considered coldplugged by SLOF. This leads to device misbehavior and, in some cases, guest kernel Ooops when trying to unplug the device. A proper fix would be to turn every device hotplugged before CAS as a colplugged device. This is not trivial to do with the current code base though - the FDT is written in the guest memory at ppc_spapr_reset and can't be retrieved without adding extra state (fdt_size for example) that will need to managed and migrated. Adding the hotplugged DT in the middle of CAS negotiation via the updated DT tree works with CPU devs, but panics the guest kernel at boot. Additional analysis would be necessary for LMBs and PCI devices. There are questions to be made in QEMU/SLOF/kernel level about how we can make this change in a sustainable way. With Linux guests, a fix would be the kernel executing check_exception at boot time, de-queueing the events that happened in early boot and processing them. However, even if/when the newer kernels start fetching these events at boot time, we need to take care of older kernels that won't be doing that. This patch works around the situation by issuing a CAS reset if a hotplugged device is detected during CAS: - the DRC conditions that warrant a CAS reset is the same as those that triggers a DRC migration - the DRC must have a device attached and the DRC state is not equal to its ready_state. With that in mind, this patch makes use of 'spapr_drc_needed' to determine if a CAS reset is needed. - In the middle of CAS negotiations, the function 'spapr_hotplugged_dev_before_cas' goes through all the DRCs to see if there are any DRC that requires a reset, using spapr_drc_needed. If that happens, returns '1' in 'spapr_h_cas_compose_response' which will set spapr->cas_reboot to true, causing the machine to reboot. No changes are made for coldplug devices. [1] http://lists.nongnu.org/archive/html/qemu-devel/2017-08/msg02855.html Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-08-30 20:21:41 +02:00
/* Create skeleton */
fdt_skel = g_malloc0(size);
_FDT((fdt_create(fdt_skel, size)));
_FDT((fdt_finish_reservemap(fdt_skel)));
_FDT((fdt_begin_node(fdt_skel, "")));
_FDT((fdt_end_node(fdt_skel)));
_FDT((fdt_finish(fdt_skel)));
fdt = g_malloc0(size);
_FDT((fdt_open_into(fdt_skel, fdt, size)));
g_free(fdt_skel);
/* Fixup cpu nodes */
_FDT((spapr_fixup_cpu_dt(fdt, spapr)));
spapr: add option vector handling in CAS-generated resets In some cases, ibm,client-architecture-support calls can fail. This could happen in the current code for situations where the modified device tree segment exceeds the buffer size provided by the guest via the call parameters. In these cases, QEMU will reset, allowing an opportunity to regenerate the device tree from scratch via boot-time handling. There are potentially other scenarios as well, not currently reachable in the current code, but possible in theory, such as cases where device-tree properties or nodes need to be removed. We currently don't handle either of these properly for option vector capabilities however. Instead of carrying the negotiated capability beyond the reset and creating the boot-time device tree accordingly, we start from scratch, generating the same boot-time device tree as we did prior to the CAS-generated and the same device tree updates as we did before. This could (in theory) cause us to get stuck in a reset loop. This hasn't been observed, but depending on the extensiveness of CAS-induced device tree updates in the future, could eventually become an issue. Address this by pulling capability-related device tree updates resulting from CAS calls into a common routine, spapr_dt_cas_updates(), and adding an sPAPROptionVector* parameter that allows us to test for newly-negotiated capabilities. We invoke it as follows: 1) When ibm,client-architecture-support gets called, we call spapr_dt_cas_updates() with the set of capabilities added since the previous call to ibm,client-architecture-support. For the initial boot, or a system reset generated by something other than the CAS call itself, this set will consist of *all* options supported both the platform and the guest. For calls to ibm,client-architecture-support immediately after a CAS-induced reset, we call spapr_dt_cas_updates() with only the set of capabilities added since the previous call, since the other capabilities will have already been addressed by the boot-time device-tree this time around. In the unlikely event that capabilities are *removed* since the previous CAS, we will generate a CAS-induced reset. In the unlikely event that we cannot fit the device-tree updates into the buffer provided by the guest, well generate a CAS-induced reset. 2) When a CAS update results in the need to reset the machine and include the updates in the boot-time device tree, we call the spapr_dt_cas_updates() using the full set of negotiated capabilities as part of the reset path. At initial boot, or after a reset generated by something other than the CAS call itself, this set will be empty, resulting in what should be the same boot-time device-tree as we generated prior to this patch. For CAS-induced reset, this routine will be called with the full set of capabilities negotiated by the platform/guest in the previous CAS call, which should result in CAS updates from previous call being accounted for in the initial boot-time device tree. Signed-off-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> [dwg: Changed an int -> bool conversion to be more explicit] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-10-25 06:47:29 +02:00
if (spapr_dt_cas_updates(spapr, fdt, ov5_updates)) {
return -1;
}
/* Pack resulting tree */
_FDT((fdt_pack(fdt)));
if (fdt_totalsize(fdt) + sizeof(hdr) > size) {
trace_spapr_cas_failed(size);
return -1;
}
cpu_physical_memory_write(addr, &hdr, sizeof(hdr));
cpu_physical_memory_write(addr + sizeof(hdr), fdt, fdt_totalsize(fdt));
trace_spapr_cas_continue(fdt_totalsize(fdt) + sizeof(hdr));
g_free(fdt);
return 0;
}
static void spapr_dt_rtas(sPAPRMachineState *spapr, void *fdt)
{
int rtas;
GString *hypertas = g_string_sized_new(256);
GString *qemu_hypertas = g_string_sized_new(256);
uint32_t refpoints[] = { cpu_to_be32(0x4), cpu_to_be32(0x4) };
uint64_t max_device_addr = MACHINE(spapr)->device_memory->base +
memory_region_size(&MACHINE(spapr)->device_memory->mr);
uint32_t lrdr_capacity[] = {
cpu_to_be32(max_device_addr >> 32),
cpu_to_be32(max_device_addr & 0xffffffff),
0, cpu_to_be32(SPAPR_MEMORY_BLOCK_SIZE),
cpu_to_be32(max_cpus / smp_threads),
};
uint32_t maxdomains[] = {
cpu_to_be32(4),
cpu_to_be32(0),
cpu_to_be32(0),
cpu_to_be32(0),
spapr: Fix ibm,max-associativity-domains property number of nodes Laurent Vivier reported off by one with maximum number of NUMA nodes provided by qemu-kvm being less by one than required according to description of "ibm,max-associativity-domains" property in LoPAPR. It appears that I incorrectly treated LoPAPR description of this property assuming it provides last valid domain (NUMA node here) instead of maximum number of domains. ### Before hot-add (qemu) info numa 3 nodes node 0 cpus: 0 node 0 size: 0 MB node 0 plugged: 0 MB node 1 cpus: node 1 size: 1024 MB node 1 plugged: 0 MB node 2 cpus: node 2 size: 0 MB node 2 plugged: 0 MB $ numactl -H available: 2 nodes (0-1) node 0 cpus: 0 node 0 size: 0 MB node 0 free: 0 MB node 1 cpus: node 1 size: 999 MB node 1 free: 658 MB node distances: node 0 1 0: 10 40 1: 40 10 ### Hot-add (qemu) object_add memory-backend-ram,id=mem0,size=1G (qemu) device_add pc-dimm,id=dimm1,memdev=mem0,node=2 (qemu) [ 87.704898] pseries-hotplug-mem: Attempting to hot-add 4 ... <there is no "Initmem setup node 2 [mem 0xHEX-0xHEX]"> [ 87.705128] lpar: Attempting to resize HPT to shift 21 ... <HPT resize messages> ### After hot-add (qemu) info numa 3 nodes node 0 cpus: 0 node 0 size: 0 MB node 0 plugged: 0 MB node 1 cpus: node 1 size: 1024 MB node 1 plugged: 0 MB node 2 cpus: node 2 size: 1024 MB node 2 plugged: 1024 MB $ numactl -H available: 2 nodes (0-1) ^^^^^^^^^^^^^^^^^^^^^^^^ Still only two nodes (and memory hot-added to node 0 below) node 0 cpus: 0 node 0 size: 1024 MB node 0 free: 1021 MB node 1 cpus: node 1 size: 999 MB node 1 free: 658 MB node distances: node 0 1 0: 10 40 1: 40 10 After fix applied numactl(8) reports 3 nodes available and memory plugged into node 2 as expected. From David Gibson: ------------------ Qemu makes a distinction between "non NUMA" (nb_numa_nodes == 0) and "NUMA with one node" (nb_numa_nodes == 1). But from a PAPR guests's point of view these are equivalent. I don't want to present two different cases to the guest when we don't need to, so even though the guest can handle it, I'd prefer we put a '1' here for both the nb_numa_nodes == 0 and nb_numa_nodes == 1 case. This consolidates everything discussed previously on mailing list. Fixes: da9f80fbad21 ("spapr: Add ibm,max-associativity-domains property") Reported-by: Laurent Vivier <lvivier@redhat.com> Signed-off-by: Serhii Popovych <spopovyc@redhat.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Greg Kurz <groug@kaod.org> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2018-11-22 14:19:27 +01:00
cpu_to_be32(nb_numa_nodes ? nb_numa_nodes : 1),
};
_FDT(rtas = fdt_add_subnode(fdt, 0, "rtas"));
/* hypertas */
add_str(hypertas, "hcall-pft");
add_str(hypertas, "hcall-term");
add_str(hypertas, "hcall-dabr");
add_str(hypertas, "hcall-interrupt");
add_str(hypertas, "hcall-tce");
add_str(hypertas, "hcall-vio");
add_str(hypertas, "hcall-splpar");
add_str(hypertas, "hcall-bulk");
add_str(hypertas, "hcall-set-mode");
add_str(hypertas, "hcall-sprg0");
add_str(hypertas, "hcall-copy");
add_str(hypertas, "hcall-debug");
add_str(hypertas, "hcall-vphn");
add_str(qemu_hypertas, "hcall-memop1");
if (!kvm_enabled() || kvmppc_spapr_use_multitce()) {
add_str(hypertas, "hcall-multi-tce");
}
if (spapr->resize_hpt != SPAPR_RESIZE_HPT_DISABLED) {
add_str(hypertas, "hcall-hpt-resize");
}
_FDT(fdt_setprop(fdt, rtas, "ibm,hypertas-functions",
hypertas->str, hypertas->len));
g_string_free(hypertas, TRUE);
_FDT(fdt_setprop(fdt, rtas, "qemu,hypertas-functions",
qemu_hypertas->str, qemu_hypertas->len));
g_string_free(qemu_hypertas, TRUE);
_FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points",
refpoints, sizeof(refpoints)));
_FDT(fdt_setprop(fdt, rtas, "ibm,max-associativity-domains",
maxdomains, sizeof(maxdomains)));
_FDT(fdt_setprop_cell(fdt, rtas, "rtas-error-log-max",
RTAS_ERROR_LOG_MAX));
_FDT(fdt_setprop_cell(fdt, rtas, "rtas-event-scan-rate",
RTAS_EVENT_SCAN_RATE));
g_assert(msi_nonbroken);
_FDT(fdt_setprop(fdt, rtas, "ibm,change-msix-capable", NULL, 0));
/*
* According to PAPR, rtas ibm,os-term does not guarantee a return
* back to the guest cpu.
*
* While an additional ibm,extended-os-term property indicates
* that rtas call return will always occur. Set this property.
*/
_FDT(fdt_setprop(fdt, rtas, "ibm,extended-os-term", NULL, 0));
_FDT(fdt_setprop(fdt, rtas, "ibm,lrdr-capacity",
lrdr_capacity, sizeof(lrdr_capacity)));
spapr_dt_rtas_tokens(fdt, rtas);
}
/*
* Prepare ibm,arch-vec-5-platform-support, which indicates the MMU
* and the XIVE features that the guest may request and thus the valid
* values for bytes 23..26 of option vector 5:
*/
static void spapr_dt_ov5_platform_support(sPAPRMachineState *spapr, void *fdt,
int chosen)
{
PowerPCCPU *first_ppc_cpu = POWERPC_CPU(first_cpu);
char val[2 * 4] = {
23, spapr->irq->ov5, /* Xive mode. */
24, 0x00, /* Hash/Radix, filled in below. */
25, 0x00, /* Hash options: Segment Tables == no, GTSE == no. */
26, 0x40, /* Radix options: GTSE == yes. */
};
target/ppc: Update setting of cpu features to account for compat modes The device tree nodes ibm,arch-vec-5-platform-support and ibm,pa-features are used to communicate features of the cpu to the guest operating system. The properties of each of these are determined based on the selected cpu model and the availability of hypervisor features. Currently the compatibility mode of the cpu is not taken into account. The ibm,arch-vec-5-platform-support node is used to communicate the level of support for various ISAv3 processor features to the guest before CAS to inform the guests' request. The available mmu mode should only be hash unless the cpu is a POWER9 which is not in a prePOWER9 compat mode, in which case the available modes depend on the accelerator and the hypervisor capabilities. The ibm,pa-featues node is used to communicate the level of cpu support for various features to the guest os. This should only contain features relevant to the operating mode of the processor, that is the selected cpu model taking into account any compat mode. This means that the compat mode should be taken into account when choosing the properties of ibm,pa-features and they should match the compat mode selected, or the cpu model selected if no compat mode. Update the setting of these cpu features in the device tree as described above to properly take into account any compat mode. We use the ppc_check_compat function which takes into account the current processor model and the cpu compat mode. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-11-17 06:39:00 +01:00
if (!ppc_check_compat(first_ppc_cpu, CPU_POWERPC_LOGICAL_3_00, 0,
first_ppc_cpu->compat_pvr)) {
/*
* If we're in a pre POWER9 compat mode then the guest should
* do hash and use the legacy interrupt mode
*/
val[1] = 0x00; /* XICS */
target/ppc: Update setting of cpu features to account for compat modes The device tree nodes ibm,arch-vec-5-platform-support and ibm,pa-features are used to communicate features of the cpu to the guest operating system. The properties of each of these are determined based on the selected cpu model and the availability of hypervisor features. Currently the compatibility mode of the cpu is not taken into account. The ibm,arch-vec-5-platform-support node is used to communicate the level of support for various ISAv3 processor features to the guest before CAS to inform the guests' request. The available mmu mode should only be hash unless the cpu is a POWER9 which is not in a prePOWER9 compat mode, in which case the available modes depend on the accelerator and the hypervisor capabilities. The ibm,pa-featues node is used to communicate the level of cpu support for various features to the guest os. This should only contain features relevant to the operating mode of the processor, that is the selected cpu model taking into account any compat mode. This means that the compat mode should be taken into account when choosing the properties of ibm,pa-features and they should match the compat mode selected, or the cpu model selected if no compat mode. Update the setting of these cpu features in the device tree as described above to properly take into account any compat mode. We use the ppc_check_compat function which takes into account the current processor model and the cpu compat mode. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-11-17 06:39:00 +01:00
val[3] = 0x00; /* Hash */
} else if (kvm_enabled()) {
if (kvmppc_has_cap_mmu_radix() && kvmppc_has_cap_mmu_hash_v3()) {
val[3] = 0x80; /* OV5_MMU_BOTH */
} else if (kvmppc_has_cap_mmu_radix()) {
val[3] = 0x40; /* OV5_MMU_RADIX_300 */
} else {
val[3] = 0x00; /* Hash */
}
} else {
target/ppc: Update setting of cpu features to account for compat modes The device tree nodes ibm,arch-vec-5-platform-support and ibm,pa-features are used to communicate features of the cpu to the guest operating system. The properties of each of these are determined based on the selected cpu model and the availability of hypervisor features. Currently the compatibility mode of the cpu is not taken into account. The ibm,arch-vec-5-platform-support node is used to communicate the level of support for various ISAv3 processor features to the guest before CAS to inform the guests' request. The available mmu mode should only be hash unless the cpu is a POWER9 which is not in a prePOWER9 compat mode, in which case the available modes depend on the accelerator and the hypervisor capabilities. The ibm,pa-featues node is used to communicate the level of cpu support for various features to the guest os. This should only contain features relevant to the operating mode of the processor, that is the selected cpu model taking into account any compat mode. This means that the compat mode should be taken into account when choosing the properties of ibm,pa-features and they should match the compat mode selected, or the cpu model selected if no compat mode. Update the setting of these cpu features in the device tree as described above to properly take into account any compat mode. We use the ppc_check_compat function which takes into account the current processor model and the cpu compat mode. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-11-17 06:39:00 +01:00
/* V3 MMU supports both hash and radix in tcg (with dynamic switching) */
val[3] = 0xC0;
}
_FDT(fdt_setprop(fdt, chosen, "ibm,arch-vec-5-platform-support",
val, sizeof(val)));
}
static void spapr_dt_chosen(sPAPRMachineState *spapr, void *fdt)
{
MachineState *machine = MACHINE(spapr);
int chosen;
const char *boot_device = machine->boot_order;
char *stdout_path = spapr_vio_stdout_path(spapr->vio_bus);
size_t cb = 0;
char *bootlist = get_boot_devices_list(&cb);
_FDT(chosen = fdt_add_subnode(fdt, 0, "chosen"));
_FDT(fdt_setprop_string(fdt, chosen, "bootargs", machine->kernel_cmdline));
_FDT(fdt_setprop_cell(fdt, chosen, "linux,initrd-start",
spapr->initrd_base));
_FDT(fdt_setprop_cell(fdt, chosen, "linux,initrd-end",
spapr->initrd_base + spapr->initrd_size));
if (spapr->kernel_size) {
uint64_t kprop[2] = { cpu_to_be64(KERNEL_LOAD_ADDR),
cpu_to_be64(spapr->kernel_size) };
_FDT(fdt_setprop(fdt, chosen, "qemu,boot-kernel",
&kprop, sizeof(kprop)));
if (spapr->kernel_le) {
_FDT(fdt_setprop(fdt, chosen, "qemu,boot-kernel-le", NULL, 0));
}
}
if (boot_menu) {
_FDT((fdt_setprop_cell(fdt, chosen, "qemu,boot-menu", boot_menu)));
}
_FDT(fdt_setprop_cell(fdt, chosen, "qemu,graphic-width", graphic_width));
_FDT(fdt_setprop_cell(fdt, chosen, "qemu,graphic-height", graphic_height));
_FDT(fdt_setprop_cell(fdt, chosen, "qemu,graphic-depth", graphic_depth));
if (cb && bootlist) {
int i;
for (i = 0; i < cb; i++) {
if (bootlist[i] == '\n') {
bootlist[i] = ' ';
}
}
_FDT(fdt_setprop_string(fdt, chosen, "qemu,boot-list", bootlist));
}
if (boot_device && strlen(boot_device)) {
_FDT(fdt_setprop_string(fdt, chosen, "qemu,boot-device", boot_device));
}
if (!spapr->has_graphics && stdout_path) {
/*
* "linux,stdout-path" and "stdout" properties are deprecated by linux
* kernel. New platforms should only use the "stdout-path" property. Set
* the new property and continue using older property to remain
* compatible with the existing firmware.
*/
_FDT(fdt_setprop_string(fdt, chosen, "linux,stdout-path", stdout_path));
_FDT(fdt_setprop_string(fdt, chosen, "stdout-path", stdout_path));
}
spapr_dt_ov5_platform_support(spapr, fdt, chosen);
g_free(stdout_path);
g_free(bootlist);
}
static void spapr_dt_hypervisor(sPAPRMachineState *spapr, void *fdt)
{
/* The /hypervisor node isn't in PAPR - this is a hack to allow PR
* KVM to work under pHyp with some guest co-operation */
int hypervisor;
uint8_t hypercall[16];
_FDT(hypervisor = fdt_add_subnode(fdt, 0, "hypervisor"));
/* indicate KVM hypercall interface */
_FDT(fdt_setprop_string(fdt, hypervisor, "compatible", "linux,kvm"));
if (kvmppc_has_cap_fixup_hcalls()) {
/*
* Older KVM versions with older guest kernels were broken
* with the magic page, don't allow the guest to map it.
*/
if (!kvmppc_get_hypercall(first_cpu->env_ptr, hypercall,
sizeof(hypercall))) {
_FDT(fdt_setprop(fdt, hypervisor, "hcall-instructions",
hypercall, sizeof(hypercall)));
}
}
}
static void *spapr_build_fdt(sPAPRMachineState *spapr)
{
MachineState *machine = MACHINE(spapr);
MachineClass *mc = MACHINE_GET_CLASS(machine);
sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine);
int ret;
void *fdt;
sPAPRPHBState *phb;
char *buf;
fdt = g_malloc0(FDT_MAX_SIZE);
_FDT((fdt_create_empty_tree(fdt, FDT_MAX_SIZE)));
/* Root node */
_FDT(fdt_setprop_string(fdt, 0, "device_type", "chrp"));
_FDT(fdt_setprop_string(fdt, 0, "model", "IBM pSeries (emulated by qemu)"));
_FDT(fdt_setprop_string(fdt, 0, "compatible", "qemu,pseries"));
/*
* Add info to guest to indentify which host is it being run on
* and what is the uuid of the guest
*/
if (spapr->host_model && !g_str_equal(spapr->host_model, "none")) {
if (g_str_equal(spapr->host_model, "passthrough")) {
/* -M host-model=passthrough */
if (kvmppc_get_host_model(&buf)) {
_FDT(fdt_setprop_string(fdt, 0, "host-model", buf));
g_free(buf);
}
} else {
/* -M host-model=<user-string> */
_FDT(fdt_setprop_string(fdt, 0, "host-model", spapr->host_model));
}
}
if (spapr->host_serial && !g_str_equal(spapr->host_serial, "none")) {
if (g_str_equal(spapr->host_serial, "passthrough")) {
/* -M host-serial=passthrough */
if (kvmppc_get_host_serial(&buf)) {
_FDT(fdt_setprop_string(fdt, 0, "host-serial", buf));
g_free(buf);
}
} else {
/* -M host-serial=<user-string> */
_FDT(fdt_setprop_string(fdt, 0, "host-serial", spapr->host_serial));
}
}
buf = qemu_uuid_unparse_strdup(&qemu_uuid);
_FDT(fdt_setprop_string(fdt, 0, "vm,uuid", buf));
if (qemu_uuid_set) {
_FDT(fdt_setprop_string(fdt, 0, "system-id", buf));
}
g_free(buf);
if (qemu_get_vm_name()) {
_FDT(fdt_setprop_string(fdt, 0, "ibm,partition-name",
qemu_get_vm_name()));
}
_FDT(fdt_setprop_cell(fdt, 0, "#address-cells", 2));
_FDT(fdt_setprop_cell(fdt, 0, "#size-cells", 2));
/* /interrupt controller */
spapr->irq->dt_populate(spapr, spapr_max_server_number(spapr), fdt,
PHANDLE_INTC);
ret = spapr_populate_memory(spapr, fdt);
if (ret < 0) {
error_report("couldn't setup memory nodes in fdt");
exit(1);
}
/* /vdevice */
spapr_dt_vdevice(spapr->vio_bus, fdt);
if (object_resolve_path_type("", TYPE_SPAPR_RNG, NULL)) {
ret = spapr_rng_populate_dt(fdt);
if (ret < 0) {
error_report("could not set up rng device in the fdt");
exit(1);
}
}
QLIST_FOREACH(phb, &spapr->phbs, list) {
ret = spapr_populate_pci_dt(phb, PHANDLE_INTC, fdt,
spapr->irq->nr_msis, NULL);
if (ret < 0) {
error_report("couldn't setup PCI devices in fdt");
exit(1);
}
}
/* cpus */
spapr_populate_cpus_dt_node(fdt, spapr);
if (smc->dr_lmb_enabled) {
_FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_LMB));
}
if (mc->has_hotpluggable_cpus) {
int offset = fdt_path_offset(fdt, "/cpus");
ret = spapr_drc_populate_dt(fdt, offset, NULL,
SPAPR_DR_CONNECTOR_TYPE_CPU);
if (ret < 0) {
error_report("Couldn't set up CPU DR device tree properties");
exit(1);
}
}
/* /event-sources */
spapr_dt_events(spapr, fdt);
/* /rtas */
spapr_dt_rtas(spapr, fdt);
/* /chosen */
spapr_dt_chosen(spapr, fdt);
/* /hypervisor */
if (kvm_enabled()) {
spapr_dt_hypervisor(spapr, fdt);
}
/* Build memory reserve map */
if (spapr->kernel_size) {
_FDT((fdt_add_mem_rsv(fdt, KERNEL_LOAD_ADDR, spapr->kernel_size)));
}
if (spapr->initrd_size) {
_FDT((fdt_add_mem_rsv(fdt, spapr->initrd_base, spapr->initrd_size)));
}
spapr: add option vector handling in CAS-generated resets In some cases, ibm,client-architecture-support calls can fail. This could happen in the current code for situations where the modified device tree segment exceeds the buffer size provided by the guest via the call parameters. In these cases, QEMU will reset, allowing an opportunity to regenerate the device tree from scratch via boot-time handling. There are potentially other scenarios as well, not currently reachable in the current code, but possible in theory, such as cases where device-tree properties or nodes need to be removed. We currently don't handle either of these properly for option vector capabilities however. Instead of carrying the negotiated capability beyond the reset and creating the boot-time device tree accordingly, we start from scratch, generating the same boot-time device tree as we did prior to the CAS-generated and the same device tree updates as we did before. This could (in theory) cause us to get stuck in a reset loop. This hasn't been observed, but depending on the extensiveness of CAS-induced device tree updates in the future, could eventually become an issue. Address this by pulling capability-related device tree updates resulting from CAS calls into a common routine, spapr_dt_cas_updates(), and adding an sPAPROptionVector* parameter that allows us to test for newly-negotiated capabilities. We invoke it as follows: 1) When ibm,client-architecture-support gets called, we call spapr_dt_cas_updates() with the set of capabilities added since the previous call to ibm,client-architecture-support. For the initial boot, or a system reset generated by something other than the CAS call itself, this set will consist of *all* options supported both the platform and the guest. For calls to ibm,client-architecture-support immediately after a CAS-induced reset, we call spapr_dt_cas_updates() with only the set of capabilities added since the previous call, since the other capabilities will have already been addressed by the boot-time device-tree this time around. In the unlikely event that capabilities are *removed* since the previous CAS, we will generate a CAS-induced reset. In the unlikely event that we cannot fit the device-tree updates into the buffer provided by the guest, well generate a CAS-induced reset. 2) When a CAS update results in the need to reset the machine and include the updates in the boot-time device tree, we call the spapr_dt_cas_updates() using the full set of negotiated capabilities as part of the reset path. At initial boot, or after a reset generated by something other than the CAS call itself, this set will be empty, resulting in what should be the same boot-time device-tree as we generated prior to this patch. For CAS-induced reset, this routine will be called with the full set of capabilities negotiated by the platform/guest in the previous CAS call, which should result in CAS updates from previous call being accounted for in the initial boot-time device tree. Signed-off-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> [dwg: Changed an int -> bool conversion to be more explicit] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-10-25 06:47:29 +02:00
/* ibm,client-architecture-support updates */
ret = spapr_dt_cas_updates(spapr, fdt, spapr->ov5_cas);
if (ret < 0) {
error_report("couldn't setup CAS properties fdt");
exit(1);
}
if (smc->dr_phb_enabled) {
ret = spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_PHB);
if (ret < 0) {
error_report("Couldn't set up PHB DR device tree properties");
exit(1);
}
}
return fdt;
}
static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
{
return (addr & 0x0fffffff) + KERNEL_LOAD_ADDR;
}
static void emulate_spapr_hypercall(PPCVirtualHypervisor *vhyp,
PowerPCCPU *cpu)
{
CPUPPCState *env = &cpu->env;
tcg: drop global lock during TCG code execution This finally allows TCG to benefit from the iothread introduction: Drop the global mutex while running pure TCG CPU code. Reacquire the lock when entering MMIO or PIO emulation, or when leaving the TCG loop. We have to revert a few optimization for the current TCG threading model, namely kicking the TCG thread in qemu_mutex_lock_iothread and not kicking it in qemu_cpu_kick. We also need to disable RAM block reordering until we have a more efficient locking mechanism at hand. Still, a Linux x86 UP guest and my Musicpal ARM model boot fine here. These numbers demonstrate where we gain something: 20338 jan 20 0 331m 75m 6904 R 99 0.9 0:50.95 qemu-system-arm 20337 jan 20 0 331m 75m 6904 S 20 0.9 0:26.50 qemu-system-arm The guest CPU was fully loaded, but the iothread could still run mostly independent on a second core. Without the patch we don't get beyond 32206 jan 20 0 330m 73m 7036 R 82 0.9 1:06.00 qemu-system-arm 32204 jan 20 0 330m 73m 7036 S 21 0.9 0:17.03 qemu-system-arm We don't benefit significantly, though, when the guest is not fully loading a host CPU. Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> Message-Id: <1439220437-23957-10-git-send-email-fred.konrad@greensocs.com> [FK: Rebase, fix qemu_devices_reset deadlock, rm address_space_* mutex] Signed-off-by: KONRAD Frederic <fred.konrad@greensocs.com> [EGC: fixed iothread lock for cpu-exec IRQ handling] Signed-off-by: Emilio G. Cota <cota@braap.org> [AJB: -smp single-threaded fix, clean commit msg, BQL fixes] Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <rth@twiddle.net> Reviewed-by: Pranith Kumar <bobby.prani@gmail.com> [PM: target-arm changes] Acked-by: Peter Maydell <peter.maydell@linaro.org>
2017-02-23 19:29:11 +01:00
/* The TCG path should also be holding the BQL at this point */
g_assert(qemu_mutex_iothread_locked());
if (msr_pr) {
hcall_dprintf("Hypercall made with MSR[PR]=1\n");
env->gpr[3] = H_PRIVILEGE;
} else {
env->gpr[3] = spapr_hypercall(cpu, env->gpr[3], &env->gpr[4]);
}
}
struct LPCRSyncState {
target_ulong value;
target_ulong mask;
};
static void do_lpcr_sync(CPUState *cs, run_on_cpu_data arg)
{
struct LPCRSyncState *s = arg.host_ptr;
PowerPCCPU *cpu = POWERPC_CPU(cs);
CPUPPCState *env = &cpu->env;
target_ulong lpcr;
cpu_synchronize_state(cs);
lpcr = env->spr[SPR_LPCR];
lpcr &= ~s->mask;
lpcr |= s->value;
ppc_store_lpcr(cpu, lpcr);
}
void spapr_set_all_lpcrs(target_ulong value, target_ulong mask)
{
CPUState *cs;
struct LPCRSyncState s = {
.value = value,
.mask = mask
};
CPU_FOREACH(cs) {
run_on_cpu(cs, do_lpcr_sync, RUN_ON_CPU_HOST_PTR(&s));
}
}
static void spapr_get_pate(PPCVirtualHypervisor *vhyp, ppc_v3_pate_t *entry)
target/ppc: Add patb_entry to sPAPRMachineState ISA v3.00 adds the idea of a partition table which is used to store the address translation details for all partitions on the system. The partition table consists of double word entries indexed by partition id where the second double word contains the location of the process table in guest memory. The process table is registered by the guest via a h-call. We need somewhere to store the address of the process table so we add an entry to the sPAPRMachineState struct called patb_entry to represent the second doubleword of a single partition table entry corresponding to the current guest. We need to store this value so we know if the guest is using radix or hash translation and the location of the corresponding process table in guest memory. Since we only have a single guest per qemu instance, we only need one entry. Since the partition table is technically a hypervisor resource we require that access to it is abstracted by the virtual hypervisor through the get_patbe() call. Currently the value of the entry is never set (and thus defaults to 0 indicating hash), but it will be required to both implement POWER9 kvm support and tcg radix support. We also add this field to be migrated as part of the sPAPRMachineState as we will need it on the receiving side as the guest will never tell us this information again and we need it to perform translation. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-03-01 07:54:36 +01:00
{
sPAPRMachineState *spapr = SPAPR_MACHINE(vhyp);
/* Copy PATE1:GR into PATE0:HR */
entry->dw0 = spapr->patb_entry & PATE0_HR;
entry->dw1 = spapr->patb_entry;
target/ppc: Add patb_entry to sPAPRMachineState ISA v3.00 adds the idea of a partition table which is used to store the address translation details for all partitions on the system. The partition table consists of double word entries indexed by partition id where the second double word contains the location of the process table in guest memory. The process table is registered by the guest via a h-call. We need somewhere to store the address of the process table so we add an entry to the sPAPRMachineState struct called patb_entry to represent the second doubleword of a single partition table entry corresponding to the current guest. We need to store this value so we know if the guest is using radix or hash translation and the location of the corresponding process table in guest memory. Since we only have a single guest per qemu instance, we only need one entry. Since the partition table is technically a hypervisor resource we require that access to it is abstracted by the virtual hypervisor through the get_patbe() call. Currently the value of the entry is never set (and thus defaults to 0 indicating hash), but it will be required to both implement POWER9 kvm support and tcg radix support. We also add this field to be migrated as part of the sPAPRMachineState as we will need it on the receiving side as the guest will never tell us this information again and we need it to perform translation. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-03-01 07:54:36 +01:00
}
#define HPTE(_table, _i) (void *)(((uint64_t *)(_table)) + ((_i) * 2))
#define HPTE_VALID(_hpte) (tswap64(*((uint64_t *)(_hpte))) & HPTE64_V_VALID)
#define HPTE_DIRTY(_hpte) (tswap64(*((uint64_t *)(_hpte))) & HPTE64_V_HPTE_DIRTY)
#define CLEAN_HPTE(_hpte) ((*(uint64_t *)(_hpte)) &= tswap64(~HPTE64_V_HPTE_DIRTY))
#define DIRTY_HPTE(_hpte) ((*(uint64_t *)(_hpte)) |= tswap64(HPTE64_V_HPTE_DIRTY))
/*
* Get the fd to access the kernel htab, re-opening it if necessary
*/
static int get_htab_fd(sPAPRMachineState *spapr)
{
Error *local_err = NULL;
if (spapr->htab_fd >= 0) {
return spapr->htab_fd;
}
spapr->htab_fd = kvmppc_get_htab_fd(false, 0, &local_err);
if (spapr->htab_fd < 0) {
error_report_err(local_err);
}
return spapr->htab_fd;
}
void close_htab_fd(sPAPRMachineState *spapr)
{
if (spapr->htab_fd >= 0) {
close(spapr->htab_fd);
}
spapr->htab_fd = -1;
}
static hwaddr spapr_hpt_mask(PPCVirtualHypervisor *vhyp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(vhyp);
return HTAB_SIZE(spapr) / HASH_PTEG_SIZE_64 - 1;
}
static target_ulong spapr_encode_hpt_for_kvm_pr(PPCVirtualHypervisor *vhyp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(vhyp);
assert(kvm_enabled());
if (!spapr->htab) {
return 0;
}
return (target_ulong)(uintptr_t)spapr->htab | (spapr->htab_shift - 18);
}
static const ppc_hash_pte64_t *spapr_map_hptes(PPCVirtualHypervisor *vhyp,
hwaddr ptex, int n)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(vhyp);
hwaddr pte_offset = ptex * HASH_PTE_SIZE_64;
if (!spapr->htab) {
/*
* HTAB is controlled by KVM. Fetch into temporary buffer
*/
ppc_hash_pte64_t *hptes = g_malloc(n * HASH_PTE_SIZE_64);
kvmppc_read_hptes(hptes, ptex, n);
return hptes;
}
/*
* HTAB is controlled by QEMU. Just point to the internally
* accessible PTEG.
*/
return (const ppc_hash_pte64_t *)(spapr->htab + pte_offset);
}
static void spapr_unmap_hptes(PPCVirtualHypervisor *vhyp,
const ppc_hash_pte64_t *hptes,
hwaddr ptex, int n)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(vhyp);
if (!spapr->htab) {
g_free((void *)hptes);
}
/* Nothing to do for qemu managed HPT */
}
static void spapr_store_hpte(PPCVirtualHypervisor *vhyp, hwaddr ptex,
uint64_t pte0, uint64_t pte1)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(vhyp);
hwaddr offset = ptex * HASH_PTE_SIZE_64;
if (!spapr->htab) {
kvmppc_write_hpte(ptex, pte0, pte1);
} else {
if (pte0 & HPTE64_V_VALID) {
stq_p(spapr->htab + offset + HASH_PTE_SIZE_64 / 2, pte1);
/*
* When setting valid, we write PTE1 first. This ensures
* proper synchronization with the reading code in
* ppc_hash64_pteg_search()
*/
smp_wmb();
stq_p(spapr->htab + offset, pte0);
} else {
stq_p(spapr->htab + offset, pte0);
/*
* When clearing it we set PTE0 first. This ensures proper
* synchronization with the reading code in
* ppc_hash64_pteg_search()
*/
smp_wmb();
stq_p(spapr->htab + offset + HASH_PTE_SIZE_64 / 2, pte1);
}
}
}
pseries: Implement HPT resizing This patch implements hypercalls allowing a PAPR guest to resize its own hash page table. This will eventually allow for more flexible memory hotplug. The implementation is partially asynchronous, handled in a special thread running the hpt_prepare_thread() function. The state of a pending resize is stored in SPAPR_MACHINE->pending_hpt. The H_RESIZE_HPT_PREPARE hypercall will kick off creation of a new HPT, or, if one is already in progress, monitor it for completion. If there is an existing HPT resize in progress that doesn't match the size specified in the call, it will cancel it, replacing it with a new one matching the given size. The H_RESIZE_HPT_COMMIT completes transition to a resized HPT, and can only be called successfully once H_RESIZE_HPT_PREPARE has successfully completed initialization of a new HPT. The guest must ensure that there are no concurrent accesses to the existing HPT while this is called (this effectively means stop_machine() for Linux guests). For now H_RESIZE_HPT_COMMIT goes through the whole old HPT, rehashing each HPTE into the new HPT. This can have quite high latency, but it seems to be of the order of typical migration downtime latencies for HPTs of size up to ~2GiB (which would be used in a 256GiB guest). In future we probably want to move more of the rehashing to the "prepare" phase, by having H_ENTER and other hcalls update both current and pending HPTs. That's a project for another day, but should be possible without any changes to the guest interface. Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-12 07:46:49 +02:00
int spapr_hpt_shift_for_ramsize(uint64_t ramsize)
{
int shift;
/* We aim for a hash table of size 1/128 the size of RAM (rounded
* up). The PAPR recommendation is actually 1/64 of RAM size, but
* that's much more than is needed for Linux guests */
shift = ctz64(pow2ceil(ramsize)) - 7;
shift = MAX(shift, 18); /* Minimum architected size */
shift = MIN(shift, 46); /* Maximum architected size */
return shift;
}
void spapr_free_hpt(sPAPRMachineState *spapr)
{
g_free(spapr->htab);
spapr->htab = NULL;
spapr->htab_shift = 0;
close_htab_fd(spapr);
}
void spapr_reallocate_hpt(sPAPRMachineState *spapr, int shift,
Error **errp)
{
long rc;
/* Clean up any HPT info from a previous boot */
spapr_free_hpt(spapr);
rc = kvmppc_reset_htab(shift);
if (rc < 0) {
/* kernel-side HPT needed, but couldn't allocate one */
error_setg_errno(errp, errno,
"Failed to allocate KVM HPT of order %d (try smaller maxmem?)",
shift);
/* This is almost certainly fatal, but if the caller really
* wants to carry on with shift == 0, it's welcome to try */
} else if (rc > 0) {
/* kernel-side HPT allocated */
if (rc != shift) {
error_setg(errp,
"Requested order %d HPT, but kernel allocated order %ld (try smaller maxmem?)",
shift, rc);
}
spapr->htab_shift = shift;
target-ppc: Eliminate kvmppc_kern_htab global fa48b43 "target-ppc: Remove hack for ppc_hash64_load_hpte*() with HV KVM" purports to remove a hack in the handling of hash page tables (HPTs) managed by KVM instead of qemu. However, it actually went in the wrong direction. That patch requires anything looking for an external HPT (that is one not managed by the guest itself) to check both env->external_htab (for a qemu managed HPT) and kvmppc_kern_htab (for a KVM managed HPT). That's a problem because kvmppc_kern_htab is local to mmu-hash64.c, but some places which need to check for an external HPT are outside that, such as kvm_arch_get_registers(). The latter was subtly broken by the earlier patch such that gdbstub can no longer access memory. Basically a KVM managed HPT is much more like a qemu managed HPT than it is like a guest managed HPT, so the original "hack" was actually on the right track. This partially reverts fa48b43, so we again mark a KVM managed external HPT by putting a special but non-NULL value in env->external_htab. It then goes further, using that marker to eliminate the kvmppc_kern_htab global entirely. The ppc_hash64_set_external_hpt() helper function is extended to set that marker if passed a NULL value (if you're setting an external HPT, but don't have an actual HPT to set, the assumption is that it must be a KVM managed HPT). This also has some flow-on changes to the HPT access helpers, required by the above changes. Reported-by: Greg Kurz <gkurz@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Thomas Huth <thuth@redhat.com> Reviewed-by: Greg Kurz <gkurz@linux.vnet.ibm.com> Tested-by: Greg Kurz <gkurz@linux.vnet.ibm.com>
2016-03-08 01:35:15 +01:00
spapr->htab = NULL;
} else {
/* kernel-side HPT not needed, allocate in userspace instead */
size_t size = 1ULL << shift;
int i;
spapr->htab = qemu_memalign(size, size);
if (!spapr->htab) {
error_setg_errno(errp, errno,
"Could not allocate HPT of order %d", shift);
return;
}
memset(spapr->htab, 0, size);
spapr->htab_shift = shift;
for (i = 0; i < size / HASH_PTE_SIZE_64; i++) {
DIRTY_HPTE(HPTE(spapr->htab, i));
}
}
/* We're setting up a hash table, so that means we're not radix */
spapr_set_all_lpcrs(0, LPCR_HR | LPCR_UPRT);
}
void spapr_setup_hpt_and_vrma(sPAPRMachineState *spapr)
{
int hpt_shift;
if ((spapr->resize_hpt == SPAPR_RESIZE_HPT_DISABLED)
|| (spapr->cas_reboot
&& !spapr_ovec_test(spapr->ov5_cas, OV5_HPT_RESIZE))) {
hpt_shift = spapr_hpt_shift_for_ramsize(MACHINE(spapr)->maxram_size);
} else {
uint64_t current_ram_size;
current_ram_size = MACHINE(spapr)->ram_size + get_plugged_memory_size();
hpt_shift = spapr_hpt_shift_for_ramsize(current_ram_size);
}
spapr_reallocate_hpt(spapr, hpt_shift, &error_fatal);
if (spapr->vrma_adjust) {
spapr->rma_size = kvmppc_rma_size(spapr_node0_size(MACHINE(spapr)),
spapr->htab_shift);
}
}
static int spapr_reset_drcs(Object *child, void *opaque)
{
sPAPRDRConnector *drc =
(sPAPRDRConnector *) object_dynamic_cast(child,
TYPE_SPAPR_DR_CONNECTOR);
if (drc) {
spapr_drc_reset(drc);
}
return 0;
}
static void spapr_machine_reset(void)
{
MachineState *machine = MACHINE(qdev_get_machine());
sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
PowerPCCPU *first_ppc_cpu;
uint32_t rtas_limit;
hwaddr rtas_addr, fdt_addr;
void *fdt;
int rc;
spapr_caps_apply(spapr);
spapr: Capabilities infrastructure Because PAPR is a paravirtual environment access to certain CPU (or other) facilities can be blocked by the hypervisor. PAPR provides ways to advertise in the device tree whether or not those features are available to the guest. In some places we automatically determine whether to make a feature available based on whether our host can support it, in most cases this is based on limitations in the available KVM implementation. Although we correctly advertise this to the guest, it means that host factors might make changes to the guest visible environment which is bad: as well as generaly reducing reproducibility, it means that a migration between different host environments can easily go bad. We've mostly gotten away with it because the environments considered mature enough to be well supported (basically, KVM on POWER8) have had consistent feature availability. But, it's still not right and some limitations on POWER9 is going to make it more of an issue in future. This introduces an infrastructure for defining "sPAPR capabilities". These are set by default based on the machine version, masked by the capabilities of the chosen cpu, but can be overriden with machine properties. The intention is at reset time we verify that the requested capabilities can be supported on the host (considering TCG, KVM and/or host cpu limitations). If not we simply fail, rather than silently modifying the advertised featureset to the guest. This does mean that certain configurations that "worked" may now fail, but such configurations were already more subtly broken. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Greg Kurz <groug@kaod.org>
2017-12-08 00:35:35 +01:00
first_ppc_cpu = POWERPC_CPU(first_cpu);
if (kvm_enabled() && kvmppc_has_cap_mmu_radix() &&
ppc_type_check_compat(machine->cpu_type, CPU_POWERPC_LOGICAL_3_00, 0,
spapr->max_compat_pvr)) {
/*
* If using KVM with radix mode available, VCPUs can be started
* without a HPT because KVM will start them in radix mode.
* Set the GR bit in PATE so that we know there is no HPT.
*/
spapr->patb_entry = PATE1_GR;
spapr_set_all_lpcrs(LPCR_HR | LPCR_UPRT, LPCR_HR | LPCR_UPRT);
} else {
spapr_setup_hpt_and_vrma(spapr);
}
/*
* If this reset wasn't generated by CAS, we should reset our
* negotiated options and start from scratch
*/
spapr: fix device tree properties when using compatibility mode Commit 51f84465dd98 changed the compatility mode setting logic: - machine reset only sets compatibility mode for the boot CPU - compatibility mode is set for other CPUs when they are put online by the guest with the "start-cpu" RTAS call This causes a regression for machines started with max-compat-cpu: the device tree nodes related to secondary CPU cores contain wrong "cpu-version" and "ibm,pa-features" values, as shown below. Guest started on a POWER8 host with: -smp cores=2 -machine pseries,max-cpu-compat=compat7 ibm,pa-features = [18 00 f6 3f c7 c0 80 f0 80 00 00 00 00 00 00 00 00 00 80 00 80 00 80 00 00 00]; cpu-version = <0x4d0200>; ^^^ second CPU core ibm,pa-features = <0x600f63f 0xc70080c0>; cpu-version = <0xf000003>; ^^^ boot CPU core The second core is advertised in raw POWER8 mode. This happens because CAS assumes all CPUs to have the same compatibility mode. Since the boot CPU already has the requested compatibility mode, the CAS code does not set it for the secondary one, and exposes the bogus device tree properties in in the CAS response to the guest. A similar situation is observed when hot-plugging a CPU core. The related device tree properties are generated and exposed to guest with the "ibm,configure-connector" RTAS before "start-cpu" is called. The CPU core is advertised to the guest in raw mode as well. It both cases, it boils down to the fact that "start-cpu" happens too late. This can be fixed globally by propagating the compatibility mode of the boot CPU to the other CPUs during reset. For this to work, the compatibility mode of the boot CPU must be set before the machine code actually resets all CPUs. It is not needed to set the compatibility mode in "start-cpu" anymore, so the code is dropped. Fixes: 51f84465dd98 Signed-off-by: Greg Kurz <groug@kaod.org> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2018-01-17 10:20:42 +01:00
if (!spapr->cas_reboot) {
spapr_ovec_cleanup(spapr->ov5_cas);
spapr->ov5_cas = spapr_ovec_new();
ppc_set_compat(first_ppc_cpu, spapr->max_compat_pvr, &error_fatal);
}
if (!SPAPR_MACHINE_GET_CLASS(spapr)->legacy_irq_allocation) {
spapr_irq_msi_reset(spapr);
}
qemu_devices_reset();
/*
* This is fixing some of the default configuration of the XIVE
* devices. To be called after the reset of the machine devices.
*/
spapr_irq_reset(spapr, &error_fatal);
/* DRC reset may cause a device to be unplugged. This will cause troubles
* if this device is used by another device (eg, a running vhost backend
* will crash QEMU if the DIMM holding the vring goes away). To avoid such
* situations, we reset DRCs after all devices have been reset.
*/
object_child_foreach_recursive(object_get_root(), spapr_reset_drcs, NULL);
spapr_clear_pending_events(spapr);
/*
* We place the device tree and RTAS just below either the top of the RMA,
* or just below 2GB, whichever is lower, so that it can be
* processed with 32-bit real mode code if necessary
*/
rtas_limit = MIN(spapr->rma_size, RTAS_MAX_ADDR);
rtas_addr = rtas_limit - RTAS_MAX_SIZE;
fdt_addr = rtas_addr - FDT_MAX_SIZE;
fdt = spapr_build_fdt(spapr);
spapr_load_rtas(spapr, fdt, rtas_addr);
rc = fdt_pack(fdt);
/* Should only fail if we've built a corrupted tree */
assert(rc == 0);
if (fdt_totalsize(fdt) > FDT_MAX_SIZE) {
error_report("FDT too big ! 0x%x bytes (max is 0x%x)",
fdt_totalsize(fdt), FDT_MAX_SIZE);
exit(1);
}
/* Load the fdt */
qemu_fdt_dumpdtb(fdt, fdt_totalsize(fdt));
cpu_physical_memory_write(fdt_addr, fdt, fdt_totalsize(fdt));
g_free(spapr->fdt_blob);
spapr->fdt_size = fdt_totalsize(fdt);
spapr->fdt_initial_size = spapr->fdt_size;
spapr->fdt_blob = fdt;
/* Set up the entry state */
spapr_cpu_set_entry_state(first_ppc_cpu, SPAPR_ENTRY_POINT, fdt_addr);
first_ppc_cpu->env.gpr[5] = 0;
spapr: add option vector handling in CAS-generated resets In some cases, ibm,client-architecture-support calls can fail. This could happen in the current code for situations where the modified device tree segment exceeds the buffer size provided by the guest via the call parameters. In these cases, QEMU will reset, allowing an opportunity to regenerate the device tree from scratch via boot-time handling. There are potentially other scenarios as well, not currently reachable in the current code, but possible in theory, such as cases where device-tree properties or nodes need to be removed. We currently don't handle either of these properly for option vector capabilities however. Instead of carrying the negotiated capability beyond the reset and creating the boot-time device tree accordingly, we start from scratch, generating the same boot-time device tree as we did prior to the CAS-generated and the same device tree updates as we did before. This could (in theory) cause us to get stuck in a reset loop. This hasn't been observed, but depending on the extensiveness of CAS-induced device tree updates in the future, could eventually become an issue. Address this by pulling capability-related device tree updates resulting from CAS calls into a common routine, spapr_dt_cas_updates(), and adding an sPAPROptionVector* parameter that allows us to test for newly-negotiated capabilities. We invoke it as follows: 1) When ibm,client-architecture-support gets called, we call spapr_dt_cas_updates() with the set of capabilities added since the previous call to ibm,client-architecture-support. For the initial boot, or a system reset generated by something other than the CAS call itself, this set will consist of *all* options supported both the platform and the guest. For calls to ibm,client-architecture-support immediately after a CAS-induced reset, we call spapr_dt_cas_updates() with only the set of capabilities added since the previous call, since the other capabilities will have already been addressed by the boot-time device-tree this time around. In the unlikely event that capabilities are *removed* since the previous CAS, we will generate a CAS-induced reset. In the unlikely event that we cannot fit the device-tree updates into the buffer provided by the guest, well generate a CAS-induced reset. 2) When a CAS update results in the need to reset the machine and include the updates in the boot-time device tree, we call the spapr_dt_cas_updates() using the full set of negotiated capabilities as part of the reset path. At initial boot, or after a reset generated by something other than the CAS call itself, this set will be empty, resulting in what should be the same boot-time device-tree as we generated prior to this patch. For CAS-induced reset, this routine will be called with the full set of capabilities negotiated by the platform/guest in the previous CAS call, which should result in CAS updates from previous call being accounted for in the initial boot-time device tree. Signed-off-by: Michael Roth <mdroth@linux.vnet.ibm.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> [dwg: Changed an int -> bool conversion to be more explicit] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2016-10-25 06:47:29 +02:00
spapr->cas_reboot = false;
}
static void spapr_create_nvram(sPAPRMachineState *spapr)
{
DeviceState *dev = qdev_create(&spapr->vio_bus->bus, "spapr-nvram");
DriveInfo *dinfo = drive_get(IF_PFLASH, 0, 0);
if (dinfo) {
qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(dinfo),
&error_fatal);
}
qdev_init_nofail(dev);
spapr->nvram = (struct sPAPRNVRAM *)dev;
}
static void spapr_rtc_create(sPAPRMachineState *spapr)
{
object_initialize_child(OBJECT(spapr), "rtc",
&spapr->rtc, sizeof(spapr->rtc), TYPE_SPAPR_RTC,
&error_fatal, NULL);
object_property_set_bool(OBJECT(&spapr->rtc), true, "realized",
&error_fatal);
object_property_add_alias(OBJECT(spapr), "rtc-time", OBJECT(&spapr->rtc),
"date", &error_fatal);
}
/* Returns whether we want to use VGA or not */
static bool spapr_vga_init(PCIBus *pci_bus, Error **errp)
{
switch (vga_interface_type) {
case VGA_NONE:
return false;
case VGA_DEVICE:
return true;
case VGA_STD:
case VGA_VIRTIO:
case VGA_CIRRUS:
return pci_vga_init(pci_bus) != NULL;
default:
error_setg(errp,
"Unsupported VGA mode, only -vga std or -vga virtio is supported");
return false;
}
}
static int spapr_pre_load(void *opaque)
{
int rc;
rc = spapr_caps_pre_load(opaque);
if (rc) {
return rc;
}
return 0;
}
static int spapr_post_load(void *opaque, int version_id)
{
sPAPRMachineState *spapr = (sPAPRMachineState *)opaque;
int err = 0;
err = spapr_caps_post_migration(spapr);
if (err) {
return err;
}
/*
* In earlier versions, there was no separate qdev for the PAPR
* RTC, so the RTC offset was stored directly in sPAPREnvironment.
* So when migrating from those versions, poke the incoming offset
* value into the RTC device
*/
if (version_id < 3) {
err = spapr_rtc_import_offset(&spapr->rtc, spapr->rtc_offset);
if (err) {
return err;
}
}
if (kvm_enabled() && spapr->patb_entry) {
PowerPCCPU *cpu = POWERPC_CPU(first_cpu);
bool radix = !!(spapr->patb_entry & PATE1_GR);
bool gtse = !!(cpu->env.spr[SPR_LPCR] & LPCR_GTSE);
/*
* Update LPCR:HR and UPRT as they may not be set properly in
* the stream
*/
spapr_set_all_lpcrs(radix ? (LPCR_HR | LPCR_UPRT) : 0,
LPCR_HR | LPCR_UPRT);
err = kvmppc_configure_v3_mmu(cpu, radix, gtse, spapr->patb_entry);
if (err) {
error_report("Process table config unsupported by the host");
return -EINVAL;
}
}
err = spapr_irq_post_load(spapr, version_id);
if (err) {
return err;
}
return err;
}
static int spapr_pre_save(void *opaque)
{
int rc;
rc = spapr_caps_pre_save(opaque);
if (rc) {
return rc;
}
return 0;
}
static bool version_before_3(void *opaque, int version_id)
{
return version_id < 3;
}
spapr: migrate pending_events of spapr state In racing situations between hotplug events and migration operation, a rtas hotplug event could have not yet be delivered to the source guest when migration is started. In this case the pending_events of spapr state need be transmitted to the target so that the hotplug event can be finished on the target. To achieve the minimal VMSD possible to migrate the pending_events list, this patch makes the changes in spapr_events.c: - 'log_type' of sPAPREventLogEntry struct deleted. This information can be derived by inspecting the rtas_error_log summary field. A new function called 'spapr_event_log_entry_type' was added to retrieve the type of a given sPAPREventLogEntry. - sPAPREventLogEntry, epow_log_full and hp_log_full were redesigned. The only data we're going to migrate in the VMSD is the event log data itself, which can be divided in two parts: a rtas_error_log header and an extended event log field. The rtas_error_log header contains information about the size of the extended log field, which can be used inside VMSD as the size parameter of the VBUFFER_ALOC field that will store it. To allow this use, the header.extended_length field must be exposed inline to the VMSD instead of embedded into a 'data' field that holds everything. With this in mind, the following changes were done: * a new 'header' field was added to sPAPREventLogEntry. This field holds a a struct rtas_error_log inline. * the declaration of the 'rtas_error_log' struct was moved to spapr.h to be visible to the VMSD macros. * 'data' field of sPAPREventLogEntry was renamed to 'extended_log' and now holds only the contents of the extended event log. * 'struct rtas_error_log hdr' were taken away from both epow_log_full and hp_log_full. This information is now available at the header field of sPAPREventLogEntry. * epow_log_full and hp_log_full were renamed to epow_extended_log and hp_extended_log respectively. This rename makes it clearer to understand the new purpose of both structures: hold the information of an extended event log field. * spapr_powerdown_req and spapr_hotplug_req_event now creates a sPAPREventLogEntry structure that contains the full rtas log entry. * rtas_event_log_queue and rtas_event_log_dequeue now receives a sPAPREventLogEntry pointer as a parameter instead of a void pointer. - the endianess of the sPAPREventLogEntry header is now native instead of be32. We can use the fields in native endianess internally and write them in be32 in the guest physical memory inside 'check_exception'. This allows the VMSD inside spapr.c to read the correct size of the entended_log field. - inside spapr.c, pending_events is put in a subsection in the spapr state VMSD to make sure migration across different versions is not broken. A small change in rtas_event_log_queue and rtas_event_log_dequeue were also made: instead of calling qdev_get_machine(), both functions now receive a pointer to the sPAPRMachineState. This pointer is already available in the callers of these functions and we don't need to waste resources calling qdev() again. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-07-11 20:07:55 +02:00
static bool spapr_pending_events_needed(void *opaque)
{
sPAPRMachineState *spapr = (sPAPRMachineState *)opaque;
return !QTAILQ_EMPTY(&spapr->pending_events);
}
static const VMStateDescription vmstate_spapr_event_entry = {
.name = "spapr_event_log_entry",
.version_id = 1,
.minimum_version_id = 1,
.fields = (VMStateField[]) {
VMSTATE_UINT32(summary, sPAPREventLogEntry),
VMSTATE_UINT32(extended_length, sPAPREventLogEntry),
spapr: migrate pending_events of spapr state In racing situations between hotplug events and migration operation, a rtas hotplug event could have not yet be delivered to the source guest when migration is started. In this case the pending_events of spapr state need be transmitted to the target so that the hotplug event can be finished on the target. To achieve the minimal VMSD possible to migrate the pending_events list, this patch makes the changes in spapr_events.c: - 'log_type' of sPAPREventLogEntry struct deleted. This information can be derived by inspecting the rtas_error_log summary field. A new function called 'spapr_event_log_entry_type' was added to retrieve the type of a given sPAPREventLogEntry. - sPAPREventLogEntry, epow_log_full and hp_log_full were redesigned. The only data we're going to migrate in the VMSD is the event log data itself, which can be divided in two parts: a rtas_error_log header and an extended event log field. The rtas_error_log header contains information about the size of the extended log field, which can be used inside VMSD as the size parameter of the VBUFFER_ALOC field that will store it. To allow this use, the header.extended_length field must be exposed inline to the VMSD instead of embedded into a 'data' field that holds everything. With this in mind, the following changes were done: * a new 'header' field was added to sPAPREventLogEntry. This field holds a a struct rtas_error_log inline. * the declaration of the 'rtas_error_log' struct was moved to spapr.h to be visible to the VMSD macros. * 'data' field of sPAPREventLogEntry was renamed to 'extended_log' and now holds only the contents of the extended event log. * 'struct rtas_error_log hdr' were taken away from both epow_log_full and hp_log_full. This information is now available at the header field of sPAPREventLogEntry. * epow_log_full and hp_log_full were renamed to epow_extended_log and hp_extended_log respectively. This rename makes it clearer to understand the new purpose of both structures: hold the information of an extended event log field. * spapr_powerdown_req and spapr_hotplug_req_event now creates a sPAPREventLogEntry structure that contains the full rtas log entry. * rtas_event_log_queue and rtas_event_log_dequeue now receives a sPAPREventLogEntry pointer as a parameter instead of a void pointer. - the endianess of the sPAPREventLogEntry header is now native instead of be32. We can use the fields in native endianess internally and write them in be32 in the guest physical memory inside 'check_exception'. This allows the VMSD inside spapr.c to read the correct size of the entended_log field. - inside spapr.c, pending_events is put in a subsection in the spapr state VMSD to make sure migration across different versions is not broken. A small change in rtas_event_log_queue and rtas_event_log_dequeue were also made: instead of calling qdev_get_machine(), both functions now receive a pointer to the sPAPRMachineState. This pointer is already available in the callers of these functions and we don't need to waste resources calling qdev() again. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-07-11 20:07:55 +02:00
VMSTATE_VBUFFER_ALLOC_UINT32(extended_log, sPAPREventLogEntry, 0,
NULL, extended_length),
spapr: migrate pending_events of spapr state In racing situations between hotplug events and migration operation, a rtas hotplug event could have not yet be delivered to the source guest when migration is started. In this case the pending_events of spapr state need be transmitted to the target so that the hotplug event can be finished on the target. To achieve the minimal VMSD possible to migrate the pending_events list, this patch makes the changes in spapr_events.c: - 'log_type' of sPAPREventLogEntry struct deleted. This information can be derived by inspecting the rtas_error_log summary field. A new function called 'spapr_event_log_entry_type' was added to retrieve the type of a given sPAPREventLogEntry. - sPAPREventLogEntry, epow_log_full and hp_log_full were redesigned. The only data we're going to migrate in the VMSD is the event log data itself, which can be divided in two parts: a rtas_error_log header and an extended event log field. The rtas_error_log header contains information about the size of the extended log field, which can be used inside VMSD as the size parameter of the VBUFFER_ALOC field that will store it. To allow this use, the header.extended_length field must be exposed inline to the VMSD instead of embedded into a 'data' field that holds everything. With this in mind, the following changes were done: * a new 'header' field was added to sPAPREventLogEntry. This field holds a a struct rtas_error_log inline. * the declaration of the 'rtas_error_log' struct was moved to spapr.h to be visible to the VMSD macros. * 'data' field of sPAPREventLogEntry was renamed to 'extended_log' and now holds only the contents of the extended event log. * 'struct rtas_error_log hdr' were taken away from both epow_log_full and hp_log_full. This information is now available at the header field of sPAPREventLogEntry. * epow_log_full and hp_log_full were renamed to epow_extended_log and hp_extended_log respectively. This rename makes it clearer to understand the new purpose of both structures: hold the information of an extended event log field. * spapr_powerdown_req and spapr_hotplug_req_event now creates a sPAPREventLogEntry structure that contains the full rtas log entry. * rtas_event_log_queue and rtas_event_log_dequeue now receives a sPAPREventLogEntry pointer as a parameter instead of a void pointer. - the endianess of the sPAPREventLogEntry header is now native instead of be32. We can use the fields in native endianess internally and write them in be32 in the guest physical memory inside 'check_exception'. This allows the VMSD inside spapr.c to read the correct size of the entended_log field. - inside spapr.c, pending_events is put in a subsection in the spapr state VMSD to make sure migration across different versions is not broken. A small change in rtas_event_log_queue and rtas_event_log_dequeue were also made: instead of calling qdev_get_machine(), both functions now receive a pointer to the sPAPRMachineState. This pointer is already available in the callers of these functions and we don't need to waste resources calling qdev() again. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-07-11 20:07:55 +02:00
VMSTATE_END_OF_LIST()
},
};
static const VMStateDescription vmstate_spapr_pending_events = {
.name = "spapr_pending_events",
.version_id = 1,
.minimum_version_id = 1,
.needed = spapr_pending_events_needed,
.fields = (VMStateField[]) {
VMSTATE_QTAILQ_V(pending_events, sPAPRMachineState, 1,
vmstate_spapr_event_entry, sPAPREventLogEntry, next),
VMSTATE_END_OF_LIST()
},
};
static bool spapr_ov5_cas_needed(void *opaque)
{
sPAPRMachineState *spapr = opaque;
sPAPROptionVector *ov5_mask = spapr_ovec_new();
sPAPROptionVector *ov5_legacy = spapr_ovec_new();
sPAPROptionVector *ov5_removed = spapr_ovec_new();
bool cas_needed;
/* Prior to the introduction of sPAPROptionVector, we had two option
* vectors we dealt with: OV5_FORM1_AFFINITY, and OV5_DRCONF_MEMORY.
* Both of these options encode machine topology into the device-tree
* in such a way that the now-booted OS should still be able to interact
* appropriately with QEMU regardless of what options were actually
* negotiatied on the source side.
*
* As such, we can avoid migrating the CAS-negotiated options if these
* are the only options available on the current machine/platform.
* Since these are the only options available for pseries-2.7 and
* earlier, this allows us to maintain old->new/new->old migration
* compatibility.
*
* For QEMU 2.8+, there are additional CAS-negotiatable options available
* via default pseries-2.8 machines and explicit command-line parameters.
* Some of these options, like OV5_HP_EVT, *do* require QEMU to be aware
* of the actual CAS-negotiated values to continue working properly. For
* example, availability of memory unplug depends on knowing whether
* OV5_HP_EVT was negotiated via CAS.
*
* Thus, for any cases where the set of available CAS-negotiatable
* options extends beyond OV5_FORM1_AFFINITY and OV5_DRCONF_MEMORY, we
* include the CAS-negotiated options in the migration stream, unless
* if they affect boot time behaviour only.
*/
spapr_ovec_set(ov5_mask, OV5_FORM1_AFFINITY);
spapr_ovec_set(ov5_mask, OV5_DRCONF_MEMORY);
spapr_ovec_set(ov5_mask, OV5_DRMEM_V2);
/* spapr_ovec_diff returns true if bits were removed. we avoid using
* the mask itself since in the future it's possible "legacy" bits may be
* removed via machine options, which could generate a false positive
* that breaks migration.
*/
spapr_ovec_intersect(ov5_legacy, spapr->ov5, ov5_mask);
cas_needed = spapr_ovec_diff(ov5_removed, spapr->ov5, ov5_legacy);
spapr_ovec_cleanup(ov5_mask);
spapr_ovec_cleanup(ov5_legacy);
spapr_ovec_cleanup(ov5_removed);
return cas_needed;
}
static const VMStateDescription vmstate_spapr_ov5_cas = {
.name = "spapr_option_vector_ov5_cas",
.version_id = 1,
.minimum_version_id = 1,
.needed = spapr_ov5_cas_needed,
.fields = (VMStateField[]) {
VMSTATE_STRUCT_POINTER_V(ov5_cas, sPAPRMachineState, 1,
vmstate_spapr_ovec, sPAPROptionVector),
VMSTATE_END_OF_LIST()
},
};
target/ppc: Add patb_entry to sPAPRMachineState ISA v3.00 adds the idea of a partition table which is used to store the address translation details for all partitions on the system. The partition table consists of double word entries indexed by partition id where the second double word contains the location of the process table in guest memory. The process table is registered by the guest via a h-call. We need somewhere to store the address of the process table so we add an entry to the sPAPRMachineState struct called patb_entry to represent the second doubleword of a single partition table entry corresponding to the current guest. We need to store this value so we know if the guest is using radix or hash translation and the location of the corresponding process table in guest memory. Since we only have a single guest per qemu instance, we only need one entry. Since the partition table is technically a hypervisor resource we require that access to it is abstracted by the virtual hypervisor through the get_patbe() call. Currently the value of the entry is never set (and thus defaults to 0 indicating hash), but it will be required to both implement POWER9 kvm support and tcg radix support. We also add this field to be migrated as part of the sPAPRMachineState as we will need it on the receiving side as the guest will never tell us this information again and we need it to perform translation. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-03-01 07:54:36 +01:00
static bool spapr_patb_entry_needed(void *opaque)
{
sPAPRMachineState *spapr = opaque;
return !!spapr->patb_entry;
}
static const VMStateDescription vmstate_spapr_patb_entry = {
.name = "spapr_patb_entry",
.version_id = 1,
.minimum_version_id = 1,
.needed = spapr_patb_entry_needed,
.fields = (VMStateField[]) {
VMSTATE_UINT64(patb_entry, sPAPRMachineState),
VMSTATE_END_OF_LIST()
},
};
static bool spapr_irq_map_needed(void *opaque)
{
sPAPRMachineState *spapr = opaque;
return spapr->irq_map && !bitmap_empty(spapr->irq_map, spapr->irq_map_nr);
}
static const VMStateDescription vmstate_spapr_irq_map = {
.name = "spapr_irq_map",
.version_id = 1,
.minimum_version_id = 1,
.needed = spapr_irq_map_needed,
.fields = (VMStateField[]) {
VMSTATE_BITMAP(irq_map, sPAPRMachineState, 0, irq_map_nr),
VMSTATE_END_OF_LIST()
},
};
static bool spapr_dtb_needed(void *opaque)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(opaque);
return smc->update_dt_enabled;
}
static int spapr_dtb_pre_load(void *opaque)
{
sPAPRMachineState *spapr = (sPAPRMachineState *)opaque;
g_free(spapr->fdt_blob);
spapr->fdt_blob = NULL;
spapr->fdt_size = 0;
return 0;
}
static const VMStateDescription vmstate_spapr_dtb = {
.name = "spapr_dtb",
.version_id = 1,
.minimum_version_id = 1,
.needed = spapr_dtb_needed,
.pre_load = spapr_dtb_pre_load,
.fields = (VMStateField[]) {
VMSTATE_UINT32(fdt_initial_size, sPAPRMachineState),
VMSTATE_UINT32(fdt_size, sPAPRMachineState),
VMSTATE_VBUFFER_ALLOC_UINT32(fdt_blob, sPAPRMachineState, 0, NULL,
fdt_size),
VMSTATE_END_OF_LIST()
},
};
static const VMStateDescription vmstate_spapr = {
.name = "spapr",
.version_id = 3,
.minimum_version_id = 1,
.pre_load = spapr_pre_load,
.post_load = spapr_post_load,
.pre_save = spapr_pre_save,
.fields = (VMStateField[]) {
/* used to be @next_irq */
VMSTATE_UNUSED_BUFFER(version_before_3, 0, 4),
/* RTC offset */
VMSTATE_UINT64_TEST(rtc_offset, sPAPRMachineState, version_before_3),
VMSTATE_PPC_TIMEBASE_V(tb, sPAPRMachineState, 2),
VMSTATE_END_OF_LIST()
},
.subsections = (const VMStateDescription*[]) {
&vmstate_spapr_ov5_cas,
target/ppc: Add patb_entry to sPAPRMachineState ISA v3.00 adds the idea of a partition table which is used to store the address translation details for all partitions on the system. The partition table consists of double word entries indexed by partition id where the second double word contains the location of the process table in guest memory. The process table is registered by the guest via a h-call. We need somewhere to store the address of the process table so we add an entry to the sPAPRMachineState struct called patb_entry to represent the second doubleword of a single partition table entry corresponding to the current guest. We need to store this value so we know if the guest is using radix or hash translation and the location of the corresponding process table in guest memory. Since we only have a single guest per qemu instance, we only need one entry. Since the partition table is technically a hypervisor resource we require that access to it is abstracted by the virtual hypervisor through the get_patbe() call. Currently the value of the entry is never set (and thus defaults to 0 indicating hash), but it will be required to both implement POWER9 kvm support and tcg radix support. We also add this field to be migrated as part of the sPAPRMachineState as we will need it on the receiving side as the guest will never tell us this information again and we need it to perform translation. Signed-off-by: Suraj Jitindar Singh <sjitindarsingh@gmail.com> Reviewed-by: David Gibson <david@gibson.dropbear.id.au> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-03-01 07:54:36 +01:00
&vmstate_spapr_patb_entry,
spapr: migrate pending_events of spapr state In racing situations between hotplug events and migration operation, a rtas hotplug event could have not yet be delivered to the source guest when migration is started. In this case the pending_events of spapr state need be transmitted to the target so that the hotplug event can be finished on the target. To achieve the minimal VMSD possible to migrate the pending_events list, this patch makes the changes in spapr_events.c: - 'log_type' of sPAPREventLogEntry struct deleted. This information can be derived by inspecting the rtas_error_log summary field. A new function called 'spapr_event_log_entry_type' was added to retrieve the type of a given sPAPREventLogEntry. - sPAPREventLogEntry, epow_log_full and hp_log_full were redesigned. The only data we're going to migrate in the VMSD is the event log data itself, which can be divided in two parts: a rtas_error_log header and an extended event log field. The rtas_error_log header contains information about the size of the extended log field, which can be used inside VMSD as the size parameter of the VBUFFER_ALOC field that will store it. To allow this use, the header.extended_length field must be exposed inline to the VMSD instead of embedded into a 'data' field that holds everything. With this in mind, the following changes were done: * a new 'header' field was added to sPAPREventLogEntry. This field holds a a struct rtas_error_log inline. * the declaration of the 'rtas_error_log' struct was moved to spapr.h to be visible to the VMSD macros. * 'data' field of sPAPREventLogEntry was renamed to 'extended_log' and now holds only the contents of the extended event log. * 'struct rtas_error_log hdr' were taken away from both epow_log_full and hp_log_full. This information is now available at the header field of sPAPREventLogEntry. * epow_log_full and hp_log_full were renamed to epow_extended_log and hp_extended_log respectively. This rename makes it clearer to understand the new purpose of both structures: hold the information of an extended event log field. * spapr_powerdown_req and spapr_hotplug_req_event now creates a sPAPREventLogEntry structure that contains the full rtas log entry. * rtas_event_log_queue and rtas_event_log_dequeue now receives a sPAPREventLogEntry pointer as a parameter instead of a void pointer. - the endianess of the sPAPREventLogEntry header is now native instead of be32. We can use the fields in native endianess internally and write them in be32 in the guest physical memory inside 'check_exception'. This allows the VMSD inside spapr.c to read the correct size of the entended_log field. - inside spapr.c, pending_events is put in a subsection in the spapr state VMSD to make sure migration across different versions is not broken. A small change in rtas_event_log_queue and rtas_event_log_dequeue were also made: instead of calling qdev_get_machine(), both functions now receive a pointer to the sPAPRMachineState. This pointer is already available in the callers of these functions and we don't need to waste resources calling qdev() again. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-07-11 20:07:55 +02:00
&vmstate_spapr_pending_events,
&vmstate_spapr_cap_htm,
&vmstate_spapr_cap_vsx,
&vmstate_spapr_cap_dfp,
&vmstate_spapr_cap_cfpc,
&vmstate_spapr_cap_sbbc,
&vmstate_spapr_cap_ibs,
&vmstate_spapr_irq_map,
&vmstate_spapr_cap_nested_kvm_hv,
&vmstate_spapr_dtb,
NULL
}
};
static int htab_save_setup(QEMUFile *f, void *opaque)
{
sPAPRMachineState *spapr = opaque;
/* "Iteration" header */
if (!spapr->htab_shift) {
qemu_put_be32(f, -1);
} else {
qemu_put_be32(f, spapr->htab_shift);
}
if (spapr->htab) {
spapr->htab_save_index = 0;
spapr->htab_first_pass = true;
} else {
if (spapr->htab_shift) {
assert(kvm_enabled());
}
}
return 0;
}
static void htab_save_chunk(QEMUFile *f, sPAPRMachineState *spapr,
int chunkstart, int n_valid, int n_invalid)
{
qemu_put_be32(f, chunkstart);
qemu_put_be16(f, n_valid);
qemu_put_be16(f, n_invalid);
qemu_put_buffer(f, HPTE(spapr->htab, chunkstart),
HASH_PTE_SIZE_64 * n_valid);
}
static void htab_save_end_marker(QEMUFile *f)
{
qemu_put_be32(f, 0);
qemu_put_be16(f, 0);
qemu_put_be16(f, 0);
}
static void htab_save_first_pass(QEMUFile *f, sPAPRMachineState *spapr,
int64_t max_ns)
{
bool has_timeout = max_ns != -1;
int htabslots = HTAB_SIZE(spapr) / HASH_PTE_SIZE_64;
int index = spapr->htab_save_index;
int64_t starttime = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
assert(spapr->htab_first_pass);
do {
int chunkstart;
/* Consume invalid HPTEs */
while ((index < htabslots)
&& !HPTE_VALID(HPTE(spapr->htab, index))) {
CLEAN_HPTE(HPTE(spapr->htab, index));
spapr: fix buffer-overflow Running postcopy-test with ASAN produces the following error: QTEST_QEMU_BINARY=ppc64-softmmu/qemu-system-ppc64 tests/postcopy-test ... ================================================================= ==23641==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x7f1556600000 at pc 0x55b8e9d28208 bp 0x7f1555f4d3c0 sp 0x7f1555f4d3b0 READ of size 8 at 0x7f1556600000 thread T6 #0 0x55b8e9d28207 in htab_save_first_pass /home/elmarco/src/qq/hw/ppc/spapr.c:1528 #1 0x55b8e9d2939c in htab_save_iterate /home/elmarco/src/qq/hw/ppc/spapr.c:1665 #2 0x55b8e9beae3a in qemu_savevm_state_iterate /home/elmarco/src/qq/migration/savevm.c:1044 #3 0x55b8ea677733 in migration_thread /home/elmarco/src/qq/migration/migration.c:1976 #4 0x7f15845f46c9 in start_thread (/lib64/libpthread.so.0+0x76c9) #5 0x7f157d9d0f7e in clone (/lib64/libc.so.6+0x107f7e) 0x7f1556600000 is located 0 bytes to the right of 2097152-byte region [0x7f1556400000,0x7f1556600000) allocated by thread T0 here: #0 0x7f159bb76980 in posix_memalign (/lib64/libasan.so.3+0xc7980) #1 0x55b8eab185b2 in qemu_try_memalign /home/elmarco/src/qq/util/oslib-posix.c:106 #2 0x55b8eab186c8 in qemu_memalign /home/elmarco/src/qq/util/oslib-posix.c:122 #3 0x55b8e9d268a8 in spapr_reallocate_hpt /home/elmarco/src/qq/hw/ppc/spapr.c:1214 #4 0x55b8e9d26e04 in ppc_spapr_reset /home/elmarco/src/qq/hw/ppc/spapr.c:1261 #5 0x55b8ea12e913 in qemu_system_reset /home/elmarco/src/qq/vl.c:1697 #6 0x55b8ea13fa40 in main /home/elmarco/src/qq/vl.c:4679 #7 0x7f157d8e9400 in __libc_start_main (/lib64/libc.so.6+0x20400) Thread T6 created by T0 here: #0 0x7f159bae0488 in __interceptor_pthread_create (/lib64/libasan.so.3+0x31488) #1 0x55b8eab1d9cb in qemu_thread_create /home/elmarco/src/qq/util/qemu-thread-posix.c:465 #2 0x55b8ea67874c in migrate_fd_connect /home/elmarco/src/qq/migration/migration.c:2096 #3 0x55b8ea66cbb0 in migration_channel_connect /home/elmarco/src/qq/migration/migration.c:500 #4 0x55b8ea678f38 in socket_outgoing_migration /home/elmarco/src/qq/migration/socket.c:87 #5 0x55b8eaa5a03a in qio_task_complete /home/elmarco/src/qq/io/task.c:142 #6 0x55b8eaa599cc in gio_task_thread_result /home/elmarco/src/qq/io/task.c:88 #7 0x7f15823e38e6 (/lib64/libglib-2.0.so.0+0x468e6) SUMMARY: AddressSanitizer: heap-buffer-overflow /home/elmarco/src/qq/hw/ppc/spapr.c:1528 in htab_save_first_pass index seems to be wrongly incremented, unless I miss something that would be worth a comment. Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-03-23 11:04:55 +01:00
index++;
}
/* Consume valid HPTEs */
chunkstart = index;
while ((index < htabslots) && (index - chunkstart < USHRT_MAX)
&& HPTE_VALID(HPTE(spapr->htab, index))) {
CLEAN_HPTE(HPTE(spapr->htab, index));
spapr: fix buffer-overflow Running postcopy-test with ASAN produces the following error: QTEST_QEMU_BINARY=ppc64-softmmu/qemu-system-ppc64 tests/postcopy-test ... ================================================================= ==23641==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x7f1556600000 at pc 0x55b8e9d28208 bp 0x7f1555f4d3c0 sp 0x7f1555f4d3b0 READ of size 8 at 0x7f1556600000 thread T6 #0 0x55b8e9d28207 in htab_save_first_pass /home/elmarco/src/qq/hw/ppc/spapr.c:1528 #1 0x55b8e9d2939c in htab_save_iterate /home/elmarco/src/qq/hw/ppc/spapr.c:1665 #2 0x55b8e9beae3a in qemu_savevm_state_iterate /home/elmarco/src/qq/migration/savevm.c:1044 #3 0x55b8ea677733 in migration_thread /home/elmarco/src/qq/migration/migration.c:1976 #4 0x7f15845f46c9 in start_thread (/lib64/libpthread.so.0+0x76c9) #5 0x7f157d9d0f7e in clone (/lib64/libc.so.6+0x107f7e) 0x7f1556600000 is located 0 bytes to the right of 2097152-byte region [0x7f1556400000,0x7f1556600000) allocated by thread T0 here: #0 0x7f159bb76980 in posix_memalign (/lib64/libasan.so.3+0xc7980) #1 0x55b8eab185b2 in qemu_try_memalign /home/elmarco/src/qq/util/oslib-posix.c:106 #2 0x55b8eab186c8 in qemu_memalign /home/elmarco/src/qq/util/oslib-posix.c:122 #3 0x55b8e9d268a8 in spapr_reallocate_hpt /home/elmarco/src/qq/hw/ppc/spapr.c:1214 #4 0x55b8e9d26e04 in ppc_spapr_reset /home/elmarco/src/qq/hw/ppc/spapr.c:1261 #5 0x55b8ea12e913 in qemu_system_reset /home/elmarco/src/qq/vl.c:1697 #6 0x55b8ea13fa40 in main /home/elmarco/src/qq/vl.c:4679 #7 0x7f157d8e9400 in __libc_start_main (/lib64/libc.so.6+0x20400) Thread T6 created by T0 here: #0 0x7f159bae0488 in __interceptor_pthread_create (/lib64/libasan.so.3+0x31488) #1 0x55b8eab1d9cb in qemu_thread_create /home/elmarco/src/qq/util/qemu-thread-posix.c:465 #2 0x55b8ea67874c in migrate_fd_connect /home/elmarco/src/qq/migration/migration.c:2096 #3 0x55b8ea66cbb0 in migration_channel_connect /home/elmarco/src/qq/migration/migration.c:500 #4 0x55b8ea678f38 in socket_outgoing_migration /home/elmarco/src/qq/migration/socket.c:87 #5 0x55b8eaa5a03a in qio_task_complete /home/elmarco/src/qq/io/task.c:142 #6 0x55b8eaa599cc in gio_task_thread_result /home/elmarco/src/qq/io/task.c:88 #7 0x7f15823e38e6 (/lib64/libglib-2.0.so.0+0x468e6) SUMMARY: AddressSanitizer: heap-buffer-overflow /home/elmarco/src/qq/hw/ppc/spapr.c:1528 in htab_save_first_pass index seems to be wrongly incremented, unless I miss something that would be worth a comment. Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-03-23 11:04:55 +01:00
index++;
}
if (index > chunkstart) {
int n_valid = index - chunkstart;
htab_save_chunk(f, spapr, chunkstart, n_valid, 0);
if (has_timeout &&
(qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - starttime) > max_ns) {
break;
}
}
} while ((index < htabslots) && !qemu_file_rate_limit(f));
if (index >= htabslots) {
assert(index == htabslots);
index = 0;
spapr->htab_first_pass = false;
}
spapr->htab_save_index = index;
}
static int htab_save_later_pass(QEMUFile *f, sPAPRMachineState *spapr,
int64_t max_ns)
{
bool final = max_ns < 0;
int htabslots = HTAB_SIZE(spapr) / HASH_PTE_SIZE_64;
int examined = 0, sent = 0;
int index = spapr->htab_save_index;
int64_t starttime = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
assert(!spapr->htab_first_pass);
do {
int chunkstart, invalidstart;
/* Consume non-dirty HPTEs */
while ((index < htabslots)
&& !HPTE_DIRTY(HPTE(spapr->htab, index))) {
index++;
examined++;
}
chunkstart = index;
/* Consume valid dirty HPTEs */
while ((index < htabslots) && (index - chunkstart < USHRT_MAX)
&& HPTE_DIRTY(HPTE(spapr->htab, index))
&& HPTE_VALID(HPTE(spapr->htab, index))) {
CLEAN_HPTE(HPTE(spapr->htab, index));
index++;
examined++;
}
invalidstart = index;
/* Consume invalid dirty HPTEs */
while ((index < htabslots) && (index - invalidstart < USHRT_MAX)
&& HPTE_DIRTY(HPTE(spapr->htab, index))
&& !HPTE_VALID(HPTE(spapr->htab, index))) {
CLEAN_HPTE(HPTE(spapr->htab, index));
index++;
examined++;
}
if (index > chunkstart) {
int n_valid = invalidstart - chunkstart;
int n_invalid = index - invalidstart;
htab_save_chunk(f, spapr, chunkstart, n_valid, n_invalid);
sent += index - chunkstart;
if (!final && (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - starttime) > max_ns) {
break;
}
}
if (examined >= htabslots) {
break;
}
if (index >= htabslots) {
assert(index == htabslots);
index = 0;
}
} while ((examined < htabslots) && (!qemu_file_rate_limit(f) || final));
if (index >= htabslots) {
assert(index == htabslots);
index = 0;
}
spapr->htab_save_index = index;
return (examined >= htabslots) && (sent == 0) ? 1 : 0;
}
#define MAX_ITERATION_NS 5000000 /* 5 ms */
#define MAX_KVM_BUF_SIZE 2048
static int htab_save_iterate(QEMUFile *f, void *opaque)
{
sPAPRMachineState *spapr = opaque;
int fd;
int rc = 0;
/* Iteration header */
if (!spapr->htab_shift) {
qemu_put_be32(f, -1);
return 1;
} else {
qemu_put_be32(f, 0);
}
if (!spapr->htab) {
assert(kvm_enabled());
fd = get_htab_fd(spapr);
if (fd < 0) {
return fd;
}
rc = kvmppc_save_htab(f, fd, MAX_KVM_BUF_SIZE, MAX_ITERATION_NS);
if (rc < 0) {
return rc;
}
} else if (spapr->htab_first_pass) {
htab_save_first_pass(f, spapr, MAX_ITERATION_NS);
} else {
rc = htab_save_later_pass(f, spapr, MAX_ITERATION_NS);
}
htab_save_end_marker(f);
return rc;
}
static int htab_save_complete(QEMUFile *f, void *opaque)
{
sPAPRMachineState *spapr = opaque;
int fd;
/* Iteration header */
if (!spapr->htab_shift) {
qemu_put_be32(f, -1);
return 0;
} else {
qemu_put_be32(f, 0);
}
if (!spapr->htab) {
int rc;
assert(kvm_enabled());
fd = get_htab_fd(spapr);
if (fd < 0) {
return fd;
}
rc = kvmppc_save_htab(f, fd, MAX_KVM_BUF_SIZE, -1);
if (rc < 0) {
return rc;
}
} else {
if (spapr->htab_first_pass) {
htab_save_first_pass(f, spapr, -1);
}
htab_save_later_pass(f, spapr, -1);
}
/* End marker */
htab_save_end_marker(f);
return 0;
}
static int htab_load(QEMUFile *f, void *opaque, int version_id)
{
sPAPRMachineState *spapr = opaque;
uint32_t section_hdr;
int fd = -1;
Error *local_err = NULL;
if (version_id < 1 || version_id > 1) {
error_report("htab_load() bad version");
return -EINVAL;
}
section_hdr = qemu_get_be32(f);
if (section_hdr == -1) {
spapr_free_hpt(spapr);
return 0;
}
if (section_hdr) {
/* First section gives the htab size */
spapr_reallocate_hpt(spapr, section_hdr, &local_err);
if (local_err) {
error_report_err(local_err);
return -EINVAL;
}
return 0;
}
if (!spapr->htab) {
assert(kvm_enabled());
fd = kvmppc_get_htab_fd(true, 0, &local_err);
if (fd < 0) {
error_report_err(local_err);
return fd;
}
}
while (true) {
uint32_t index;
uint16_t n_valid, n_invalid;
index = qemu_get_be32(f);
n_valid = qemu_get_be16(f);
n_invalid = qemu_get_be16(f);
if ((index == 0) && (n_valid == 0) && (n_invalid == 0)) {
/* End of Stream */
break;
}
if ((index + n_valid + n_invalid) >
(HTAB_SIZE(spapr) / HASH_PTE_SIZE_64)) {
/* Bad index in stream */
error_report(
"htab_load() bad index %d (%hd+%hd entries) in htab stream (htab_shift=%d)",
index, n_valid, n_invalid, spapr->htab_shift);
return -EINVAL;
}
if (spapr->htab) {
if (n_valid) {
qemu_get_buffer(f, HPTE(spapr->htab, index),
HASH_PTE_SIZE_64 * n_valid);
}
if (n_invalid) {
memset(HPTE(spapr->htab, index + n_valid), 0,
HASH_PTE_SIZE_64 * n_invalid);
}
} else {
int rc;
assert(fd >= 0);
rc = kvmppc_load_htab_chunk(f, fd, index, n_valid, n_invalid);
if (rc < 0) {
return rc;
}
}
}
if (!spapr->htab) {
assert(fd >= 0);
close(fd);
}
return 0;
}
static void htab_save_cleanup(void *opaque)
{
sPAPRMachineState *spapr = opaque;
close_htab_fd(spapr);
}
static SaveVMHandlers savevm_htab_handlers = {
.save_setup = htab_save_setup,
.save_live_iterate = htab_save_iterate,
.save_live_complete_precopy = htab_save_complete,
.save_cleanup = htab_save_cleanup,
.load_state = htab_load,
};
static void spapr_boot_set(void *opaque, const char *boot_device,
Error **errp)
{
MachineState *machine = MACHINE(opaque);
machine->boot_order = g_strdup(boot_device);
}
static void spapr_create_lmb_dr_connectors(sPAPRMachineState *spapr)
{
MachineState *machine = MACHINE(spapr);
uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
uint32_t nr_lmbs = (machine->maxram_size - machine->ram_size)/lmb_size;
int i;
for (i = 0; i < nr_lmbs; i++) {
uint64_t addr;
addr = i * lmb_size + machine->device_memory->base;
spapr_dr_connector_new(OBJECT(spapr), TYPE_SPAPR_DRC_LMB,
addr / lmb_size);
}
}
/*
* If RAM size, maxmem size and individual node mem sizes aren't aligned
* to SPAPR_MEMORY_BLOCK_SIZE(256MB), then refuse to start the guest
* since we can't support such unaligned sizes with DRCONF_MEMORY.
*/
static void spapr_validate_node_memory(MachineState *machine, Error **errp)
{
int i;
if (machine->ram_size % SPAPR_MEMORY_BLOCK_SIZE) {
error_setg(errp, "Memory size 0x" RAM_ADDR_FMT
" is not aligned to %" PRIu64 " MiB",
machine->ram_size,
SPAPR_MEMORY_BLOCK_SIZE / MiB);
return;
}
if (machine->maxram_size % SPAPR_MEMORY_BLOCK_SIZE) {
error_setg(errp, "Maximum memory size 0x" RAM_ADDR_FMT
" is not aligned to %" PRIu64 " MiB",
machine->ram_size,
SPAPR_MEMORY_BLOCK_SIZE / MiB);
return;
}
for (i = 0; i < nb_numa_nodes; i++) {
if (numa_info[i].node_mem % SPAPR_MEMORY_BLOCK_SIZE) {
error_setg(errp,
"Node %d memory size 0x%" PRIx64
" is not aligned to %" PRIu64 " MiB",
i, numa_info[i].node_mem,
SPAPR_MEMORY_BLOCK_SIZE / MiB);
return;
}
}
}
/* find cpu slot in machine->possible_cpus by core_id */
static CPUArchId *spapr_find_cpu_slot(MachineState *ms, uint32_t id, int *idx)
{
int index = id / smp_threads;
if (index >= ms->possible_cpus->len) {
return NULL;
}
if (idx) {
*idx = index;
}
return &ms->possible_cpus->cpus[index];
}
static void spapr_set_vsmt_mode(sPAPRMachineState *spapr, Error **errp)
{
Error *local_err = NULL;
bool vsmt_user = !!spapr->vsmt;
int kvm_smt = kvmppc_smt_threads();
int ret;
if (!kvm_enabled() && (smp_threads > 1)) {
error_setg(&local_err, "TCG cannot support more than 1 thread/core "
"on a pseries machine");
goto out;
}
if (!is_power_of_2(smp_threads)) {
error_setg(&local_err, "Cannot support %d threads/core on a pseries "
"machine because it must be a power of 2", smp_threads);
goto out;
}
/* Detemine the VSMT mode to use: */
if (vsmt_user) {
if (spapr->vsmt < smp_threads) {
error_setg(&local_err, "Cannot support VSMT mode %d"
" because it must be >= threads/core (%d)",
spapr->vsmt, smp_threads);
goto out;
}
/* In this case, spapr->vsmt has been set by the command line */
} else {
spapr: Adjust default VSMT value for better migration compatibility fa98fbfc "PC: KVM: Support machine option to set VSMT mode" introduced the "vsmt" parameter for the pseries machine type, which controls the spacing of the vcpu ids of thread 0 for each virtual core. This was done to bring some consistency and stability to how that was done, while still allowing backwards compatibility for migration and otherwise. The default value we used for vsmt was set to the max of the host's advertised default number of threads and the number of vthreads per vcore in the guest. This was done to continue running without extra parameters on older KVM versions which don't allow the VSMT value to be changed. Unfortunately, even that smaller than before leakage of host configuration into guest visible configuration still breaks things. Specifically a guest with 4 (or less) vthread/vcore will get a different vsmt value when running on a POWER8 (vsmt==8) and POWER9 (vsmt==4) host. That means the vcpu ids don't line up so you can't migrate between them, though you should be able to. Long term we really want to make vsmt == smp_threads for sufficiently new machine types. However, that means that qemu will then require a sufficiently recent KVM (one which supports changing VSMT) - that's still not widely enough deployed to be really comfortable to do. In the meantime we need some default that will work as often as possible. This patch changes that default to 8 in all circumstances. This does change guest visible behaviour (including for existing machine versions) for many cases - just not the most common/important case. Following is case by case justification for why this is still the least worst option. Note that any of the old behaviours can still be duplicated after this patch, it's just that it requires manual intervention by setting the vsmt property on the command line. KVM HV on POWER8 host: This is the overwhelmingly common case in production setups, and is unchanged by design. POWER8 hosts will advertise a default VSMT mode of 8, and > 8 vthreads/vcore isn't permitted KVM HV on POWER7 host: Will break, but POWER7s allowing KVM were never released to the public. KVM HV on POWER9 host: Not yet released to the public, breaking this now will reduce other breakage later. KVM HV on PowerPC 970: Will theoretically break it, but it was barely supported to begin with and already required various user visible hacks to work. Also so old that I just don't care. TCG: This is the nastiest one; it means migration of TCG guests (without manual vsmt setting) will break. Since TCG is rarely used in production I think this is worth it for the other benefits. It does also remove one more barrier to TCG<->KVM migration which could be interesting for debugging applications. KVM PR: As with TCG, this will break migration of existing configurations, without adding extra manual vsmt options. As with TCG, it is rare in production so I think the benefits outweigh breakages. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com> Reviewed-by: Jose Ricardo Ziviani <joserz@linux.vnet.ibm.com> Reviewed-by: Greg Kurz <groug@kaod.org>
2018-01-15 07:51:33 +01:00
/*
* Default VSMT value is tricky, because we need it to be as
* consistent as possible (for migration), but this requires
* changing it for at least some existing cases. We pick 8 as
* the value that we'd get with KVM on POWER8, the
* overwhelmingly common case in production systems.
*/
spapr->vsmt = MAX(8, smp_threads);
}
/* KVM: If necessary, set the SMT mode: */
if (kvm_enabled() && (spapr->vsmt != kvm_smt)) {
ret = kvmppc_set_smt_threads(spapr->vsmt);
if (ret) {
/* Looks like KVM isn't able to change VSMT mode */
error_setg(&local_err,
"Failed to set KVM's VSMT mode to %d (errno %d)",
spapr->vsmt, ret);
/* We can live with that if the default one is big enough
* for the number of threads, and a submultiple of the one
* we want. In this case we'll waste some vcpu ids, but
* behaviour will be correct */
if ((kvm_smt >= smp_threads) && ((spapr->vsmt % kvm_smt) == 0)) {
warn_report_err(local_err);
local_err = NULL;
goto out;
} else {
if (!vsmt_user) {
error_append_hint(&local_err,
"On PPC, a VM with %d threads/core"
" on a host with %d threads/core"
" requires the use of VSMT mode %d.\n",
smp_threads, kvm_smt, spapr->vsmt);
}
kvmppc_hint_smt_possible(&local_err);
goto out;
}
}
}
/* else TCG: nothing to do currently */
out:
error_propagate(errp, local_err);
}
static void spapr_init_cpus(sPAPRMachineState *spapr)
{
MachineState *machine = MACHINE(spapr);
MachineClass *mc = MACHINE_GET_CLASS(machine);
sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine);
const char *type = spapr_get_cpu_core_type(machine->cpu_type);
const CPUArchIdList *possible_cpus;
int boot_cores_nr = smp_cpus / smp_threads;
int i;
possible_cpus = mc->possible_cpu_arch_ids(machine);
if (mc->has_hotpluggable_cpus) {
if (smp_cpus % smp_threads) {
error_report("smp_cpus (%u) must be multiple of threads (%u)",
smp_cpus, smp_threads);
exit(1);
}
if (max_cpus % smp_threads) {
error_report("max_cpus (%u) must be multiple of threads (%u)",
max_cpus, smp_threads);
exit(1);
}
} else {
if (max_cpus != smp_cpus) {
error_report("This machine version does not support CPU hotplug");
exit(1);
}
boot_cores_nr = possible_cpus->len;
}
if (smc->pre_2_10_has_unused_icps) {
int i;
for (i = 0; i < spapr_max_server_number(spapr); i++) {
/* Dummy entries get deregistered when real ICPState objects
* are registered during CPU core hotplug.
*/
pre_2_10_vmstate_register_dummy_icp(i);
}
}
for (i = 0; i < possible_cpus->len; i++) {
int core_id = i * smp_threads;
if (mc->has_hotpluggable_cpus) {
spapr_dr_connector_new(OBJECT(spapr), TYPE_SPAPR_DRC_CPU,
spapr_vcpu_id(spapr, core_id));
}
if (i < boot_cores_nr) {
Object *core = object_new(type);
int nr_threads = smp_threads;
/* Handle the partially filled core for older machine types */
if ((i + 1) * smp_threads >= smp_cpus) {
nr_threads = smp_cpus - i * smp_threads;
}
object_property_set_int(core, nr_threads, "nr-threads",
&error_fatal);
object_property_set_int(core, core_id, CPU_CORE_PROP_CORE_ID,
&error_fatal);
object_property_set_bool(core, true, "realized", &error_fatal);
object_unref(core);
}
}
}
static PCIHostState *spapr_create_default_phb(void)
{
DeviceState *dev;
dev = qdev_create(NULL, TYPE_SPAPR_PCI_HOST_BRIDGE);
qdev_prop_set_uint32(dev, "index", 0);
qdev_init_nofail(dev);
return PCI_HOST_BRIDGE(dev);
}
/* pSeries LPAR / sPAPR hardware init */
static void spapr_machine_init(MachineState *machine)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine);
const char *kernel_filename = machine->kernel_filename;
const char *initrd_filename = machine->initrd_filename;
PCIHostState *phb;
int i;
MemoryRegion *sysmem = get_system_memory();
MemoryRegion *ram = g_new(MemoryRegion, 1);
hwaddr node0_size = spapr_node0_size(machine);
long load_limit, fw_size;
char *filename;
Error *resize_hpt_err = NULL;
msi_nonbroken = true;
QLIST_INIT(&spapr->phbs);
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
QTAILQ_INIT(&spapr->pending_dimm_unplugs);
/* Determine capabilities to run with */
spapr_caps_init(spapr);
kvmppc_check_papr_resize_hpt(&resize_hpt_err);
if (spapr->resize_hpt == SPAPR_RESIZE_HPT_DEFAULT) {
/*
* If the user explicitly requested a mode we should either
* supply it, or fail completely (which we do below). But if
* it's not set explicitly, we reset our mode to something
* that works
*/
if (resize_hpt_err) {
spapr->resize_hpt = SPAPR_RESIZE_HPT_DISABLED;
error_free(resize_hpt_err);
resize_hpt_err = NULL;
} else {
spapr->resize_hpt = smc->resize_hpt_default;
}
}
assert(spapr->resize_hpt != SPAPR_RESIZE_HPT_DEFAULT);
if ((spapr->resize_hpt != SPAPR_RESIZE_HPT_DISABLED) && resize_hpt_err) {
/*
* User requested HPT resize, but this host can't supply it. Bail out
*/
error_report_err(resize_hpt_err);
exit(1);
}
spapr->rma_size = node0_size;
/* With KVM, we don't actually know whether KVM supports an
* unbounded RMA (PR KVM) or is limited by the hash table size
* (HV KVM using VRMA), so we always assume the latter
*
* In that case, we also limit the initial allocations for RTAS
* etc... to 256M since we have no way to know what the VRMA size
* is going to be as it depends on the size of the hash table
* which isn't determined yet.
*/
if (kvm_enabled()) {
spapr->vrma_adjust = 1;
spapr->rma_size = MIN(spapr->rma_size, 0x10000000);
}
/* Actually we don't support unbounded RMA anymore since we added
* proper emulation of HV mode. The max we can get is 16G which
* also happens to be what we configure for PAPR mode so make sure
* we don't do anything bigger than that
*/
spapr->rma_size = MIN(spapr->rma_size, 0x400000000ull);
if (spapr->rma_size > node0_size) {
error_report("Numa node 0 has to span the RMA (%#08"HWADDR_PRIx")",
spapr->rma_size);
exit(1);
}
/* Setup a load limit for the ramdisk leaving room for SLOF and FDT */
load_limit = MIN(spapr->rma_size, RTAS_MAX_ADDR) - FW_OVERHEAD;
/*
* VSMT must be set in order to be able to compute VCPU ids, ie to
* call spapr_max_server_number() or spapr_vcpu_id().
*/
spapr_set_vsmt_mode(spapr, &error_fatal);
/* Set up Interrupt Controller before we create the VCPUs */
spapr_irq_init(spapr, &error_fatal);
/* Set up containers for ibm,client-architecture-support negotiated options
*/
spapr->ov5 = spapr_ovec_new();
spapr->ov5_cas = spapr_ovec_new();
if (smc->dr_lmb_enabled) {
spapr_ovec_set(spapr->ov5, OV5_DRCONF_MEMORY);
spapr_validate_node_memory(machine, &error_fatal);
}
spapr_ovec_set(spapr->ov5, OV5_FORM1_AFFINITY);
/* advertise support for dedicated HP event source to guests */
if (spapr->use_hotplug_event_source) {
spapr_ovec_set(spapr->ov5, OV5_HP_EVT);
}
/* advertise support for HPT resizing */
if (spapr->resize_hpt != SPAPR_RESIZE_HPT_DISABLED) {
spapr_ovec_set(spapr->ov5, OV5_HPT_RESIZE);
}
/* advertise support for ibm,dyamic-memory-v2 */
spapr_ovec_set(spapr->ov5, OV5_DRMEM_V2);
/* advertise XIVE on POWER9 machines */
if (spapr->irq->ov5 & (SPAPR_OV5_XIVE_EXPLOIT | SPAPR_OV5_XIVE_BOTH)) {
if (ppc_type_check_compat(machine->cpu_type, CPU_POWERPC_LOGICAL_3_00,
0, spapr->max_compat_pvr)) {
spapr_ovec_set(spapr->ov5, OV5_XIVE_EXPLOIT);
} else if (spapr->irq->ov5 & SPAPR_OV5_XIVE_EXPLOIT) {
error_report("XIVE-only machines require a POWER9 CPU");
exit(1);
}
}
/* init CPUs */
spapr_init_cpus(spapr);
if ((!kvm_enabled() || kvmppc_has_cap_mmu_radix()) &&
ppc_type_check_compat(machine->cpu_type, CPU_POWERPC_LOGICAL_3_00, 0,
spapr->max_compat_pvr)) {
/* KVM and TCG always allow GTSE with radix... */
spapr_ovec_set(spapr->ov5, OV5_MMU_RADIX_GTSE);
}
/* ... but not with hash (currently). */
if (kvm_enabled()) {
/* Enable H_LOGICAL_CI_* so SLOF can talk to in-kernel devices */
kvmppc_enable_logical_ci_hcalls();
kvmppc_enable_set_mode_hcall();
/* H_CLEAR_MOD/_REF are mandatory in PAPR, but off by default */
kvmppc_enable_clear_ref_mod_hcalls();
}
/* allocate RAM */
memory_region_allocate_system_memory(ram, NULL, "ppc_spapr.ram",
machine->ram_size);
memory_region_add_subregion(sysmem, 0, ram);
/* always allocate the device memory information */
machine->device_memory = g_malloc0(sizeof(*machine->device_memory));
/* initialize hotplug memory address space */
if (machine->ram_size < machine->maxram_size) {
ram_addr_t device_mem_size = machine->maxram_size - machine->ram_size;
/*
* Limit the number of hotpluggable memory slots to half the number
* slots that KVM supports, leaving the other half for PCI and other
* devices. However ensure that number of slots doesn't drop below 32.
*/
int max_memslots = kvm_enabled() ? kvm_get_max_memslots() / 2 :
SPAPR_MAX_RAM_SLOTS;
if (max_memslots < SPAPR_MAX_RAM_SLOTS) {
max_memslots = SPAPR_MAX_RAM_SLOTS;
}
if (machine->ram_slots > max_memslots) {
error_report("Specified number of memory slots %"
PRIu64" exceeds max supported %d",
machine->ram_slots, max_memslots);
exit(1);
}
machine->device_memory->base = ROUND_UP(machine->ram_size,
SPAPR_DEVICE_MEM_ALIGN);
memory_region_init(&machine->device_memory->mr, OBJECT(spapr),
"device-memory", device_mem_size);
memory_region_add_subregion(sysmem, machine->device_memory->base,
&machine->device_memory->mr);
}
if (smc->dr_lmb_enabled) {
spapr_create_lmb_dr_connectors(spapr);
}
filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, "spapr-rtas.bin");
if (!filename) {
error_report("Could not find LPAR rtas '%s'", "spapr-rtas.bin");
exit(1);
}
spapr->rtas_size = get_image_size(filename);
if (spapr->rtas_size < 0) {
error_report("Could not get size of LPAR rtas '%s'", filename);
exit(1);
}
spapr->rtas_blob = g_malloc(spapr->rtas_size);
if (load_image_size(filename, spapr->rtas_blob, spapr->rtas_size) < 0) {
error_report("Could not load LPAR rtas '%s'", filename);
exit(1);
}
if (spapr->rtas_size > RTAS_MAX_SIZE) {
error_report("RTAS too big ! 0x%zx bytes (max is 0x%x)",
(size_t)spapr->rtas_size, RTAS_MAX_SIZE);
exit(1);
}
g_free(filename);
/* Set up RTAS event infrastructure */
spapr_events_init(spapr);
/* Set up the RTC RTAS interfaces */
spapr_rtc_create(spapr);
Implement the PAPR (pSeries) virtualized interrupt controller (xics) PAPR defines an interrupt control architecture which is logically divided into ICS (Interrupt Control Presentation, each unit is responsible for presenting interrupts to a particular "interrupt server", i.e. CPU) and ICS (Interrupt Control Source, each unit responsible for one or more hardware interrupts as numbered globally across the system). All PAPR virtual IO devices expect to deliver interrupts via this mechanism. In Linux, this interrupt controller system is handled by the "xics" driver. On pSeries systems, access to the interrupt controller is virtualized via hypercalls and RTAS methods. However, the virtualized interface is very similar to the underlying interrupt controller hardware, and similar PICs exist un-virtualized in some other systems. This patch implements both the ICP and ICS sides of the PAPR interrupt controller. For now, only the hypercall virtualized interface is provided, however it would be relatively straightforward to graft an emulated register interface onto the underlying interrupt logic if we want to add a machine with a hardware ICS/ICP system in the future. There are some limitations in this implementation: it is assumed for now that only one instance of the ICS exists, although a full xics system can have several, each responsible for a different group of hardware irqs. ICP/ICS can handle both level-sensitve (LSI) and message signalled (MSI) interrupt inputs. For now, this implementation supports only MSI interrupts, since that is used by PAPR virtual IO devices. Signed-off-by: Paul Mackerras <paulus@samba.org> Signed-off-by: David Gibson <dwg@au1.ibm.com> Signed-off-by: Alexander Graf <agraf@suse.de>
2011-04-01 06:15:25 +02:00
/* Set up VIO bus */
spapr->vio_bus = spapr_vio_bus_init();
for (i = 0; i < serial_max_hds(); i++) {
if (serial_hd(i)) {
spapr_vty_create(spapr->vio_bus, serial_hd(i));
}
}
/* We always have at least the nvram device on VIO */
spapr_create_nvram(spapr);
/*
* Setup hotplug / dynamic-reconfiguration connectors. top-level
* connectors (described in root DT node's "ibm,drc-types" property)
* are pre-initialized here. additional child connectors (such as
* connectors for a PHBs PCI slots) are added as needed during their
* parent's realization.
*/
if (smc->dr_phb_enabled) {
for (i = 0; i < SPAPR_MAX_PHBS; i++) {
spapr_dr_connector_new(OBJECT(machine), TYPE_SPAPR_DRC_PHB, i);
}
}
/* Set up PCI */
spapr_pci_rtas_init();
phb = spapr_create_default_phb();
for (i = 0; i < nb_nics; i++) {
NICInfo *nd = &nd_table[i];
if (!nd->model) {
nd->model = g_strdup("spapr-vlan");
}
if (g_str_equal(nd->model, "spapr-vlan") ||
g_str_equal(nd->model, "ibmveth")) {
spapr_vlan_create(spapr->vio_bus, nd);
} else {
pci_nic_init_nofail(&nd_table[i], phb->bus, nd->model, NULL);
}
}
for (i = 0; i <= drive_get_max_bus(IF_SCSI); i++) {
spapr_vscsi_create(spapr->vio_bus);
}
/* Graphics */
if (spapr_vga_init(phb->bus, &error_fatal)) {
spapr->has_graphics = true;
machine->usb |= defaults_enabled() && !machine->usb_disabled;
}
if (machine->usb) {
if (smc->use_ohci_by_default) {
pci_create_simple(phb->bus, -1, "pci-ohci");
} else {
pci_create_simple(phb->bus, -1, "nec-usb-xhci");
}
if (spapr->has_graphics) {
USBBus *usb_bus = usb_bus_find(-1);
usb_create_simple(usb_bus, "usb-kbd");
usb_create_simple(usb_bus, "usb-mouse");
}
}
if (spapr->rma_size < (MIN_RMA_SLOF * MiB)) {
error_report(
"pSeries SLOF firmware requires >= %ldM guest RMA (Real Mode Area memory)",
MIN_RMA_SLOF);
exit(1);
}
if (kernel_filename) {
uint64_t lowaddr = 0;
spapr->kernel_size = load_elf(kernel_filename, NULL,
translate_kernel_address, NULL,
NULL, &lowaddr, NULL, 1,
PPC_ELF_MACHINE, 0, 0);
if (spapr->kernel_size == ELF_LOAD_WRONG_ENDIAN) {
spapr->kernel_size = load_elf(kernel_filename, NULL,
translate_kernel_address, NULL, NULL,
&lowaddr, NULL, 0, PPC_ELF_MACHINE,
0, 0);
spapr->kernel_le = spapr->kernel_size > 0;
}
if (spapr->kernel_size < 0) {
error_report("error loading %s: %s", kernel_filename,
load_elf_strerror(spapr->kernel_size));
exit(1);
}
/* load initrd */
if (initrd_filename) {
/* Try to locate the initrd in the gap between the kernel
* and the firmware. Add a bit of space just in case
*/
spapr->initrd_base = (KERNEL_LOAD_ADDR + spapr->kernel_size
+ 0x1ffff) & ~0xffff;
spapr->initrd_size = load_image_targphys(initrd_filename,
spapr->initrd_base,
load_limit
- spapr->initrd_base);
if (spapr->initrd_size < 0) {
error_report("could not load initial ram disk '%s'",
initrd_filename);
exit(1);
}
}
}
if (bios_name == NULL) {
bios_name = FW_FILE_NAME;
}
filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
if (!filename) {
error_report("Could not find LPAR firmware '%s'", bios_name);
exit(1);
}
fw_size = load_image_targphys(filename, 0, FW_MAX_SIZE);
if (fw_size <= 0) {
error_report("Could not load LPAR firmware '%s'", filename);
exit(1);
}
g_free(filename);
/* FIXME: Should register things through the MachineState's qdev
* interface, this is a legacy from the sPAPREnvironment structure
* which predated MachineState but had a similar function */
vmstate_register(NULL, 0, &vmstate_spapr, spapr);
register_savevm_live(NULL, "spapr/htab", -1, 1,
&savevm_htab_handlers, spapr);
qbus_set_hotplug_handler(sysbus_get_default(), OBJECT(machine),
&error_fatal);
qemu_register_boot_set(spapr_boot_set, spapr);
if (kvm_enabled()) {
/* to stop and start vmclock */
qemu_add_vm_change_state_handler(cpu_ppc_clock_vm_state_change,
&spapr->tb);
kvmppc_spapr_enable_inkernel_multitce();
}
}
static int spapr_kvm_type(MachineState *machine, const char *vm_type)
{
if (!vm_type) {
return 0;
}
if (!strcmp(vm_type, "HV")) {
return 1;
}
if (!strcmp(vm_type, "PR")) {
return 2;
}
error_report("Unknown kvm-type specified '%s'", vm_type);
exit(1);
}
/*
* Implementation of an interface to adjust firmware path
* for the bootindex property handling.
*/
static char *spapr_get_fw_dev_path(FWPathProvider *p, BusState *bus,
DeviceState *dev)
{
#define CAST(type, obj, name) \
((type *)object_dynamic_cast(OBJECT(obj), (name)))
SCSIDevice *d = CAST(SCSIDevice, dev, TYPE_SCSI_DEVICE);
sPAPRPHBState *phb = CAST(sPAPRPHBState, dev, TYPE_SPAPR_PCI_HOST_BRIDGE);
VHostSCSICommon *vsc = CAST(VHostSCSICommon, dev, TYPE_VHOST_SCSI_COMMON);
if (d) {
void *spapr = CAST(void, bus->parent, "spapr-vscsi");
VirtIOSCSI *virtio = CAST(VirtIOSCSI, bus->parent, TYPE_VIRTIO_SCSI);
USBDevice *usb = CAST(USBDevice, bus->parent, TYPE_USB_DEVICE);
if (spapr) {
/*
* Replace "channel@0/disk@0,0" with "disk@8000000000000000":
* In the top 16 bits of the 64-bit LUN, we use SRP luns of the form
* 0x8000 | (target << 8) | (bus << 5) | lun
* (see the "Logical unit addressing format" table in SAM5)
*/
unsigned id = 0x8000 | (d->id << 8) | (d->channel << 5) | d->lun;
return g_strdup_printf("%s@%"PRIX64, qdev_fw_name(dev),
(uint64_t)id << 48);
} else if (virtio) {
/*
* We use SRP luns of the form 01000000 | (target << 8) | lun
* in the top 32 bits of the 64-bit LUN
* Note: the quote above is from SLOF and it is wrong,
* the actual binding is:
* swap 0100 or 10 << or 20 << ( target lun-id -- srplun )
*/
unsigned id = 0x1000000 | (d->id << 16) | d->lun;
if (d->lun >= 256) {
/* Use the LUN "flat space addressing method" */
id |= 0x4000;
}
return g_strdup_printf("%s@%"PRIX64, qdev_fw_name(dev),
(uint64_t)id << 32);
} else if (usb) {
/*
* We use SRP luns of the form 01000000 | (usb-port << 16) | lun
* in the top 32 bits of the 64-bit LUN
*/
unsigned usb_port = atoi(usb->port->path);
unsigned id = 0x1000000 | (usb_port << 16) | d->lun;
return g_strdup_printf("%s@%"PRIX64, qdev_fw_name(dev),
(uint64_t)id << 32);
}
}
/*
* SLOF probes the USB devices, and if it recognizes that the device is a
* storage device, it changes its name to "storage" instead of "usb-host",
* and additionally adds a child node for the SCSI LUN, so the correct
* boot path in SLOF is something like .../storage@1/disk@xxx" instead.
*/
if (strcmp("usb-host", qdev_fw_name(dev)) == 0) {
USBDevice *usbdev = CAST(USBDevice, dev, TYPE_USB_DEVICE);
if (usb_host_dev_is_scsi_storage(usbdev)) {
return g_strdup_printf("storage@%s/disk", usbdev->port->path);
}
}
if (phb) {
/* Replace "pci" with "pci@800000020000000" */
return g_strdup_printf("pci@%"PRIX64, phb->buid);
}
if (vsc) {
/* Same logic as virtio above */
unsigned id = 0x1000000 | (vsc->target << 16) | vsc->lun;
return g_strdup_printf("disk@%"PRIX64, (uint64_t)id << 32);
}
if (g_str_equal("pci-bridge", qdev_fw_name(dev))) {
/* SLOF uses "pci" instead of "pci-bridge" for PCI bridges */
PCIDevice *pcidev = CAST(PCIDevice, dev, TYPE_PCI_DEVICE);
return g_strdup_printf("pci@%x", PCI_SLOT(pcidev->devfn));
}
return NULL;
}
static char *spapr_get_kvm_type(Object *obj, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
return g_strdup(spapr->kvm_type);
}
static void spapr_set_kvm_type(Object *obj, const char *value, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
g_free(spapr->kvm_type);
spapr->kvm_type = g_strdup(value);
}
static bool spapr_get_modern_hotplug_events(Object *obj, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
return spapr->use_hotplug_event_source;
}
static void spapr_set_modern_hotplug_events(Object *obj, bool value,
Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
spapr->use_hotplug_event_source = value;
}
static bool spapr_get_msix_emulation(Object *obj, Error **errp)
{
return true;
}
static char *spapr_get_resize_hpt(Object *obj, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
switch (spapr->resize_hpt) {
case SPAPR_RESIZE_HPT_DEFAULT:
return g_strdup("default");
case SPAPR_RESIZE_HPT_DISABLED:
return g_strdup("disabled");
case SPAPR_RESIZE_HPT_ENABLED:
return g_strdup("enabled");
case SPAPR_RESIZE_HPT_REQUIRED:
return g_strdup("required");
}
g_assert_not_reached();
}
static void spapr_set_resize_hpt(Object *obj, const char *value, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
if (strcmp(value, "default") == 0) {
spapr->resize_hpt = SPAPR_RESIZE_HPT_DEFAULT;
} else if (strcmp(value, "disabled") == 0) {
spapr->resize_hpt = SPAPR_RESIZE_HPT_DISABLED;
} else if (strcmp(value, "enabled") == 0) {
spapr->resize_hpt = SPAPR_RESIZE_HPT_ENABLED;
} else if (strcmp(value, "required") == 0) {
spapr->resize_hpt = SPAPR_RESIZE_HPT_REQUIRED;
} else {
error_setg(errp, "Bad value for \"resize-hpt\" property");
}
}
static void spapr_get_vsmt(Object *obj, Visitor *v, const char *name,
void *opaque, Error **errp)
{
visit_type_uint32(v, name, (uint32_t *)opaque, errp);
}
static void spapr_set_vsmt(Object *obj, Visitor *v, const char *name,
void *opaque, Error **errp)
{
visit_type_uint32(v, name, (uint32_t *)opaque, errp);
}
static char *spapr_get_ic_mode(Object *obj, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
if (spapr->irq == &spapr_irq_xics_legacy) {
return g_strdup("legacy");
} else if (spapr->irq == &spapr_irq_xics) {
return g_strdup("xics");
} else if (spapr->irq == &spapr_irq_xive) {
return g_strdup("xive");
} else if (spapr->irq == &spapr_irq_dual) {
return g_strdup("dual");
}
g_assert_not_reached();
}
static void spapr_set_ic_mode(Object *obj, const char *value, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
if (SPAPR_MACHINE_GET_CLASS(spapr)->legacy_irq_allocation) {
error_setg(errp, "This machine only uses the legacy XICS backend, don't pass ic-mode");
return;
}
/* The legacy IRQ backend can not be set */
if (strcmp(value, "xics") == 0) {
spapr->irq = &spapr_irq_xics;
} else if (strcmp(value, "xive") == 0) {
spapr->irq = &spapr_irq_xive;
} else if (strcmp(value, "dual") == 0) {
spapr->irq = &spapr_irq_dual;
} else {
error_setg(errp, "Bad value for \"ic-mode\" property");
}
}
static char *spapr_get_host_model(Object *obj, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
return g_strdup(spapr->host_model);
}
static void spapr_set_host_model(Object *obj, const char *value, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
g_free(spapr->host_model);
spapr->host_model = g_strdup(value);
}
static char *spapr_get_host_serial(Object *obj, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
return g_strdup(spapr->host_serial);
}
static void spapr_set_host_serial(Object *obj, const char *value, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
g_free(spapr->host_serial);
spapr->host_serial = g_strdup(value);
}
static void spapr_instance_init(Object *obj)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
spapr->htab_fd = -1;
spapr->use_hotplug_event_source = true;
object_property_add_str(obj, "kvm-type",
spapr_get_kvm_type, spapr_set_kvm_type, NULL);
object_property_set_description(obj, "kvm-type",
"Specifies the KVM virtualization mode (HV, PR)",
NULL);
object_property_add_bool(obj, "modern-hotplug-events",
spapr_get_modern_hotplug_events,
spapr_set_modern_hotplug_events,
NULL);
object_property_set_description(obj, "modern-hotplug-events",
"Use dedicated hotplug event mechanism in"
" place of standard EPOW events when possible"
" (required for memory hot-unplug support)",
NULL);
ppc_compat_add_property(obj, "max-cpu-compat", &spapr->max_compat_pvr,
"Maximum permitted CPU compatibility mode",
&error_fatal);
object_property_add_str(obj, "resize-hpt",
spapr_get_resize_hpt, spapr_set_resize_hpt, NULL);
object_property_set_description(obj, "resize-hpt",
"Resizing of the Hash Page Table (enabled, disabled, required)",
NULL);
object_property_add(obj, "vsmt", "uint32", spapr_get_vsmt,
spapr_set_vsmt, NULL, &spapr->vsmt, &error_abort);
object_property_set_description(obj, "vsmt",
"Virtual SMT: KVM behaves as if this were"
" the host's SMT mode", &error_abort);
object_property_add_bool(obj, "vfio-no-msix-emulation",
spapr_get_msix_emulation, NULL, NULL);
/* The machine class defines the default interrupt controller mode */
spapr->irq = smc->irq;
object_property_add_str(obj, "ic-mode", spapr_get_ic_mode,
spapr_set_ic_mode, NULL);
object_property_set_description(obj, "ic-mode",
"Specifies the interrupt controller mode (xics, xive, dual)",
NULL);
object_property_add_str(obj, "host-model",
spapr_get_host_model, spapr_set_host_model,
&error_abort);
object_property_set_description(obj, "host-model",
"Set host's model-id to use - none|passthrough|string", &error_abort);
object_property_add_str(obj, "host-serial",
spapr_get_host_serial, spapr_set_host_serial,
&error_abort);
object_property_set_description(obj, "host-serial",
"Set host's system-id to use - none|passthrough|string", &error_abort);
}
static void spapr_machine_finalizefn(Object *obj)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
g_free(spapr->kvm_type);
}
void spapr_do_system_reset_on_cpu(CPUState *cs, run_on_cpu_data arg)
{
cpu_synchronize_state(cs);
ppc_cpu_do_system_reset(cs);
}
static void spapr_nmi(NMIState *n, int cpu_index, Error **errp)
{
CPUState *cs;
CPU_FOREACH(cs) {
async_run_on_cpu(cs, spapr_do_system_reset_on_cpu, RUN_ON_CPU_NULL);
}
}
int spapr_lmb_dt_populate(sPAPRDRConnector *drc, sPAPRMachineState *spapr,
void *fdt, int *fdt_start_offset, Error **errp)
{
uint64_t addr;
uint32_t node;
addr = spapr_drc_index(drc) * SPAPR_MEMORY_BLOCK_SIZE;
node = object_property_get_uint(OBJECT(drc->dev), PC_DIMM_NODE_PROP,
&error_abort);
*fdt_start_offset = spapr_populate_memory_node(fdt, node, addr,
SPAPR_MEMORY_BLOCK_SIZE);
return 0;
}
static void spapr_add_lmbs(DeviceState *dev, uint64_t addr_start, uint64_t size,
bool dedicated_hp_event_source, Error **errp)
{
sPAPRDRConnector *drc;
uint32_t nr_lmbs = size/SPAPR_MEMORY_BLOCK_SIZE;
int i;
uint64_t addr = addr_start;
spapr: Treat devices added before inbound migration as coldplugged When migrating a guest which has already had devices hotplugged, libvirt typically starts the destination qemu with -incoming defer, adds those hotplugged devices with qmp, then initiates the incoming migration. This causes problems for the management of spapr DRC state. Because the device is treated as hotplugged, it goes into a DRC state for a device immediately after it's plugged, but before the guest has acknowledged its presence. However, chances are the guest on the source machine *has* acknowledged the device's presence and configured it. If the source has fully configured the device, then DRC state won't be sent in the migration stream: for maximum migration compatibility with earlier versions we don't migrate DRCs in coldplug-equivalent state. That means that the DRC effectively changes state over the migrate, causing problems later on. In addition, logging hotplug events for these devices isn't what we want because a) those events should already have been issued on the source host and b) the event queue should get wiped out by the incoming state anyway. In short, what we really want is to treat devices added before an incoming migration as if they were coldplugged. To do this, we first add a spapr_drc_hotplugged() helper which determines if the device is hotplugged in the sense relevant for DRC state management. We only send hotplug events when this is true. Second, when we add a device which isn't hotplugged in this sense, we force a reset of the DRC state - this ensures the DRC is in a coldplug-equivalent state (there isn't usually a system reset between these device adds and the incoming migration). This is based on an earlier patch by Laurent Vivier, cleaned up and extended. Signed-off-by: Laurent Vivier <lvivier@redhat.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Greg Kurz <groug@kaod.org> Tested-by: Daniel Barboza <danielhb@linux.vnet.ibm.com>
2017-06-09 13:08:10 +02:00
bool hotplugged = spapr_drc_hotplugged(dev);
Error *local_err = NULL;
for (i = 0; i < nr_lmbs; i++) {
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB,
addr / SPAPR_MEMORY_BLOCK_SIZE);
g_assert(drc);
spapr_drc_attach(drc, dev, &local_err);
if (local_err) {
while (addr > addr_start) {
addr -= SPAPR_MEMORY_BLOCK_SIZE;
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB,
addr / SPAPR_MEMORY_BLOCK_SIZE);
spapr_drc_detach(drc);
}
error_propagate(errp, local_err);
return;
}
spapr: Treat devices added before inbound migration as coldplugged When migrating a guest which has already had devices hotplugged, libvirt typically starts the destination qemu with -incoming defer, adds those hotplugged devices with qmp, then initiates the incoming migration. This causes problems for the management of spapr DRC state. Because the device is treated as hotplugged, it goes into a DRC state for a device immediately after it's plugged, but before the guest has acknowledged its presence. However, chances are the guest on the source machine *has* acknowledged the device's presence and configured it. If the source has fully configured the device, then DRC state won't be sent in the migration stream: for maximum migration compatibility with earlier versions we don't migrate DRCs in coldplug-equivalent state. That means that the DRC effectively changes state over the migrate, causing problems later on. In addition, logging hotplug events for these devices isn't what we want because a) those events should already have been issued on the source host and b) the event queue should get wiped out by the incoming state anyway. In short, what we really want is to treat devices added before an incoming migration as if they were coldplugged. To do this, we first add a spapr_drc_hotplugged() helper which determines if the device is hotplugged in the sense relevant for DRC state management. We only send hotplug events when this is true. Second, when we add a device which isn't hotplugged in this sense, we force a reset of the DRC state - this ensures the DRC is in a coldplug-equivalent state (there isn't usually a system reset between these device adds and the incoming migration). This is based on an earlier patch by Laurent Vivier, cleaned up and extended. Signed-off-by: Laurent Vivier <lvivier@redhat.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Greg Kurz <groug@kaod.org> Tested-by: Daniel Barboza <danielhb@linux.vnet.ibm.com>
2017-06-09 13:08:10 +02:00
if (!hotplugged) {
spapr_drc_reset(drc);
}
addr += SPAPR_MEMORY_BLOCK_SIZE;
}
/* send hotplug notification to the
* guest only in case of hotplugged memory
*/
spapr: Treat devices added before inbound migration as coldplugged When migrating a guest which has already had devices hotplugged, libvirt typically starts the destination qemu with -incoming defer, adds those hotplugged devices with qmp, then initiates the incoming migration. This causes problems for the management of spapr DRC state. Because the device is treated as hotplugged, it goes into a DRC state for a device immediately after it's plugged, but before the guest has acknowledged its presence. However, chances are the guest on the source machine *has* acknowledged the device's presence and configured it. If the source has fully configured the device, then DRC state won't be sent in the migration stream: for maximum migration compatibility with earlier versions we don't migrate DRCs in coldplug-equivalent state. That means that the DRC effectively changes state over the migrate, causing problems later on. In addition, logging hotplug events for these devices isn't what we want because a) those events should already have been issued on the source host and b) the event queue should get wiped out by the incoming state anyway. In short, what we really want is to treat devices added before an incoming migration as if they were coldplugged. To do this, we first add a spapr_drc_hotplugged() helper which determines if the device is hotplugged in the sense relevant for DRC state management. We only send hotplug events when this is true. Second, when we add a device which isn't hotplugged in this sense, we force a reset of the DRC state - this ensures the DRC is in a coldplug-equivalent state (there isn't usually a system reset between these device adds and the incoming migration). This is based on an earlier patch by Laurent Vivier, cleaned up and extended. Signed-off-by: Laurent Vivier <lvivier@redhat.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Greg Kurz <groug@kaod.org> Tested-by: Daniel Barboza <danielhb@linux.vnet.ibm.com>
2017-06-09 13:08:10 +02:00
if (hotplugged) {
if (dedicated_hp_event_source) {
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB,
addr_start / SPAPR_MEMORY_BLOCK_SIZE);
spapr_hotplug_req_add_by_count_indexed(SPAPR_DR_CONNECTOR_TYPE_LMB,
nr_lmbs,
spapr_drc_index(drc));
} else {
spapr_hotplug_req_add_by_count(SPAPR_DR_CONNECTOR_TYPE_LMB,
nr_lmbs);
}
}
}
static void spapr_memory_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
Error **errp)
{
Error *local_err = NULL;
sPAPRMachineState *ms = SPAPR_MACHINE(hotplug_dev);
PCDIMMDevice *dimm = PC_DIMM(dev);
uint64_t size, addr;
size = memory_device_get_region_size(MEMORY_DEVICE(dev), &error_abort);
pc_dimm_plug(dimm, MACHINE(ms), &local_err);
if (local_err) {
goto out;
}
addr = object_property_get_uint(OBJECT(dimm),
PC_DIMM_ADDR_PROP, &local_err);
if (local_err) {
goto out_unplug;
}
spapr_add_lmbs(dev, addr, size, spapr_ovec_test(ms->ov5_cas, OV5_HP_EVT),
&local_err);
if (local_err) {
goto out_unplug;
}
return;
out_unplug:
pc_dimm_unplug(dimm, MACHINE(ms));
out:
error_propagate(errp, local_err);
}
static void spapr_memory_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
Error **errp)
{
const sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(hotplug_dev);
sPAPRMachineState *spapr = SPAPR_MACHINE(hotplug_dev);
PCDIMMDevice *dimm = PC_DIMM(dev);
Error *local_err = NULL;
uint64_t size;
Object *memdev;
hwaddr pagesize;
if (!smc->dr_lmb_enabled) {
error_setg(errp, "Memory hotplug not supported for this machine");
return;
}
size = memory_device_get_region_size(MEMORY_DEVICE(dimm), &local_err);
if (local_err) {
error_propagate(errp, local_err);
return;
}
if (size % SPAPR_MEMORY_BLOCK_SIZE) {
error_setg(errp, "Hotplugged memory size must be a multiple of "
"%" PRIu64 " MB", SPAPR_MEMORY_BLOCK_SIZE / MiB);
return;
}
memdev = object_property_get_link(OBJECT(dimm), PC_DIMM_MEMDEV_PROP,
&error_abort);
pagesize = host_memory_backend_pagesize(MEMORY_BACKEND(memdev));
spapr_check_pagesize(spapr, pagesize, &local_err);
if (local_err) {
error_propagate(errp, local_err);
return;
}
pc_dimm_pre_plug(dimm, MACHINE(hotplug_dev), NULL, errp);
}
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
struct sPAPRDIMMState {
PCDIMMDevice *dimm;
uint32_t nr_lmbs;
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
QTAILQ_ENTRY(sPAPRDIMMState) next;
};
static sPAPRDIMMState *spapr_pending_dimm_unplugs_find(sPAPRMachineState *s,
PCDIMMDevice *dimm)
{
sPAPRDIMMState *dimm_state = NULL;
QTAILQ_FOREACH(dimm_state, &s->pending_dimm_unplugs, next) {
if (dimm_state->dimm == dimm) {
break;
}
}
return dimm_state;
}
static sPAPRDIMMState *spapr_pending_dimm_unplugs_add(sPAPRMachineState *spapr,
uint32_t nr_lmbs,
PCDIMMDevice *dimm)
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
{
sPAPRDIMMState *ds = NULL;
/*
* If this request is for a DIMM whose removal had failed earlier
* (due to guest's refusal to remove the LMBs), we would have this
* dimm already in the pending_dimm_unplugs list. In that
* case don't add again.
*/
ds = spapr_pending_dimm_unplugs_find(spapr, dimm);
if (!ds) {
ds = g_malloc0(sizeof(sPAPRDIMMState));
ds->nr_lmbs = nr_lmbs;
ds->dimm = dimm;
QTAILQ_INSERT_HEAD(&spapr->pending_dimm_unplugs, ds, next);
}
return ds;
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
}
static void spapr_pending_dimm_unplugs_remove(sPAPRMachineState *spapr,
sPAPRDIMMState *dimm_state)
{
QTAILQ_REMOVE(&spapr->pending_dimm_unplugs, dimm_state, next);
g_free(dimm_state);
}
static sPAPRDIMMState *spapr_recover_pending_dimm_state(sPAPRMachineState *ms,
PCDIMMDevice *dimm)
{
sPAPRDRConnector *drc;
uint64_t size = memory_device_get_region_size(MEMORY_DEVICE(dimm),
&error_abort);
uint32_t nr_lmbs = size / SPAPR_MEMORY_BLOCK_SIZE;
uint32_t avail_lmbs = 0;
uint64_t addr_start, addr;
int i;
addr_start = object_property_get_int(OBJECT(dimm), PC_DIMM_ADDR_PROP,
&error_abort);
addr = addr_start;
for (i = 0; i < nr_lmbs; i++) {
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB,
addr / SPAPR_MEMORY_BLOCK_SIZE);
g_assert(drc);
if (drc->dev) {
avail_lmbs++;
}
addr += SPAPR_MEMORY_BLOCK_SIZE;
}
return spapr_pending_dimm_unplugs_add(ms, avail_lmbs, dimm);
}
/* Callback to be called during DRC release. */
void spapr_lmb_release(DeviceState *dev)
{
HotplugHandler *hotplug_ctrl = qdev_get_hotplug_handler(dev);
sPAPRMachineState *spapr = SPAPR_MACHINE(hotplug_ctrl);
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
sPAPRDIMMState *ds = spapr_pending_dimm_unplugs_find(spapr, PC_DIMM(dev));
/* This information will get lost if a migration occurs
* during the unplug process. In this case recover it. */
if (ds == NULL) {
ds = spapr_recover_pending_dimm_state(spapr, PC_DIMM(dev));
g_assert(ds);
/* The DRC being examined by the caller at least must be counted */
g_assert(ds->nr_lmbs);
}
if (--ds->nr_lmbs) {
return;
}
/*
* Now that all the LMBs have been removed by the guest, call the
* unplug handler chain. This can never fail.
*/
hotplug_handler_unplug(hotplug_ctrl, dev, &error_abort);
qdev: Let the hotplug_handler_unplug() caller delete the device When unplugging a device, at one point the device will be destroyed via object_unparent(). This will, one the one hand, unrealize the removed device hierarchy, and on the other hand, destroy/free the device hierarchy. When chaining hotplug handlers, we want to overwrite a bus hotplug handler by the machine hotplug handler, to be able to perform some part of the plug/unplug and to forward the calls to the bus hotplug handler. For now, the bus hotplug handler would trigger an object_unparent(), not allowing us to perform some unplug action on a device after we forwarded the call to the bus hotplug handler. The device would be gone at that point. machine_unplug_handler(dev) /* eventually do unplug stuff */ bus_unplug_handler(dev) /* dev is gone, we can't do more unplug stuff */ So move the object_unparent() to the original caller of the unplug. For now, keep the unrealize() at the original places of the object_unparent(). For implicitly chained hotplug handlers (e.g. pc code calling acpi hotplug handlers), the object_unparent() has to be done by the outermost caller. So when calling hotplug_handler_unplug() from inside an unplug handler, nothing is to be done. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> calls unrealize(dev) /* we can do more unplug stuff but device already unrealized */ } object_unparent(dev) In the long run, every unplug action should be factored out of the unrealize() function into the unplug handler (especially for PCI). Then we can get rid of the additonal unrealize() calls and object_unparent() will properly unrealize the device hierarchy after the device has been unplugged. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> only unplugs, does not unrealize /* we can do more unplug stuff */ } object_unparent(dev) -> will unrealize The original approach was suggested by Igor Mammedov for the PCI part, but I extended it to all hotplug handlers. I consider this one step into the right direction. To summarize: - object_unparent() on synchronous unplugs is done by common code -- "Caller of hotplug_handler_unplug" - object_unparent() on asynchronous unplugs ("unplug requests") has to be done manually -- "Caller of hotplug_handler_unplug" Reviewed-by: Igor Mammedov <imammedo@redhat.com> Acked-by: Cornelia Huck <cohuck@redhat.com> Signed-off-by: David Hildenbrand <david@redhat.com> Message-Id: <20190228122849.4296-2-david@redhat.com> Reviewed-by: Greg Kurz <groug@kaod.org> Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2019-02-28 13:28:47 +01:00
object_unparent(OBJECT(dev));
}
static void spapr_memory_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(hotplug_dev);
sPAPRDIMMState *ds = spapr_pending_dimm_unplugs_find(spapr, PC_DIMM(dev));
pc_dimm_unplug(PC_DIMM(dev), MACHINE(hotplug_dev));
qdev: Let the hotplug_handler_unplug() caller delete the device When unplugging a device, at one point the device will be destroyed via object_unparent(). This will, one the one hand, unrealize the removed device hierarchy, and on the other hand, destroy/free the device hierarchy. When chaining hotplug handlers, we want to overwrite a bus hotplug handler by the machine hotplug handler, to be able to perform some part of the plug/unplug and to forward the calls to the bus hotplug handler. For now, the bus hotplug handler would trigger an object_unparent(), not allowing us to perform some unplug action on a device after we forwarded the call to the bus hotplug handler. The device would be gone at that point. machine_unplug_handler(dev) /* eventually do unplug stuff */ bus_unplug_handler(dev) /* dev is gone, we can't do more unplug stuff */ So move the object_unparent() to the original caller of the unplug. For now, keep the unrealize() at the original places of the object_unparent(). For implicitly chained hotplug handlers (e.g. pc code calling acpi hotplug handlers), the object_unparent() has to be done by the outermost caller. So when calling hotplug_handler_unplug() from inside an unplug handler, nothing is to be done. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> calls unrealize(dev) /* we can do more unplug stuff but device already unrealized */ } object_unparent(dev) In the long run, every unplug action should be factored out of the unrealize() function into the unplug handler (especially for PCI). Then we can get rid of the additonal unrealize() calls and object_unparent() will properly unrealize the device hierarchy after the device has been unplugged. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> only unplugs, does not unrealize /* we can do more unplug stuff */ } object_unparent(dev) -> will unrealize The original approach was suggested by Igor Mammedov for the PCI part, but I extended it to all hotplug handlers. I consider this one step into the right direction. To summarize: - object_unparent() on synchronous unplugs is done by common code -- "Caller of hotplug_handler_unplug" - object_unparent() on asynchronous unplugs ("unplug requests") has to be done manually -- "Caller of hotplug_handler_unplug" Reviewed-by: Igor Mammedov <imammedo@redhat.com> Acked-by: Cornelia Huck <cohuck@redhat.com> Signed-off-by: David Hildenbrand <david@redhat.com> Message-Id: <20190228122849.4296-2-david@redhat.com> Reviewed-by: Greg Kurz <groug@kaod.org> Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2019-02-28 13:28:47 +01:00
object_property_set_bool(OBJECT(dev), false, "realized", NULL);
hw/ppc/spapr.c: abort unplug_request if previous unplug isn't done LMB removal is completed only when the spapr_lmb_release callback is called after all DRCs of the dimm are detached. During this time, it is possible that a unplug request for the same dimm arrives, trying to detach DRCs that were detached by the guest in the first unplug_request. BQL doesn't help in this case - the lock will prevent any concurrent removal from happening until the end of spapr_memory_unplug_request only. What happens is that the second unplug_request ends up calling spapr_drc_detach in a DRC that were detached already, causing an assert error in spapr_drc_detach (e.g https://bugs.launchpad.net/qemu/+bug/1718118). spapr_lmb_release uses a structure called sPAPRDIMMState, stored in the spapr->pending_dimm_unplugs QTAIL, to track how many LMB DRCs are left to be detached by the guest. When there are no more DRCs left, this structure is deleted and the pc-dimm unplug handler is called to finish the process. This patch reuses the sPAPRDIMMState to allow unplug_request to know if there is an ongoing unplug process for a given dimm, aborting the unplug request in this case, by doing the following changes: - in spapr_lmb_release callback, move the dimm state removal to the end, after pc-dimm unplug handler. With this change we can check for the existence of the dimm state to see if the unplug process is done. - use spapr_pending_dimm_unplugs_find in spapr_memory_unplug_request to check if the dimm state exists. If positive, there is an unplug operation already in progress for this dimm, meaning that we should abort it and warn the user about it. Fixes: https://bugs.launchpad.net/qemu/+bug/1718118 Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-10-09 23:11:36 +02:00
spapr_pending_dimm_unplugs_remove(spapr, ds);
}
static void spapr_memory_unplug_request(HotplugHandler *hotplug_dev,
DeviceState *dev, Error **errp)
{
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
sPAPRMachineState *spapr = SPAPR_MACHINE(hotplug_dev);
Error *local_err = NULL;
PCDIMMDevice *dimm = PC_DIMM(dev);
uint32_t nr_lmbs;
uint64_t size, addr_start, addr;
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
int i;
sPAPRDRConnector *drc;
size = memory_device_get_region_size(MEMORY_DEVICE(dimm), &error_abort);
nr_lmbs = size / SPAPR_MEMORY_BLOCK_SIZE;
addr_start = object_property_get_uint(OBJECT(dimm), PC_DIMM_ADDR_PROP,
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
&local_err);
if (local_err) {
goto out;
}
hw/ppc/spapr.c: abort unplug_request if previous unplug isn't done LMB removal is completed only when the spapr_lmb_release callback is called after all DRCs of the dimm are detached. During this time, it is possible that a unplug request for the same dimm arrives, trying to detach DRCs that were detached by the guest in the first unplug_request. BQL doesn't help in this case - the lock will prevent any concurrent removal from happening until the end of spapr_memory_unplug_request only. What happens is that the second unplug_request ends up calling spapr_drc_detach in a DRC that were detached already, causing an assert error in spapr_drc_detach (e.g https://bugs.launchpad.net/qemu/+bug/1718118). spapr_lmb_release uses a structure called sPAPRDIMMState, stored in the spapr->pending_dimm_unplugs QTAIL, to track how many LMB DRCs are left to be detached by the guest. When there are no more DRCs left, this structure is deleted and the pc-dimm unplug handler is called to finish the process. This patch reuses the sPAPRDIMMState to allow unplug_request to know if there is an ongoing unplug process for a given dimm, aborting the unplug request in this case, by doing the following changes: - in spapr_lmb_release callback, move the dimm state removal to the end, after pc-dimm unplug handler. With this change we can check for the existence of the dimm state to see if the unplug process is done. - use spapr_pending_dimm_unplugs_find in spapr_memory_unplug_request to check if the dimm state exists. If positive, there is an unplug operation already in progress for this dimm, meaning that we should abort it and warn the user about it. Fixes: https://bugs.launchpad.net/qemu/+bug/1718118 Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-10-09 23:11:36 +02:00
/*
* An existing pending dimm state for this DIMM means that there is an
* unplug operation in progress, waiting for the spapr_lmb_release
* callback to complete the job (BQL can't cover that far). In this case,
* bail out to avoid detaching DRCs that were already released.
*/
if (spapr_pending_dimm_unplugs_find(spapr, dimm)) {
error_setg(&local_err,
"Memory unplug already in progress for device %s",
dev->id);
goto out;
}
spapr_pending_dimm_unplugs_add(spapr, nr_lmbs, dimm);
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
addr = addr_start;
for (i = 0; i < nr_lmbs; i++) {
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB,
addr / SPAPR_MEMORY_BLOCK_SIZE);
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
g_assert(drc);
spapr_drc_detach(drc);
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
addr += SPAPR_MEMORY_BLOCK_SIZE;
}
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB,
addr_start / SPAPR_MEMORY_BLOCK_SIZE);
hw/ppc/spapr.c: adding pending_dimm_unplugs to sPAPRMachineState The LMB DRC release callback, spapr_lmb_release(), uses an opaque parameter, a sPAPRDIMMState struct that stores the current LMBs that are allocated to a DIMM (nr_lmbs). After each call to this callback, the nr_lmbs is decremented by one and, when it reaches zero, the callback proceeds with the qdev calls to hot unplug the LMB. Using drc->detach_cb_opaque is problematic because it can't be migrated in the future DRC migration work. This patch makes the following changes to eliminate the usage of this opaque callback inside spapr_lmb_release: - sPAPRDIMMState was moved from spapr.c and added to spapr.h. A new attribute called 'addr' was added to it. This is used as an unique identifier to associate a sPAPRDIMMState to a PCDIMM element. - sPAPRMachineState now hosts a new QTAILQ called 'pending_dimm_unplugs'. This queue of sPAPRDIMMState elements will store the DIMM state of DIMMs that are currently going under an unplug process. - spapr_lmb_release() will now retrieve the nr_lmbs value by getting the correspondent sPAPRDIMMState. A helper function called spapr_dimm_get_address was created to fetch the address of a PCDIMM device inside spapr_lmb_release. When nr_lmbs reaches zero and the callback proceeds with the qdev hot unplug calls, the sPAPRDIMMState struct is removed from spapr->pending_dimm_unplugs. After these changes, the opaque argument for spapr_lmb_release is now unused and is passed as NULL inside spapr_del_lmbs. This and the other opaque arguments can now be safely removed from the code. As an additional cleanup made by this patch, the spapr_del_lmbs function was merged with spapr_memory_unplug_request. The former was being called only by the latter and both were small enough to fit one single function. Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com> [dwg: Minor stylistic cleanups] Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2017-05-24 09:01:48 +02:00
spapr_hotplug_req_remove_by_count_indexed(SPAPR_DR_CONNECTOR_TYPE_LMB,
nr_lmbs, spapr_drc_index(drc));
out:
error_propagate(errp, local_err);
}
/* Callback to be called during DRC release. */
void spapr_core_release(DeviceState *dev)
{
HotplugHandler *hotplug_ctrl = qdev_get_hotplug_handler(dev);
/* Call the unplug handler chain. This can never fail. */
hotplug_handler_unplug(hotplug_ctrl, dev, &error_abort);
qdev: Let the hotplug_handler_unplug() caller delete the device When unplugging a device, at one point the device will be destroyed via object_unparent(). This will, one the one hand, unrealize the removed device hierarchy, and on the other hand, destroy/free the device hierarchy. When chaining hotplug handlers, we want to overwrite a bus hotplug handler by the machine hotplug handler, to be able to perform some part of the plug/unplug and to forward the calls to the bus hotplug handler. For now, the bus hotplug handler would trigger an object_unparent(), not allowing us to perform some unplug action on a device after we forwarded the call to the bus hotplug handler. The device would be gone at that point. machine_unplug_handler(dev) /* eventually do unplug stuff */ bus_unplug_handler(dev) /* dev is gone, we can't do more unplug stuff */ So move the object_unparent() to the original caller of the unplug. For now, keep the unrealize() at the original places of the object_unparent(). For implicitly chained hotplug handlers (e.g. pc code calling acpi hotplug handlers), the object_unparent() has to be done by the outermost caller. So when calling hotplug_handler_unplug() from inside an unplug handler, nothing is to be done. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> calls unrealize(dev) /* we can do more unplug stuff but device already unrealized */ } object_unparent(dev) In the long run, every unplug action should be factored out of the unrealize() function into the unplug handler (especially for PCI). Then we can get rid of the additonal unrealize() calls and object_unparent() will properly unrealize the device hierarchy after the device has been unplugged. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> only unplugs, does not unrealize /* we can do more unplug stuff */ } object_unparent(dev) -> will unrealize The original approach was suggested by Igor Mammedov for the PCI part, but I extended it to all hotplug handlers. I consider this one step into the right direction. To summarize: - object_unparent() on synchronous unplugs is done by common code -- "Caller of hotplug_handler_unplug" - object_unparent() on asynchronous unplugs ("unplug requests") has to be done manually -- "Caller of hotplug_handler_unplug" Reviewed-by: Igor Mammedov <imammedo@redhat.com> Acked-by: Cornelia Huck <cohuck@redhat.com> Signed-off-by: David Hildenbrand <david@redhat.com> Message-Id: <20190228122849.4296-2-david@redhat.com> Reviewed-by: Greg Kurz <groug@kaod.org> Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2019-02-28 13:28:47 +01:00
object_unparent(OBJECT(dev));
}
static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
{
MachineState *ms = MACHINE(hotplug_dev);
sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(ms);
CPUCore *cc = CPU_CORE(dev);
CPUArchId *core_slot = spapr_find_cpu_slot(ms, cc->core_id, NULL);
if (smc->pre_2_10_has_unused_icps) {
sPAPRCPUCore *sc = SPAPR_CPU_CORE(OBJECT(dev));
int i;
for (i = 0; i < cc->nr_threads; i++) {
CPUState *cs = CPU(sc->threads[i]);
pre_2_10_vmstate_register_dummy_icp(cs->cpu_index);
}
}
assert(core_slot);
core_slot->cpu = NULL;
qdev: Let the hotplug_handler_unplug() caller delete the device When unplugging a device, at one point the device will be destroyed via object_unparent(). This will, one the one hand, unrealize the removed device hierarchy, and on the other hand, destroy/free the device hierarchy. When chaining hotplug handlers, we want to overwrite a bus hotplug handler by the machine hotplug handler, to be able to perform some part of the plug/unplug and to forward the calls to the bus hotplug handler. For now, the bus hotplug handler would trigger an object_unparent(), not allowing us to perform some unplug action on a device after we forwarded the call to the bus hotplug handler. The device would be gone at that point. machine_unplug_handler(dev) /* eventually do unplug stuff */ bus_unplug_handler(dev) /* dev is gone, we can't do more unplug stuff */ So move the object_unparent() to the original caller of the unplug. For now, keep the unrealize() at the original places of the object_unparent(). For implicitly chained hotplug handlers (e.g. pc code calling acpi hotplug handlers), the object_unparent() has to be done by the outermost caller. So when calling hotplug_handler_unplug() from inside an unplug handler, nothing is to be done. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> calls unrealize(dev) /* we can do more unplug stuff but device already unrealized */ } object_unparent(dev) In the long run, every unplug action should be factored out of the unrealize() function into the unplug handler (especially for PCI). Then we can get rid of the additonal unrealize() calls and object_unparent() will properly unrealize the device hierarchy after the device has been unplugged. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> only unplugs, does not unrealize /* we can do more unplug stuff */ } object_unparent(dev) -> will unrealize The original approach was suggested by Igor Mammedov for the PCI part, but I extended it to all hotplug handlers. I consider this one step into the right direction. To summarize: - object_unparent() on synchronous unplugs is done by common code -- "Caller of hotplug_handler_unplug" - object_unparent() on asynchronous unplugs ("unplug requests") has to be done manually -- "Caller of hotplug_handler_unplug" Reviewed-by: Igor Mammedov <imammedo@redhat.com> Acked-by: Cornelia Huck <cohuck@redhat.com> Signed-off-by: David Hildenbrand <david@redhat.com> Message-Id: <20190228122849.4296-2-david@redhat.com> Reviewed-by: Greg Kurz <groug@kaod.org> Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2019-02-28 13:28:47 +01:00
object_property_set_bool(OBJECT(dev), false, "realized", NULL);
}
static
void spapr_core_unplug_request(HotplugHandler *hotplug_dev, DeviceState *dev,
Error **errp)
{
spapr: use spapr->vsmt to compute VCPU ids Since the introduction of VSMT in 2.11, the spacing of VCPU ids between cores is controllable through a machine property instead of being only dictated by the SMT mode of the host: cpu->vcpu_id = (cc->core_id * spapr->vsmt / smp_threads) + i Until recently, the machine code would try to change the SMT mode of the host to be equal to VSMT or exit. This allowed the rest of the code to assume that kvmppc_smt_threads() == spapr->vsmt is always true. Recent commit "8904e5a75005 spapr: Adjust default VSMT value for better migration compatibility" relaxed the rule. If the VSMT mode cannot be set in KVM for some reasons, but the requested CPU topology is compatible with the current SMT mode, then we let the guest run with kvmppc_smt_threads() != spapr->vsmt. This breaks quite a few places in the code, in particular when calculating DRC indexes. This is what happens on a POWER host with subcores-per-core=2 (ie, supports up to SMT4) when passing the following topology: -smp threads=4,maxcpus=16 \ -device host-spapr-cpu-core,core-id=4,id=core1 \ -device host-spapr-cpu-core,core-id=8,id=core2 qemu-system-ppc64: warning: Failed to set KVM's VSMT mode to 8 (errno -22) This is expected since KVM is limited to SMT4, but the guest is started anyway because this topology can run on SMT4 even with a VSMT8 spacing. But when we look at the DT, things get nastier: cpus { ... ibm,drc-indexes = <0x4 0x10000000 0x10000004 0x10000008 0x1000000c>; This means that we have the following association: CPU core device | DRC | VCPU id -----------------+------------+--------- boot core | 0x10000000 | 0 core1 | 0x10000004 | 4 core2 | 0x10000008 | 8 core3 | 0x1000000c | 12 But since the spacing of VCPU ids is 8, the DRC for core1 points to a VCPU that doesn't exist, the DRC for core2 points to the first VCPU of core1 and and so on... ... PowerPC,POWER8@0 { ... ibm,my-drc-index = <0x10000000>; ... }; PowerPC,POWER8@8 { ... ibm,my-drc-index = <0x10000008>; ... }; PowerPC,POWER8@10 { ... No ibm,my-drc-index property for this core since 0x10000010 doesn't exist in ibm,drc-indexes above. ... }; }; ... interrupt-controller { ... ibm,interrupt-server-ranges = <0x0 0x10>; With a spacing of 8, the highest VCPU id for the given topology should be: 16 * 8 / 4 = 32 and not 16 ... linux,phandle = <0x7e7323b8>; interrupt-controller; }; And CPU hot-plug/unplug is broken: (qemu) device_del core1 pseries-hotplug-cpu: Cannot find CPU (drc index 10000004) to remove (qemu) device_del core2 cpu 4 (hwid 8) Ready to die... cpu 5 (hwid 9) Ready to die... cpu 6 (hwid 10) Ready to die... cpu 7 (hwid 11) Ready to die... These are the VCPU ids of core1 actually (qemu) device_add host-spapr-cpu-core,core-id=12,id=core3 (qemu) device_del core3 pseries-hotplug-cpu: Cannot find CPU (drc index 1000000c) to remove This patches all the code in hw/ppc/spapr.c to assume the VSMT spacing when manipulating VCPU ids. Fixes: 8904e5a75005 Signed-off-by: Greg Kurz <groug@kaod.org> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2018-02-14 20:40:26 +01:00
sPAPRMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
int index;
sPAPRDRConnector *drc;
CPUCore *cc = CPU_CORE(dev);
if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
error_setg(errp, "Unable to find CPU core with core-id: %d",
cc->core_id);
return;
}
if (index == 0) {
error_setg(errp, "Boot CPU core may not be unplugged");
return;
}
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_CPU,
spapr_vcpu_id(spapr, cc->core_id));
g_assert(drc);
spapr_drc_detach(drc);
spapr_hotplug_req_remove_by_index(drc);
}
int spapr_core_dt_populate(sPAPRDRConnector *drc, sPAPRMachineState *spapr,
void *fdt, int *fdt_start_offset, Error **errp)
{
sPAPRCPUCore *core = SPAPR_CPU_CORE(drc->dev);
CPUState *cs = CPU(core->threads[0]);
PowerPCCPU *cpu = POWERPC_CPU(cs);
DeviceClass *dc = DEVICE_GET_CLASS(cs);
int id = spapr_get_vcpu_id(cpu);
char *nodename;
int offset;
nodename = g_strdup_printf("%s@%x", dc->fw_name, id);
offset = fdt_add_subnode(fdt, 0, nodename);
g_free(nodename);
spapr_populate_cpu_dt(cs, fdt, offset, spapr);
*fdt_start_offset = offset;
return 0;
}
static void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
MachineClass *mc = MACHINE_GET_CLASS(spapr);
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
CPUCore *cc = CPU_CORE(dev);
CPUState *cs;
sPAPRDRConnector *drc;
Error *local_err = NULL;
CPUArchId *core_slot;
int index;
spapr: Treat devices added before inbound migration as coldplugged When migrating a guest which has already had devices hotplugged, libvirt typically starts the destination qemu with -incoming defer, adds those hotplugged devices with qmp, then initiates the incoming migration. This causes problems for the management of spapr DRC state. Because the device is treated as hotplugged, it goes into a DRC state for a device immediately after it's plugged, but before the guest has acknowledged its presence. However, chances are the guest on the source machine *has* acknowledged the device's presence and configured it. If the source has fully configured the device, then DRC state won't be sent in the migration stream: for maximum migration compatibility with earlier versions we don't migrate DRCs in coldplug-equivalent state. That means that the DRC effectively changes state over the migrate, causing problems later on. In addition, logging hotplug events for these devices isn't what we want because a) those events should already have been issued on the source host and b) the event queue should get wiped out by the incoming state anyway. In short, what we really want is to treat devices added before an incoming migration as if they were coldplugged. To do this, we first add a spapr_drc_hotplugged() helper which determines if the device is hotplugged in the sense relevant for DRC state management. We only send hotplug events when this is true. Second, when we add a device which isn't hotplugged in this sense, we force a reset of the DRC state - this ensures the DRC is in a coldplug-equivalent state (there isn't usually a system reset between these device adds and the incoming migration). This is based on an earlier patch by Laurent Vivier, cleaned up and extended. Signed-off-by: Laurent Vivier <lvivier@redhat.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Greg Kurz <groug@kaod.org> Tested-by: Daniel Barboza <danielhb@linux.vnet.ibm.com>
2017-06-09 13:08:10 +02:00
bool hotplugged = spapr_drc_hotplugged(dev);
core_slot = spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index);
if (!core_slot) {
error_setg(errp, "Unable to find CPU core with core-id: %d",
cc->core_id);
return;
}
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_CPU,
spapr_vcpu_id(spapr, cc->core_id));
g_assert(drc || !mc->has_hotpluggable_cpus);
if (drc) {
spapr_drc_attach(drc, dev, &local_err);
if (local_err) {
error_propagate(errp, local_err);
return;
}
spapr: Treat devices added before inbound migration as coldplugged When migrating a guest which has already had devices hotplugged, libvirt typically starts the destination qemu with -incoming defer, adds those hotplugged devices with qmp, then initiates the incoming migration. This causes problems for the management of spapr DRC state. Because the device is treated as hotplugged, it goes into a DRC state for a device immediately after it's plugged, but before the guest has acknowledged its presence. However, chances are the guest on the source machine *has* acknowledged the device's presence and configured it. If the source has fully configured the device, then DRC state won't be sent in the migration stream: for maximum migration compatibility with earlier versions we don't migrate DRCs in coldplug-equivalent state. That means that the DRC effectively changes state over the migrate, causing problems later on. In addition, logging hotplug events for these devices isn't what we want because a) those events should already have been issued on the source host and b) the event queue should get wiped out by the incoming state anyway. In short, what we really want is to treat devices added before an incoming migration as if they were coldplugged. To do this, we first add a spapr_drc_hotplugged() helper which determines if the device is hotplugged in the sense relevant for DRC state management. We only send hotplug events when this is true. Second, when we add a device which isn't hotplugged in this sense, we force a reset of the DRC state - this ensures the DRC is in a coldplug-equivalent state (there isn't usually a system reset between these device adds and the incoming migration). This is based on an earlier patch by Laurent Vivier, cleaned up and extended. Signed-off-by: Laurent Vivier <lvivier@redhat.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Greg Kurz <groug@kaod.org> Tested-by: Daniel Barboza <danielhb@linux.vnet.ibm.com>
2017-06-09 13:08:10 +02:00
if (hotplugged) {
/*
* Send hotplug notification interrupt to the guest only
* in case of hotplugged CPUs.
*/
spapr_hotplug_req_add_by_index(drc);
} else {
spapr_drc_reset(drc);
}
}
spapr: Treat devices added before inbound migration as coldplugged When migrating a guest which has already had devices hotplugged, libvirt typically starts the destination qemu with -incoming defer, adds those hotplugged devices with qmp, then initiates the incoming migration. This causes problems for the management of spapr DRC state. Because the device is treated as hotplugged, it goes into a DRC state for a device immediately after it's plugged, but before the guest has acknowledged its presence. However, chances are the guest on the source machine *has* acknowledged the device's presence and configured it. If the source has fully configured the device, then DRC state won't be sent in the migration stream: for maximum migration compatibility with earlier versions we don't migrate DRCs in coldplug-equivalent state. That means that the DRC effectively changes state over the migrate, causing problems later on. In addition, logging hotplug events for these devices isn't what we want because a) those events should already have been issued on the source host and b) the event queue should get wiped out by the incoming state anyway. In short, what we really want is to treat devices added before an incoming migration as if they were coldplugged. To do this, we first add a spapr_drc_hotplugged() helper which determines if the device is hotplugged in the sense relevant for DRC state management. We only send hotplug events when this is true. Second, when we add a device which isn't hotplugged in this sense, we force a reset of the DRC state - this ensures the DRC is in a coldplug-equivalent state (there isn't usually a system reset between these device adds and the incoming migration). This is based on an earlier patch by Laurent Vivier, cleaned up and extended. Signed-off-by: Laurent Vivier <lvivier@redhat.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Greg Kurz <groug@kaod.org> Tested-by: Daniel Barboza <danielhb@linux.vnet.ibm.com>
2017-06-09 13:08:10 +02:00
core_slot->cpu = OBJECT(dev);
if (smc->pre_2_10_has_unused_icps) {
int i;
for (i = 0; i < cc->nr_threads; i++) {
cs = CPU(core->threads[i]);
pre_2_10_vmstate_unregister_dummy_icp(cs->cpu_index);
}
}
}
static void spapr_core_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
Error **errp)
{
MachineState *machine = MACHINE(OBJECT(hotplug_dev));
MachineClass *mc = MACHINE_GET_CLASS(hotplug_dev);
Error *local_err = NULL;
CPUCore *cc = CPU_CORE(dev);
const char *base_core_type = spapr_get_cpu_core_type(machine->cpu_type);
const char *type = object_get_typename(OBJECT(dev));
CPUArchId *core_slot;
int index;
if (dev->hotplugged && !mc->has_hotpluggable_cpus) {
error_setg(&local_err, "CPU hotplug not supported for this machine");
goto out;
}
if (strcmp(base_core_type, type)) {
error_setg(&local_err, "CPU core type should be %s", base_core_type);
goto out;
}
if (cc->core_id % smp_threads) {
error_setg(&local_err, "invalid core id %d", cc->core_id);
goto out;
}
/*
* In general we should have homogeneous threads-per-core, but old
* (pre hotplug support) machine types allow the last core to have
* reduced threads as a compatibility hack for when we allowed
* total vcpus not a multiple of threads-per-core.
*/
if (mc->has_hotpluggable_cpus && (cc->nr_threads != smp_threads)) {
error_setg(&local_err, "invalid nr-threads %d, must be %d",
cc->nr_threads, smp_threads);
goto out;
}
core_slot = spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index);
if (!core_slot) {
error_setg(&local_err, "core id %d out of range", cc->core_id);
goto out;
}
if (core_slot->cpu) {
error_setg(&local_err, "core %d already populated", cc->core_id);
goto out;
}
numa_cpu_pre_plug(core_slot, dev, &local_err);
out:
error_propagate(errp, local_err);
}
int spapr_phb_dt_populate(sPAPRDRConnector *drc, sPAPRMachineState *spapr,
void *fdt, int *fdt_start_offset, Error **errp)
{
sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(drc->dev);
int intc_phandle;
intc_phandle = spapr_irq_get_phandle(spapr, spapr->fdt_blob, errp);
if (intc_phandle <= 0) {
return -1;
}
if (spapr_populate_pci_dt(sphb, intc_phandle, fdt, spapr->irq->nr_msis,
fdt_start_offset)) {
error_setg(errp, "unable to create FDT node for PHB %d", sphb->index);
return -1;
}
/* generally SLOF creates these, for hotplug it's up to QEMU */
_FDT(fdt_setprop_string(fdt, *fdt_start_offset, "name", "pci"));
return 0;
}
static void spapr_phb_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(dev);
sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
const unsigned windows_supported = spapr_phb_windows_supported(sphb);
if (dev->hotplugged && !smc->dr_phb_enabled) {
error_setg(errp, "PHB hotplug not supported for this machine");
return;
}
if (sphb->index == (uint32_t)-1) {
error_setg(errp, "\"index\" for PAPR PHB is mandatory");
return;
}
/*
* This will check that sphb->index doesn't exceed the maximum number of
* PHBs for the current machine type.
*/
smc->phb_placement(spapr, sphb->index,
&sphb->buid, &sphb->io_win_addr,
&sphb->mem_win_addr, &sphb->mem64_win_addr,
windows_supported, sphb->dma_liobn, errp);
}
static void spapr_phb_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(dev);
sPAPRDRConnector *drc;
bool hotplugged = spapr_drc_hotplugged(dev);
Error *local_err = NULL;
if (!smc->dr_phb_enabled) {
return;
}
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_PHB, sphb->index);
/* hotplug hooks should check it's enabled before getting this far */
assert(drc);
spapr_drc_attach(drc, DEVICE(dev), &local_err);
if (local_err) {
error_propagate(errp, local_err);
return;
}
if (hotplugged) {
spapr_hotplug_req_add_by_index(drc);
} else {
spapr_drc_reset(drc);
}
}
void spapr_phb_release(DeviceState *dev)
{
HotplugHandler *hotplug_ctrl = qdev_get_hotplug_handler(dev);
hotplug_handler_unplug(hotplug_ctrl, dev, &error_abort);
qdev: Let the hotplug_handler_unplug() caller delete the device When unplugging a device, at one point the device will be destroyed via object_unparent(). This will, one the one hand, unrealize the removed device hierarchy, and on the other hand, destroy/free the device hierarchy. When chaining hotplug handlers, we want to overwrite a bus hotplug handler by the machine hotplug handler, to be able to perform some part of the plug/unplug and to forward the calls to the bus hotplug handler. For now, the bus hotplug handler would trigger an object_unparent(), not allowing us to perform some unplug action on a device after we forwarded the call to the bus hotplug handler. The device would be gone at that point. machine_unplug_handler(dev) /* eventually do unplug stuff */ bus_unplug_handler(dev) /* dev is gone, we can't do more unplug stuff */ So move the object_unparent() to the original caller of the unplug. For now, keep the unrealize() at the original places of the object_unparent(). For implicitly chained hotplug handlers (e.g. pc code calling acpi hotplug handlers), the object_unparent() has to be done by the outermost caller. So when calling hotplug_handler_unplug() from inside an unplug handler, nothing is to be done. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> calls unrealize(dev) /* we can do more unplug stuff but device already unrealized */ } object_unparent(dev) In the long run, every unplug action should be factored out of the unrealize() function into the unplug handler (especially for PCI). Then we can get rid of the additonal unrealize() calls and object_unparent() will properly unrealize the device hierarchy after the device has been unplugged. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> only unplugs, does not unrealize /* we can do more unplug stuff */ } object_unparent(dev) -> will unrealize The original approach was suggested by Igor Mammedov for the PCI part, but I extended it to all hotplug handlers. I consider this one step into the right direction. To summarize: - object_unparent() on synchronous unplugs is done by common code -- "Caller of hotplug_handler_unplug" - object_unparent() on asynchronous unplugs ("unplug requests") has to be done manually -- "Caller of hotplug_handler_unplug" Reviewed-by: Igor Mammedov <imammedo@redhat.com> Acked-by: Cornelia Huck <cohuck@redhat.com> Signed-off-by: David Hildenbrand <david@redhat.com> Message-Id: <20190228122849.4296-2-david@redhat.com> Reviewed-by: Greg Kurz <groug@kaod.org> Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2019-02-28 13:28:47 +01:00
object_unparent(OBJECT(dev));
}
static void spapr_phb_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
{
qdev: Let the hotplug_handler_unplug() caller delete the device When unplugging a device, at one point the device will be destroyed via object_unparent(). This will, one the one hand, unrealize the removed device hierarchy, and on the other hand, destroy/free the device hierarchy. When chaining hotplug handlers, we want to overwrite a bus hotplug handler by the machine hotplug handler, to be able to perform some part of the plug/unplug and to forward the calls to the bus hotplug handler. For now, the bus hotplug handler would trigger an object_unparent(), not allowing us to perform some unplug action on a device after we forwarded the call to the bus hotplug handler. The device would be gone at that point. machine_unplug_handler(dev) /* eventually do unplug stuff */ bus_unplug_handler(dev) /* dev is gone, we can't do more unplug stuff */ So move the object_unparent() to the original caller of the unplug. For now, keep the unrealize() at the original places of the object_unparent(). For implicitly chained hotplug handlers (e.g. pc code calling acpi hotplug handlers), the object_unparent() has to be done by the outermost caller. So when calling hotplug_handler_unplug() from inside an unplug handler, nothing is to be done. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> calls unrealize(dev) /* we can do more unplug stuff but device already unrealized */ } object_unparent(dev) In the long run, every unplug action should be factored out of the unrealize() function into the unplug handler (especially for PCI). Then we can get rid of the additonal unrealize() calls and object_unparent() will properly unrealize the device hierarchy after the device has been unplugged. hotplug_handler_unplug(dev) -> calls machine_unplug_handler() machine_unplug_handler(dev) { /* eventually do unplug stuff */ bus_unplug_handler(dev) -> only unplugs, does not unrealize /* we can do more unplug stuff */ } object_unparent(dev) -> will unrealize The original approach was suggested by Igor Mammedov for the PCI part, but I extended it to all hotplug handlers. I consider this one step into the right direction. To summarize: - object_unparent() on synchronous unplugs is done by common code -- "Caller of hotplug_handler_unplug" - object_unparent() on asynchronous unplugs ("unplug requests") has to be done manually -- "Caller of hotplug_handler_unplug" Reviewed-by: Igor Mammedov <imammedo@redhat.com> Acked-by: Cornelia Huck <cohuck@redhat.com> Signed-off-by: David Hildenbrand <david@redhat.com> Message-Id: <20190228122849.4296-2-david@redhat.com> Reviewed-by: Greg Kurz <groug@kaod.org> Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2019-02-28 13:28:47 +01:00
object_property_set_bool(OBJECT(dev), false, "realized", NULL);
}
static void spapr_phb_unplug_request(HotplugHandler *hotplug_dev,
DeviceState *dev, Error **errp)
{
sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(dev);
sPAPRDRConnector *drc;
drc = spapr_drc_by_id(TYPE_SPAPR_DRC_PHB, sphb->index);
assert(drc);
if (!spapr_drc_unplug_requested(drc)) {
spapr_drc_detach(drc);
spapr_hotplug_req_remove_by_index(drc);
}
}
static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
DeviceState *dev, Error **errp)
{
if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
spapr_memory_plug(hotplug_dev, dev, errp);
} else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
spapr_core_plug(hotplug_dev, dev, errp);
} else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_PCI_HOST_BRIDGE)) {
spapr_phb_plug(hotplug_dev, dev, errp);
}
}
static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
DeviceState *dev, Error **errp)
{
if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
spapr_memory_unplug(hotplug_dev, dev);
} else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
spapr_core_unplug(hotplug_dev, dev);
} else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_PCI_HOST_BRIDGE)) {
spapr_phb_unplug(hotplug_dev, dev);
}
}
static void spapr_machine_device_unplug_request(HotplugHandler *hotplug_dev,
DeviceState *dev, Error **errp)
{
sPAPRMachineState *sms = SPAPR_MACHINE(OBJECT(hotplug_dev));
MachineClass *mc = MACHINE_GET_CLASS(sms);
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
if (spapr_ovec_test(sms->ov5_cas, OV5_HP_EVT)) {
spapr_memory_unplug_request(hotplug_dev, dev, errp);
} else {
/* NOTE: this means there is a window after guest reset, prior to
* CAS negotiation, where unplug requests will fail due to the
* capability not being detected yet. This is a bit different than
* the case with PCI unplug, where the events will be queued and
* eventually handled by the guest after boot
*/
error_setg(errp, "Memory hot unplug not supported for this guest");
}
} else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
if (!mc->has_hotpluggable_cpus) {
error_setg(errp, "CPU hot unplug not supported on this machine");
return;
}
spapr_core_unplug_request(hotplug_dev, dev, errp);
} else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_PCI_HOST_BRIDGE)) {
if (!smc->dr_phb_enabled) {
error_setg(errp, "PHB hot unplug not supported on this machine");
return;
}
spapr_phb_unplug_request(hotplug_dev, dev, errp);
}
}
static void spapr_machine_device_pre_plug(HotplugHandler *hotplug_dev,
DeviceState *dev, Error **errp)
{
if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
spapr_memory_pre_plug(hotplug_dev, dev, errp);
} else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
spapr_core_pre_plug(hotplug_dev, dev, errp);
} else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_PCI_HOST_BRIDGE)) {
spapr_phb_pre_plug(hotplug_dev, dev, errp);
}
}
static HotplugHandler *spapr_get_hotplug_handler(MachineState *machine,
DeviceState *dev)
{
if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM) ||
object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE) ||
object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_PCI_HOST_BRIDGE)) {
return HOTPLUG_HANDLER(machine);
}
return NULL;
}
static CpuInstanceProperties
spapr_cpu_index_to_props(MachineState *machine, unsigned cpu_index)
{
CPUArchId *core_slot;
MachineClass *mc = MACHINE_GET_CLASS(machine);
/* make sure possible_cpu are intialized */
mc->possible_cpu_arch_ids(machine);
/* get CPU core slot containing thread that matches cpu_index */
core_slot = spapr_find_cpu_slot(machine, cpu_index, NULL);
assert(core_slot);
return core_slot->props;
}
static int64_t spapr_get_default_cpu_node_id(const MachineState *ms, int idx)
{
return idx / smp_cores % nb_numa_nodes;
}
static const CPUArchIdList *spapr_possible_cpu_arch_ids(MachineState *machine)
{
int i;
const char *core_type;
int spapr_max_cores = max_cpus / smp_threads;
MachineClass *mc = MACHINE_GET_CLASS(machine);
if (!mc->has_hotpluggable_cpus) {
spapr_max_cores = QEMU_ALIGN_UP(smp_cpus, smp_threads) / smp_threads;
}
if (machine->possible_cpus) {
assert(machine->possible_cpus->len == spapr_max_cores);
return machine->possible_cpus;
}
core_type = spapr_get_cpu_core_type(machine->cpu_type);
if (!core_type) {
error_report("Unable to find sPAPR CPU Core definition");
exit(1);
}
machine->possible_cpus = g_malloc0(sizeof(CPUArchIdList) +
sizeof(CPUArchId) * spapr_max_cores);
machine->possible_cpus->len = spapr_max_cores;
for (i = 0; i < machine->possible_cpus->len; i++) {
int core_id = i * smp_threads;
machine->possible_cpus->cpus[i].type = core_type;
machine->possible_cpus->cpus[i].vcpus_count = smp_threads;
machine->possible_cpus->cpus[i].arch_id = core_id;
machine->possible_cpus->cpus[i].props.has_core_id = true;
machine->possible_cpus->cpus[i].props.core_id = core_id;
}
return machine->possible_cpus;
}
spapr_pci: Delegate placement of PCI host bridges to machine type The 'spapr-pci-host-bridge' represents the virtual PCI host bridge (PHB) for a PAPR guest. Unlike on x86, it's routine on Power (both bare metal and PAPR guests) to have numerous independent PHBs, each controlling a separate PCI domain. There are two ways of configuring the spapr-pci-host-bridge device: first it can be done fully manually, specifying the locations and sizes of all the IO windows. This gives the most control, but is very awkward with 6 mandatory parameters. Alternatively just an "index" can be specified which essentially selects from an array of predefined PHB locations. The PHB at index 0 is automatically created as the default PHB. The current set of default locations causes some problems for guests with large RAM (> 1 TiB) or PCI devices with very large BARs (e.g. big nVidia GPGPU cards via VFIO). Obviously, for migration we can only change the locations on a new machine type, however. This is awkward, because the placement is currently decided within the spapr-pci-host-bridge code, so it breaks abstraction to look inside the machine type version. So, this patch delegates the "default mode" PHB placement from the spapr-pci-host-bridge device back to the machine type via a public method in sPAPRMachineClass. It's still a bit ugly, but it's about the best we can do. For now, this just changes where the calculation is done. It doesn't change the actual location of the host bridges, or any other behaviour. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-13 01:26:09 +02:00
static void spapr_phb_placement(sPAPRMachineState *spapr, uint32_t index,
spapr_pci: Add a 64-bit MMIO window On real hardware, and under pHyp, the PCI host bridges on Power machines typically advertise two outbound MMIO windows from the guest's physical memory space to PCI memory space: - A 32-bit window which maps onto 2GiB..4GiB in the PCI address space - A 64-bit window which maps onto a large region somewhere high in PCI address space (traditionally this used an identity mapping from guest physical address to PCI address, but that's not always the case) The qemu implementation in spapr-pci-host-bridge, however, only supports a single outbound MMIO window, however. At least some Linux versions expect the two windows however, so we arranged this window to map onto the PCI memory space from 2 GiB..~64 GiB, then advertised it as two contiguous windows, the "32-bit" window from 2G..4G and the "64-bit" window from 4G..~64G. This approach means, however, that the 64G window is not naturally aligned. In turn this limits the size of the largest BAR we can map (which does have to be naturally aligned) to roughly half of the total window. With some large nVidia GPGPU cards which have huge memory BARs, this is starting to be a problem. This patch adds true support for separate 32-bit and 64-bit outbound MMIO windows to the spapr-pci-host-bridge implementation, each of which can be independently configured. The 32-bit window always maps to 2G.. in PCI space, but the PCI address of the 64-bit window can be configured (it defaults to the same as the guest physical address). So as not to break possible existing configurations, as long as a 64-bit window is not specified, a large single window can be specified. This will appear the same way to the guest as the old approach, although it's now implemented by two contiguous memory regions rather than a single one. For now, this only adds the possibility of 64-bit windows. The default configuration still uses the legacy mode. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-11 05:23:33 +02:00
uint64_t *buid, hwaddr *pio,
hwaddr *mmio32, hwaddr *mmio64,
spapr_pci: Delegate placement of PCI host bridges to machine type The 'spapr-pci-host-bridge' represents the virtual PCI host bridge (PHB) for a PAPR guest. Unlike on x86, it's routine on Power (both bare metal and PAPR guests) to have numerous independent PHBs, each controlling a separate PCI domain. There are two ways of configuring the spapr-pci-host-bridge device: first it can be done fully manually, specifying the locations and sizes of all the IO windows. This gives the most control, but is very awkward with 6 mandatory parameters. Alternatively just an "index" can be specified which essentially selects from an array of predefined PHB locations. The PHB at index 0 is automatically created as the default PHB. The current set of default locations causes some problems for guests with large RAM (> 1 TiB) or PCI devices with very large BARs (e.g. big nVidia GPGPU cards via VFIO). Obviously, for migration we can only change the locations on a new machine type, however. This is awkward, because the placement is currently decided within the spapr-pci-host-bridge code, so it breaks abstraction to look inside the machine type version. So, this patch delegates the "default mode" PHB placement from the spapr-pci-host-bridge device back to the machine type via a public method in sPAPRMachineClass. It's still a bit ugly, but it's about the best we can do. For now, this just changes where the calculation is done. It doesn't change the actual location of the host bridges, or any other behaviour. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-13 01:26:09 +02:00
unsigned n_dma, uint32_t *liobns, Error **errp)
{
spapr: Improved placement of PCI host bridges in guest memory map Currently, the MMIO space for accessing PCI on pseries guests begins at 1 TiB in guest address space. Each PCI host bridge (PHB) has a 64 GiB chunk of address space in which it places its outbound PIO and 32-bit and 64-bit MMIO windows. This scheme as several problems: - It limits guest RAM to 1 TiB (though we have a limited fix for this now) - It limits the total MMIO window to 64 GiB. This is not always enough for some of the large nVidia GPGPU cards - Putting all the windows into a single 64 GiB area means that naturally aligning things within there will waste more address space. In addition there was a miscalculation in some of the defaults, which meant that the MMIO windows for each PHB actually slightly overran the 64 GiB region for that PHB. We got away without nasty consequences because the overrun fit within an unused area at the beginning of the next PHB's region, but it's not pretty. This patch implements a new scheme which addresses those problems, and is also closer to what bare metal hardware and pHyp guests generally use. Because some guest versions (including most current distro kernels) can't access PCI MMIO above 64 TiB, we put all the PCI windows between 32 TiB and 64 TiB. This is broken into 1 TiB chunks. The first 1 TiB contains the PIO (64 kiB) and 32-bit MMIO (2 GiB) windows for all of the PHBs. Each subsequent TiB chunk contains a naturally aligned 64-bit MMIO window for one PHB each. This reduces the number of allowed PHBs (without full manual configuration of all the windows) from 256 to 31, but this should still be plenty in practice. We also change some of the default window sizes for manually configured PHBs to saner values. Finally we adjust some tests and libqos so that it correctly uses the new default locations. Ideally it would parse the device tree given to the guest, but that's a more complex problem for another time. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-16 03:04:15 +02:00
/*
* New-style PHB window placement.
*
* Goals: Gives large (1TiB), naturally aligned 64-bit MMIO window
* for each PHB, in addition to 2GiB 32-bit MMIO and 64kiB PIO
* windows.
*
* Some guest kernels can't work with MMIO windows above 1<<46
* (64TiB), so we place up to 31 PHBs in the area 32TiB..64TiB
*
* 32TiB..(33TiB+1984kiB) contains the 64kiB PIO windows for each
* PHB stacked together. (32TiB+2GiB)..(32TiB+64GiB) contains the
* 2GiB 32-bit MMIO windows for each PHB. Then 33..64TiB has the
* 1TiB 64-bit MMIO windows for each PHB.
*/
spapr_pci: Delegate placement of PCI host bridges to machine type The 'spapr-pci-host-bridge' represents the virtual PCI host bridge (PHB) for a PAPR guest. Unlike on x86, it's routine on Power (both bare metal and PAPR guests) to have numerous independent PHBs, each controlling a separate PCI domain. There are two ways of configuring the spapr-pci-host-bridge device: first it can be done fully manually, specifying the locations and sizes of all the IO windows. This gives the most control, but is very awkward with 6 mandatory parameters. Alternatively just an "index" can be specified which essentially selects from an array of predefined PHB locations. The PHB at index 0 is automatically created as the default PHB. The current set of default locations causes some problems for guests with large RAM (> 1 TiB) or PCI devices with very large BARs (e.g. big nVidia GPGPU cards via VFIO). Obviously, for migration we can only change the locations on a new machine type, however. This is awkward, because the placement is currently decided within the spapr-pci-host-bridge code, so it breaks abstraction to look inside the machine type version. So, this patch delegates the "default mode" PHB placement from the spapr-pci-host-bridge device back to the machine type via a public method in sPAPRMachineClass. It's still a bit ugly, but it's about the best we can do. For now, this just changes where the calculation is done. It doesn't change the actual location of the host bridges, or any other behaviour. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-13 01:26:09 +02:00
const uint64_t base_buid = 0x800000020000000ULL;
int i;
spapr: Improved placement of PCI host bridges in guest memory map Currently, the MMIO space for accessing PCI on pseries guests begins at 1 TiB in guest address space. Each PCI host bridge (PHB) has a 64 GiB chunk of address space in which it places its outbound PIO and 32-bit and 64-bit MMIO windows. This scheme as several problems: - It limits guest RAM to 1 TiB (though we have a limited fix for this now) - It limits the total MMIO window to 64 GiB. This is not always enough for some of the large nVidia GPGPU cards - Putting all the windows into a single 64 GiB area means that naturally aligning things within there will waste more address space. In addition there was a miscalculation in some of the defaults, which meant that the MMIO windows for each PHB actually slightly overran the 64 GiB region for that PHB. We got away without nasty consequences because the overrun fit within an unused area at the beginning of the next PHB's region, but it's not pretty. This patch implements a new scheme which addresses those problems, and is also closer to what bare metal hardware and pHyp guests generally use. Because some guest versions (including most current distro kernels) can't access PCI MMIO above 64 TiB, we put all the PCI windows between 32 TiB and 64 TiB. This is broken into 1 TiB chunks. The first 1 TiB contains the PIO (64 kiB) and 32-bit MMIO (2 GiB) windows for all of the PHBs. Each subsequent TiB chunk contains a naturally aligned 64-bit MMIO window for one PHB each. This reduces the number of allowed PHBs (without full manual configuration of all the windows) from 256 to 31, but this should still be plenty in practice. We also change some of the default window sizes for manually configured PHBs to saner values. Finally we adjust some tests and libqos so that it correctly uses the new default locations. Ideally it would parse the device tree given to the guest, but that's a more complex problem for another time. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-16 03:04:15 +02:00
/* Sanity check natural alignments */
QEMU_BUILD_BUG_ON((SPAPR_PCI_BASE % SPAPR_PCI_MEM64_WIN_SIZE) != 0);
QEMU_BUILD_BUG_ON((SPAPR_PCI_LIMIT % SPAPR_PCI_MEM64_WIN_SIZE) != 0);
QEMU_BUILD_BUG_ON((SPAPR_PCI_MEM64_WIN_SIZE % SPAPR_PCI_MEM32_WIN_SIZE) != 0);
QEMU_BUILD_BUG_ON((SPAPR_PCI_MEM32_WIN_SIZE % SPAPR_PCI_IO_WIN_SIZE) != 0);
/* Sanity check bounds */
QEMU_BUILD_BUG_ON((SPAPR_MAX_PHBS * SPAPR_PCI_IO_WIN_SIZE) >
SPAPR_PCI_MEM32_WIN_SIZE);
QEMU_BUILD_BUG_ON((SPAPR_MAX_PHBS * SPAPR_PCI_MEM32_WIN_SIZE) >
SPAPR_PCI_MEM64_WIN_SIZE);
if (index >= SPAPR_MAX_PHBS) {
error_setg(errp, "\"index\" for PAPR PHB is too large (max %llu)",
SPAPR_MAX_PHBS - 1);
spapr_pci: Delegate placement of PCI host bridges to machine type The 'spapr-pci-host-bridge' represents the virtual PCI host bridge (PHB) for a PAPR guest. Unlike on x86, it's routine on Power (both bare metal and PAPR guests) to have numerous independent PHBs, each controlling a separate PCI domain. There are two ways of configuring the spapr-pci-host-bridge device: first it can be done fully manually, specifying the locations and sizes of all the IO windows. This gives the most control, but is very awkward with 6 mandatory parameters. Alternatively just an "index" can be specified which essentially selects from an array of predefined PHB locations. The PHB at index 0 is automatically created as the default PHB. The current set of default locations causes some problems for guests with large RAM (> 1 TiB) or PCI devices with very large BARs (e.g. big nVidia GPGPU cards via VFIO). Obviously, for migration we can only change the locations on a new machine type, however. This is awkward, because the placement is currently decided within the spapr-pci-host-bridge code, so it breaks abstraction to look inside the machine type version. So, this patch delegates the "default mode" PHB placement from the spapr-pci-host-bridge device back to the machine type via a public method in sPAPRMachineClass. It's still a bit ugly, but it's about the best we can do. For now, this just changes where the calculation is done. It doesn't change the actual location of the host bridges, or any other behaviour. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-13 01:26:09 +02:00
return;
}
*buid = base_buid + index;
for (i = 0; i < n_dma; ++i) {
liobns[i] = SPAPR_PCI_LIOBN(index, i);
}
spapr: Improved placement of PCI host bridges in guest memory map Currently, the MMIO space for accessing PCI on pseries guests begins at 1 TiB in guest address space. Each PCI host bridge (PHB) has a 64 GiB chunk of address space in which it places its outbound PIO and 32-bit and 64-bit MMIO windows. This scheme as several problems: - It limits guest RAM to 1 TiB (though we have a limited fix for this now) - It limits the total MMIO window to 64 GiB. This is not always enough for some of the large nVidia GPGPU cards - Putting all the windows into a single 64 GiB area means that naturally aligning things within there will waste more address space. In addition there was a miscalculation in some of the defaults, which meant that the MMIO windows for each PHB actually slightly overran the 64 GiB region for that PHB. We got away without nasty consequences because the overrun fit within an unused area at the beginning of the next PHB's region, but it's not pretty. This patch implements a new scheme which addresses those problems, and is also closer to what bare metal hardware and pHyp guests generally use. Because some guest versions (including most current distro kernels) can't access PCI MMIO above 64 TiB, we put all the PCI windows between 32 TiB and 64 TiB. This is broken into 1 TiB chunks. The first 1 TiB contains the PIO (64 kiB) and 32-bit MMIO (2 GiB) windows for all of the PHBs. Each subsequent TiB chunk contains a naturally aligned 64-bit MMIO window for one PHB each. This reduces the number of allowed PHBs (without full manual configuration of all the windows) from 256 to 31, but this should still be plenty in practice. We also change some of the default window sizes for manually configured PHBs to saner values. Finally we adjust some tests and libqos so that it correctly uses the new default locations. Ideally it would parse the device tree given to the guest, but that's a more complex problem for another time. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-16 03:04:15 +02:00
*pio = SPAPR_PCI_BASE + index * SPAPR_PCI_IO_WIN_SIZE;
*mmio32 = SPAPR_PCI_BASE + (index + 1) * SPAPR_PCI_MEM32_WIN_SIZE;
*mmio64 = SPAPR_PCI_BASE + (index + 1) * SPAPR_PCI_MEM64_WIN_SIZE;
spapr_pci: Delegate placement of PCI host bridges to machine type The 'spapr-pci-host-bridge' represents the virtual PCI host bridge (PHB) for a PAPR guest. Unlike on x86, it's routine on Power (both bare metal and PAPR guests) to have numerous independent PHBs, each controlling a separate PCI domain. There are two ways of configuring the spapr-pci-host-bridge device: first it can be done fully manually, specifying the locations and sizes of all the IO windows. This gives the most control, but is very awkward with 6 mandatory parameters. Alternatively just an "index" can be specified which essentially selects from an array of predefined PHB locations. The PHB at index 0 is automatically created as the default PHB. The current set of default locations causes some problems for guests with large RAM (> 1 TiB) or PCI devices with very large BARs (e.g. big nVidia GPGPU cards via VFIO). Obviously, for migration we can only change the locations on a new machine type, however. This is awkward, because the placement is currently decided within the spapr-pci-host-bridge code, so it breaks abstraction to look inside the machine type version. So, this patch delegates the "default mode" PHB placement from the spapr-pci-host-bridge device back to the machine type via a public method in sPAPRMachineClass. It's still a bit ugly, but it's about the best we can do. For now, this just changes where the calculation is done. It doesn't change the actual location of the host bridges, or any other behaviour. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-13 01:26:09 +02:00
}
static ICSState *spapr_ics_get(XICSFabric *dev, int irq)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(dev);
return ics_valid_irq(spapr->ics, irq) ? spapr->ics : NULL;
}
static void spapr_ics_resend(XICSFabric *dev)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(dev);
ics_resend(spapr->ics);
}
static ICPState *spapr_icp_get(XICSFabric *xi, int vcpu_id)
{
PowerPCCPU *cpu = spapr_find_cpu(vcpu_id);
return cpu ? spapr_cpu_state(cpu)->icp : NULL;
}
static void spapr_pic_print_info(InterruptStatsProvider *obj,
Monitor *mon)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
spapr->irq->print_info(spapr, mon);
}
int spapr_get_vcpu_id(PowerPCCPU *cpu)
{
spapr: fix missing CPU core nodes in DT when running with TCG Commit 5d0fb1508e2d "spapr: consolidate the VCPU id numbering logic in a single place" introduced a helper to detect thread0 of a virtual core based on its VCPU id. This is used to create CPU core nodes in the DT, but it is broken in TCG. $ qemu-system-ppc64 -nographic -accel tcg -machine dumpdtb=dtb.bin \ -smp cores=16,maxcpus=16,threads=1 $ dtc -f -O dts dtb.bin | grep POWER8 PowerPC,POWER8@0 { PowerPC,POWER8@8 { instead of the expected 16 cores that we get with KVM: $ dtc -f -O dts dtb.bin | grep POWER8 PowerPC,POWER8@0 { PowerPC,POWER8@8 { PowerPC,POWER8@10 { PowerPC,POWER8@18 { PowerPC,POWER8@20 { PowerPC,POWER8@28 { PowerPC,POWER8@30 { PowerPC,POWER8@38 { PowerPC,POWER8@40 { PowerPC,POWER8@48 { PowerPC,POWER8@50 { PowerPC,POWER8@58 { PowerPC,POWER8@60 { PowerPC,POWER8@68 { PowerPC,POWER8@70 { PowerPC,POWER8@78 { This happens because spapr_get_vcpu_id() maps VCPU ids to cs->cpu_index in TCG mode. This confuses the code in spapr_is_thread0_in_vcore(), since it assumes thread0 VCPU ids to have a spapr->vsmt spacing. spapr_get_vcpu_id(cpu) % spapr->vsmt == 0 Actually, there's no real reason to expose cs->cpu_index instead of the VCPU id, since we also generate it with TCG. Also we already set it explicitly in spapr_set_vcpu_id(), so there's no real reason either to call kvm_arch_vcpu_id() with KVM. This patch unifies spapr_get_vcpu_id() to always return the computed VCPU id both in TCG and KVM. This is one step forward towards KVM<->TCG migration. Fixes: 5d0fb1508e2d Reported-by: Cédric Le Goater <clg@kaod.org> Signed-off-by: Greg Kurz <groug@kaod.org> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2018-02-16 19:58:06 +01:00
return cpu->vcpu_id;
}
void spapr_set_vcpu_id(PowerPCCPU *cpu, int cpu_index, Error **errp)
{
sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
int vcpu_id;
vcpu_id = spapr_vcpu_id(spapr, cpu_index);
if (kvm_enabled() && !kvm_vcpu_id_is_valid(vcpu_id)) {
error_setg(errp, "Can't create CPU with id %d in KVM", vcpu_id);
error_append_hint(errp, "Adjust the number of cpus to %d "
"or try to raise the number of threads per core\n",
vcpu_id * smp_threads / spapr->vsmt);
return;
}
cpu->vcpu_id = vcpu_id;
}
PowerPCCPU *spapr_find_cpu(int vcpu_id)
{
CPUState *cs;
CPU_FOREACH(cs) {
PowerPCCPU *cpu = POWERPC_CPU(cs);
if (spapr_get_vcpu_id(cpu) == vcpu_id) {
return cpu;
}
}
return NULL;
}
static void spapr_machine_class_init(ObjectClass *oc, void *data)
{
MachineClass *mc = MACHINE_CLASS(oc);
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(oc);
FWPathProviderClass *fwc = FW_PATH_PROVIDER_CLASS(oc);
NMIClass *nc = NMI_CLASS(oc);
HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
PPCVirtualHypervisorClass *vhc = PPC_VIRTUAL_HYPERVISOR_CLASS(oc);
XICSFabricClass *xic = XICS_FABRIC_CLASS(oc);
InterruptStatsProviderClass *ispc = INTERRUPT_STATS_PROVIDER_CLASS(oc);
mc->desc = "pSeries Logical Partition (PAPR compliant)";
mc->ignore_boot_device_suffixes = true;
/*
* We set up the default / latest behaviour here. The class_init
* functions for the specific versioned machine types can override
* these details for backwards compatibility
*/
mc->init = spapr_machine_init;
mc->reset = spapr_machine_reset;
mc->block_default_type = IF_SCSI;
mc->max_cpus = 1024;
mc->no_parallel = 1;
mc->default_boot_order = "";
mc->default_ram_size = 512 * MiB;
mc->default_display = "std";
mc->kvm_type = spapr_kvm_type;
machine_class_allow_dynamic_sysbus_dev(mc, TYPE_SPAPR_PCI_HOST_BRIDGE);
mc->pci_allow_0_address = true;
assert(!mc->get_hotplug_handler);
mc->get_hotplug_handler = spapr_get_hotplug_handler;
hc->pre_plug = spapr_machine_device_pre_plug;
hc->plug = spapr_machine_device_plug;
mc->cpu_index_to_instance_props = spapr_cpu_index_to_props;
mc->get_default_cpu_node_id = spapr_get_default_cpu_node_id;
mc->possible_cpu_arch_ids = spapr_possible_cpu_arch_ids;
hc->unplug_request = spapr_machine_device_unplug_request;
hc->unplug = spapr_machine_device_unplug;
smc->dr_lmb_enabled = true;
smc->update_dt_enabled = true;
mc->default_cpu_type = POWERPC_CPU_TYPE_NAME("power9_v2.0");
mc->has_hotpluggable_cpus = true;
smc->resize_hpt_default = SPAPR_RESIZE_HPT_ENABLED;
fwc->get_dev_path = spapr_get_fw_dev_path;
nc->nmi_monitor_handler = spapr_nmi;
spapr_pci: Delegate placement of PCI host bridges to machine type The 'spapr-pci-host-bridge' represents the virtual PCI host bridge (PHB) for a PAPR guest. Unlike on x86, it's routine on Power (both bare metal and PAPR guests) to have numerous independent PHBs, each controlling a separate PCI domain. There are two ways of configuring the spapr-pci-host-bridge device: first it can be done fully manually, specifying the locations and sizes of all the IO windows. This gives the most control, but is very awkward with 6 mandatory parameters. Alternatively just an "index" can be specified which essentially selects from an array of predefined PHB locations. The PHB at index 0 is automatically created as the default PHB. The current set of default locations causes some problems for guests with large RAM (> 1 TiB) or PCI devices with very large BARs (e.g. big nVidia GPGPU cards via VFIO). Obviously, for migration we can only change the locations on a new machine type, however. This is awkward, because the placement is currently decided within the spapr-pci-host-bridge code, so it breaks abstraction to look inside the machine type version. So, this patch delegates the "default mode" PHB placement from the spapr-pci-host-bridge device back to the machine type via a public method in sPAPRMachineClass. It's still a bit ugly, but it's about the best we can do. For now, this just changes where the calculation is done. It doesn't change the actual location of the host bridges, or any other behaviour. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-13 01:26:09 +02:00
smc->phb_placement = spapr_phb_placement;
vhc->hypercall = emulate_spapr_hypercall;
vhc->hpt_mask = spapr_hpt_mask;
vhc->map_hptes = spapr_map_hptes;
vhc->unmap_hptes = spapr_unmap_hptes;
vhc->store_hpte = spapr_store_hpte;
vhc->get_pate = spapr_get_pate;
vhc->encode_hpt_for_kvm_pr = spapr_encode_hpt_for_kvm_pr;
xic->ics_get = spapr_ics_get;
xic->ics_resend = spapr_ics_resend;
xic->icp_get = spapr_icp_get;
ispc->print_info = spapr_pic_print_info;
/* Force NUMA node memory size to be a multiple of
* SPAPR_MEMORY_BLOCK_SIZE (256M) since that's the granularity
* in which LMBs are represented and hot-added
*/
mc->numa_mem_align_shift = 28;
spapr: Capabilities infrastructure Because PAPR is a paravirtual environment access to certain CPU (or other) facilities can be blocked by the hypervisor. PAPR provides ways to advertise in the device tree whether or not those features are available to the guest. In some places we automatically determine whether to make a feature available based on whether our host can support it, in most cases this is based on limitations in the available KVM implementation. Although we correctly advertise this to the guest, it means that host factors might make changes to the guest visible environment which is bad: as well as generaly reducing reproducibility, it means that a migration between different host environments can easily go bad. We've mostly gotten away with it because the environments considered mature enough to be well supported (basically, KVM on POWER8) have had consistent feature availability. But, it's still not right and some limitations on POWER9 is going to make it more of an issue in future. This introduces an infrastructure for defining "sPAPR capabilities". These are set by default based on the machine version, masked by the capabilities of the chosen cpu, but can be overriden with machine properties. The intention is at reset time we verify that the requested capabilities can be supported on the host (considering TCG, KVM and/or host cpu limitations). If not we simply fail, rather than silently modifying the advertised featureset to the guest. This does mean that certain configurations that "worked" may now fail, but such configurations were already more subtly broken. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Greg Kurz <groug@kaod.org>
2017-12-08 00:35:35 +01:00
smc->default_caps.caps[SPAPR_CAP_HTM] = SPAPR_CAP_OFF;
smc->default_caps.caps[SPAPR_CAP_VSX] = SPAPR_CAP_ON;
smc->default_caps.caps[SPAPR_CAP_DFP] = SPAPR_CAP_ON;
smc->default_caps.caps[SPAPR_CAP_CFPC] = SPAPR_CAP_BROKEN;
smc->default_caps.caps[SPAPR_CAP_SBBC] = SPAPR_CAP_BROKEN;
smc->default_caps.caps[SPAPR_CAP_IBS] = SPAPR_CAP_BROKEN;
smc->default_caps.caps[SPAPR_CAP_HPT_MAXPAGESIZE] = 16; /* 64kiB */
smc->default_caps.caps[SPAPR_CAP_NESTED_KVM_HV] = SPAPR_CAP_OFF;
spapr: Capabilities infrastructure Because PAPR is a paravirtual environment access to certain CPU (or other) facilities can be blocked by the hypervisor. PAPR provides ways to advertise in the device tree whether or not those features are available to the guest. In some places we automatically determine whether to make a feature available based on whether our host can support it, in most cases this is based on limitations in the available KVM implementation. Although we correctly advertise this to the guest, it means that host factors might make changes to the guest visible environment which is bad: as well as generaly reducing reproducibility, it means that a migration between different host environments can easily go bad. We've mostly gotten away with it because the environments considered mature enough to be well supported (basically, KVM on POWER8) have had consistent feature availability. But, it's still not right and some limitations on POWER9 is going to make it more of an issue in future. This introduces an infrastructure for defining "sPAPR capabilities". These are set by default based on the machine version, masked by the capabilities of the chosen cpu, but can be overriden with machine properties. The intention is at reset time we verify that the requested capabilities can be supported on the host (considering TCG, KVM and/or host cpu limitations). If not we simply fail, rather than silently modifying the advertised featureset to the guest. This does mean that certain configurations that "worked" may now fail, but such configurations were already more subtly broken. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Greg Kurz <groug@kaod.org>
2017-12-08 00:35:35 +01:00
spapr_caps_add_properties(smc, &error_abort);
smc->irq = &spapr_irq_xics;
smc->dr_phb_enabled = true;
}
static const TypeInfo spapr_machine_info = {
.name = TYPE_SPAPR_MACHINE,
.parent = TYPE_MACHINE,
.abstract = true,
.instance_size = sizeof(sPAPRMachineState),
.instance_init = spapr_instance_init,
.instance_finalize = spapr_machine_finalizefn,
.class_size = sizeof(sPAPRMachineClass),
.class_init = spapr_machine_class_init,
.interfaces = (InterfaceInfo[]) {
{ TYPE_FW_PATH_PROVIDER },
{ TYPE_NMI },
{ TYPE_HOTPLUG_HANDLER },
{ TYPE_PPC_VIRTUAL_HYPERVISOR },
{ TYPE_XICS_FABRIC },
{ TYPE_INTERRUPT_STATS_PROVIDER },
{ }
},
};
#define DEFINE_SPAPR_MACHINE(suffix, verstr, latest) \
static void spapr_machine_##suffix##_class_init(ObjectClass *oc, \
void *data) \
{ \
MachineClass *mc = MACHINE_CLASS(oc); \
spapr_machine_##suffix##_class_options(mc); \
if (latest) { \
mc->alias = "pseries"; \
mc->is_default = 1; \
} \
} \
static const TypeInfo spapr_machine_##suffix##_info = { \
.name = MACHINE_TYPE_NAME("pseries-" verstr), \
.parent = TYPE_SPAPR_MACHINE, \
.class_init = spapr_machine_##suffix##_class_init, \
}; \
static void spapr_machine_register_##suffix(void) \
{ \
type_register(&spapr_machine_##suffix##_info); \
} \
type_init(spapr_machine_register_##suffix)
/*
* pseries-4.0
*/
static void spapr_machine_4_0_class_options(MachineClass *mc)
{
/* Defaults for the latest behaviour inherited from the base class */
}
DEFINE_SPAPR_MACHINE(4_0, "4.0", true);
/*
* pseries-3.1
*/
static void spapr_machine_3_1_class_options(MachineClass *mc)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
static GlobalProperty compat[] = {
{ TYPE_SPAPR_MACHINE, "host-model", "passthrough" },
{ TYPE_SPAPR_MACHINE, "host-serial", "passthrough" },
};
spapr_machine_4_0_class_options(mc);
compat_props_add(mc->compat_props, hw_compat_3_1, hw_compat_3_1_len);
compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
mc->default_cpu_type = POWERPC_CPU_TYPE_NAME("power8_v2.0");
smc->update_dt_enabled = false;
smc->dr_phb_enabled = false;
}
DEFINE_SPAPR_MACHINE(3_1, "3.1", false);
/*
* pseries-3.0
*/
static void spapr_machine_3_0_class_options(MachineClass *mc)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
spapr_machine_3_1_class_options(mc);
compat_props_add(mc->compat_props, hw_compat_3_0, hw_compat_3_0_len);
smc->legacy_irq_allocation = true;
smc->irq = &spapr_irq_xics_legacy;
}
DEFINE_SPAPR_MACHINE(3_0, "3.0", false);
/*
* pseries-2.12
*/
static void spapr_machine_2_12_class_options(MachineClass *mc)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
static GlobalProperty compat[] = {
{ TYPE_POWERPC_CPU, "pre-3.0-migration", "on" },
{ TYPE_SPAPR_CPU_CORE, "pre-3.0-migration", "on" },
};
spapr_machine_3_0_class_options(mc);
compat_props_add(mc->compat_props, hw_compat_2_12, hw_compat_2_12_len);
compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
spapr: compute default value of "hpt-max-page-size" later It is currently not possible to run a pseries-2.12 or older machine with HV KVM. QEMU prints the following and exits right away. qemu-system-ppc64: KVM doesn't support for base page shift 34 The "hpt-max-page-size" capability was recently added to spapr to hide host configuration details from HPT mode guests. Its default value for newer machine types is 64k. For backwards compatibility, pseries-2.12 and older machine types need a different value. This is handled as usual in a class init function. The default value is 16G, ie, all page sizes supported by POWER7 and newer CPUs, but HV KVM requires guest pages to be hpa contiguous as well as gpa contiguous. The default value is the page size used to back the guest RAM in this case. Unfortunately kvmppc_hpt_needs_host_contiguous_pages()->kvm_enabled() is called way before KVM init and returns false, even if the user requested KVM. We thus end up selecting 16G, which isn't supported by HV KVM. The default value must be set during machine init, because we can safely assume that KVM is initialized at this point. We fix this by moving the logic to default_caps_with_cpu(). Since the user cannot pass cap-hpt-max-page-size=0, we set the default to 0 in the pseries-2.12 class init function and use that as a flag to do the real work. Signed-off-by: Greg Kurz <groug@kaod.org> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2018-07-02 10:54:56 +02:00
/* We depend on kvm_enabled() to choose a default value for the
* hpt-max-page-size capability. Of course we can't do it here
* because this is too early and the HW accelerator isn't initialzed
* yet. Postpone this to machine init (see default_caps_with_cpu()).
*/
smc->default_caps.caps[SPAPR_CAP_HPT_MAXPAGESIZE] = 0;
}
DEFINE_SPAPR_MACHINE(2_12, "2.12", false);
static void spapr_machine_2_12_sxxm_class_options(MachineClass *mc)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
spapr_machine_2_12_class_options(mc);
smc->default_caps.caps[SPAPR_CAP_CFPC] = SPAPR_CAP_WORKAROUND;
smc->default_caps.caps[SPAPR_CAP_SBBC] = SPAPR_CAP_WORKAROUND;
smc->default_caps.caps[SPAPR_CAP_IBS] = SPAPR_CAP_FIXED_CCD;
}
DEFINE_SPAPR_MACHINE(2_12_sxxm, "2.12-sxxm", false);
/*
* pseries-2.11
*/
static void spapr_machine_2_11_class_options(MachineClass *mc)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
spapr_machine_2_12_class_options(mc);
smc->default_caps.caps[SPAPR_CAP_HTM] = SPAPR_CAP_ON;
compat_props_add(mc->compat_props, hw_compat_2_11, hw_compat_2_11_len);
}
DEFINE_SPAPR_MACHINE(2_11, "2.11", false);
/*
* pseries-2.10
*/
static void spapr_machine_2_10_class_options(MachineClass *mc)
{
spapr_machine_2_11_class_options(mc);
compat_props_add(mc->compat_props, hw_compat_2_10, hw_compat_2_10_len);
}
DEFINE_SPAPR_MACHINE(2_10, "2.10", false);
/*
* pseries-2.9
*/
static void spapr_machine_2_9_class_options(MachineClass *mc)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
static GlobalProperty compat[] = {
{ TYPE_POWERPC_CPU, "pre-2.10-migration", "on" },
};
spapr_machine_2_10_class_options(mc);
compat_props_add(mc->compat_props, hw_compat_2_9, hw_compat_2_9_len);
compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
numa: equally distribute memory on nodes When there are more nodes than available memory to put the minimum allowed memory by node, all the memory is put on the last node. This is because we put (ram_size / nb_numa_nodes) & ~((1 << mc->numa_mem_align_shift) - 1); on each node, and in this case the value is 0. This is particularly true with pseries, as the memory must be aligned to 256MB. To avoid this problem, this patch uses an error diffusion algorithm [1] to distribute equally the memory on nodes. We introduce numa_auto_assign_ram() function in MachineClass to keep compatibility between machine type versions. The legacy function is used with pseries-2.9, pc-q35-2.9 and pc-i440fx-2.9 (and previous), the new one with all others. Example: qemu-system-ppc64 -S -nographic -nodefaults -monitor stdio -m 1G -smp 8 \ -numa node -numa node -numa node \ -numa node -numa node -numa node Before: (qemu) info numa 6 nodes node 0 cpus: 0 6 node 0 size: 0 MB node 1 cpus: 1 7 node 1 size: 0 MB node 2 cpus: 2 node 2 size: 0 MB node 3 cpus: 3 node 3 size: 0 MB node 4 cpus: 4 node 4 size: 0 MB node 5 cpus: 5 node 5 size: 1024 MB After: (qemu) info numa 6 nodes node 0 cpus: 0 6 node 0 size: 0 MB node 1 cpus: 1 7 node 1 size: 256 MB node 2 cpus: 2 node 2 size: 0 MB node 3 cpus: 3 node 3 size: 256 MB node 4 cpus: 4 node 4 size: 256 MB node 5 cpus: 5 node 5 size: 256 MB [1] https://en.wikipedia.org/wiki/Error_diffusion Signed-off-by: Laurent Vivier <lvivier@redhat.com> Message-Id: <20170502162955.1610-2-lvivier@redhat.com> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com> [ehabkost: s/ram_size/size/ at numa_default_auto_assign_ram()] Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2017-05-02 18:29:55 +02:00
mc->numa_auto_assign_ram = numa_legacy_auto_assign_ram;
smc->pre_2_10_has_unused_icps = true;
smc->resize_hpt_default = SPAPR_RESIZE_HPT_DISABLED;
}
DEFINE_SPAPR_MACHINE(2_9, "2.9", false);
/*
* pseries-2.8
*/
static void spapr_machine_2_8_class_options(MachineClass *mc)
{
static GlobalProperty compat[] = {
{ TYPE_SPAPR_PCI_HOST_BRIDGE, "pcie-extended-configuration-space", "off" },
};
spapr_machine_2_9_class_options(mc);
compat_props_add(mc->compat_props, hw_compat_2_8, hw_compat_2_8_len);
compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
mc->numa_mem_align_shift = 23;
}
DEFINE_SPAPR_MACHINE(2_8, "2.8", false);
/*
* pseries-2.7
*/
spapr: Improved placement of PCI host bridges in guest memory map Currently, the MMIO space for accessing PCI on pseries guests begins at 1 TiB in guest address space. Each PCI host bridge (PHB) has a 64 GiB chunk of address space in which it places its outbound PIO and 32-bit and 64-bit MMIO windows. This scheme as several problems: - It limits guest RAM to 1 TiB (though we have a limited fix for this now) - It limits the total MMIO window to 64 GiB. This is not always enough for some of the large nVidia GPGPU cards - Putting all the windows into a single 64 GiB area means that naturally aligning things within there will waste more address space. In addition there was a miscalculation in some of the defaults, which meant that the MMIO windows for each PHB actually slightly overran the 64 GiB region for that PHB. We got away without nasty consequences because the overrun fit within an unused area at the beginning of the next PHB's region, but it's not pretty. This patch implements a new scheme which addresses those problems, and is also closer to what bare metal hardware and pHyp guests generally use. Because some guest versions (including most current distro kernels) can't access PCI MMIO above 64 TiB, we put all the PCI windows between 32 TiB and 64 TiB. This is broken into 1 TiB chunks. The first 1 TiB contains the PIO (64 kiB) and 32-bit MMIO (2 GiB) windows for all of the PHBs. Each subsequent TiB chunk contains a naturally aligned 64-bit MMIO window for one PHB each. This reduces the number of allowed PHBs (without full manual configuration of all the windows) from 256 to 31, but this should still be plenty in practice. We also change some of the default window sizes for manually configured PHBs to saner values. Finally we adjust some tests and libqos so that it correctly uses the new default locations. Ideally it would parse the device tree given to the guest, but that's a more complex problem for another time. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-16 03:04:15 +02:00
static void phb_placement_2_7(sPAPRMachineState *spapr, uint32_t index,
uint64_t *buid, hwaddr *pio,
hwaddr *mmio32, hwaddr *mmio64,
unsigned n_dma, uint32_t *liobns, Error **errp)
{
/* Legacy PHB placement for pseries-2.7 and earlier machine types */
const uint64_t base_buid = 0x800000020000000ULL;
const hwaddr phb_spacing = 0x1000000000ULL; /* 64 GiB */
const hwaddr mmio_offset = 0xa0000000; /* 2 GiB + 512 MiB */
const hwaddr pio_offset = 0x80000000; /* 2 GiB */
const uint32_t max_index = 255;
const hwaddr phb0_alignment = 0x10000000000ULL; /* 1 TiB */
uint64_t ram_top = MACHINE(spapr)->ram_size;
hwaddr phb0_base, phb_base;
int i;
/* Do we have device memory? */
spapr: Improved placement of PCI host bridges in guest memory map Currently, the MMIO space for accessing PCI on pseries guests begins at 1 TiB in guest address space. Each PCI host bridge (PHB) has a 64 GiB chunk of address space in which it places its outbound PIO and 32-bit and 64-bit MMIO windows. This scheme as several problems: - It limits guest RAM to 1 TiB (though we have a limited fix for this now) - It limits the total MMIO window to 64 GiB. This is not always enough for some of the large nVidia GPGPU cards - Putting all the windows into a single 64 GiB area means that naturally aligning things within there will waste more address space. In addition there was a miscalculation in some of the defaults, which meant that the MMIO windows for each PHB actually slightly overran the 64 GiB region for that PHB. We got away without nasty consequences because the overrun fit within an unused area at the beginning of the next PHB's region, but it's not pretty. This patch implements a new scheme which addresses those problems, and is also closer to what bare metal hardware and pHyp guests generally use. Because some guest versions (including most current distro kernels) can't access PCI MMIO above 64 TiB, we put all the PCI windows between 32 TiB and 64 TiB. This is broken into 1 TiB chunks. The first 1 TiB contains the PIO (64 kiB) and 32-bit MMIO (2 GiB) windows for all of the PHBs. Each subsequent TiB chunk contains a naturally aligned 64-bit MMIO window for one PHB each. This reduces the number of allowed PHBs (without full manual configuration of all the windows) from 256 to 31, but this should still be plenty in practice. We also change some of the default window sizes for manually configured PHBs to saner values. Finally we adjust some tests and libqos so that it correctly uses the new default locations. Ideally it would parse the device tree given to the guest, but that's a more complex problem for another time. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-16 03:04:15 +02:00
if (MACHINE(spapr)->maxram_size > ram_top) {
/* Can't just use maxram_size, because there may be an
* alignment gap between normal and device memory regions
*/
ram_top = MACHINE(spapr)->device_memory->base +
memory_region_size(&MACHINE(spapr)->device_memory->mr);
spapr: Improved placement of PCI host bridges in guest memory map Currently, the MMIO space for accessing PCI on pseries guests begins at 1 TiB in guest address space. Each PCI host bridge (PHB) has a 64 GiB chunk of address space in which it places its outbound PIO and 32-bit and 64-bit MMIO windows. This scheme as several problems: - It limits guest RAM to 1 TiB (though we have a limited fix for this now) - It limits the total MMIO window to 64 GiB. This is not always enough for some of the large nVidia GPGPU cards - Putting all the windows into a single 64 GiB area means that naturally aligning things within there will waste more address space. In addition there was a miscalculation in some of the defaults, which meant that the MMIO windows for each PHB actually slightly overran the 64 GiB region for that PHB. We got away without nasty consequences because the overrun fit within an unused area at the beginning of the next PHB's region, but it's not pretty. This patch implements a new scheme which addresses those problems, and is also closer to what bare metal hardware and pHyp guests generally use. Because some guest versions (including most current distro kernels) can't access PCI MMIO above 64 TiB, we put all the PCI windows between 32 TiB and 64 TiB. This is broken into 1 TiB chunks. The first 1 TiB contains the PIO (64 kiB) and 32-bit MMIO (2 GiB) windows for all of the PHBs. Each subsequent TiB chunk contains a naturally aligned 64-bit MMIO window for one PHB each. This reduces the number of allowed PHBs (without full manual configuration of all the windows) from 256 to 31, but this should still be plenty in practice. We also change some of the default window sizes for manually configured PHBs to saner values. Finally we adjust some tests and libqos so that it correctly uses the new default locations. Ideally it would parse the device tree given to the guest, but that's a more complex problem for another time. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-16 03:04:15 +02:00
}
phb0_base = QEMU_ALIGN_UP(ram_top, phb0_alignment);
if (index > max_index) {
error_setg(errp, "\"index\" for PAPR PHB is too large (max %u)",
max_index);
return;
}
*buid = base_buid + index;
for (i = 0; i < n_dma; ++i) {
liobns[i] = SPAPR_PCI_LIOBN(index, i);
}
phb_base = phb0_base + index * phb_spacing;
*pio = phb_base + pio_offset;
*mmio32 = phb_base + mmio_offset;
/*
* We don't set the 64-bit MMIO window, relying on the PHB's
* fallback behaviour of automatically splitting a large "32-bit"
* window into contiguous 32-bit and 64-bit windows
*/
}
static void spapr_machine_2_7_class_options(MachineClass *mc)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
static GlobalProperty compat[] = {
{ TYPE_SPAPR_PCI_HOST_BRIDGE, "mem_win_size", "0xf80000000", },
{ TYPE_SPAPR_PCI_HOST_BRIDGE, "mem64_win_size", "0", },
{ TYPE_POWERPC_CPU, "pre-2.8-migration", "on", },
{ TYPE_SPAPR_PCI_HOST_BRIDGE, "pre-2.8-migration", "on", },
};
spapr_machine_2_8_class_options(mc);
mc->default_cpu_type = POWERPC_CPU_TYPE_NAME("power7_v2.3");
mc->default_machine_opts = "modern-hotplug-events=off";
compat_props_add(mc->compat_props, hw_compat_2_7, hw_compat_2_7_len);
compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
spapr: Improved placement of PCI host bridges in guest memory map Currently, the MMIO space for accessing PCI on pseries guests begins at 1 TiB in guest address space. Each PCI host bridge (PHB) has a 64 GiB chunk of address space in which it places its outbound PIO and 32-bit and 64-bit MMIO windows. This scheme as several problems: - It limits guest RAM to 1 TiB (though we have a limited fix for this now) - It limits the total MMIO window to 64 GiB. This is not always enough for some of the large nVidia GPGPU cards - Putting all the windows into a single 64 GiB area means that naturally aligning things within there will waste more address space. In addition there was a miscalculation in some of the defaults, which meant that the MMIO windows for each PHB actually slightly overran the 64 GiB region for that PHB. We got away without nasty consequences because the overrun fit within an unused area at the beginning of the next PHB's region, but it's not pretty. This patch implements a new scheme which addresses those problems, and is also closer to what bare metal hardware and pHyp guests generally use. Because some guest versions (including most current distro kernels) can't access PCI MMIO above 64 TiB, we put all the PCI windows between 32 TiB and 64 TiB. This is broken into 1 TiB chunks. The first 1 TiB contains the PIO (64 kiB) and 32-bit MMIO (2 GiB) windows for all of the PHBs. Each subsequent TiB chunk contains a naturally aligned 64-bit MMIO window for one PHB each. This reduces the number of allowed PHBs (without full manual configuration of all the windows) from 256 to 31, but this should still be plenty in practice. We also change some of the default window sizes for manually configured PHBs to saner values. Finally we adjust some tests and libqos so that it correctly uses the new default locations. Ideally it would parse the device tree given to the guest, but that's a more complex problem for another time. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
2016-10-16 03:04:15 +02:00
smc->phb_placement = phb_placement_2_7;
}
DEFINE_SPAPR_MACHINE(2_7, "2.7", false);
/*
* pseries-2.6
*/
static void spapr_machine_2_6_class_options(MachineClass *mc)
{
static GlobalProperty compat[] = {
{ TYPE_SPAPR_PCI_HOST_BRIDGE, "ddw", "off" },
};
spapr_machine_2_7_class_options(mc);
mc->has_hotpluggable_cpus = false;
compat_props_add(mc->compat_props, hw_compat_2_6, hw_compat_2_6_len);
compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
}
DEFINE_SPAPR_MACHINE(2_6, "2.6", false);
/*
* pseries-2.5
*/
static void spapr_machine_2_5_class_options(MachineClass *mc)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
static GlobalProperty compat[] = {
{ "spapr-vlan", "use-rx-buffer-pools", "off" },
};
spapr_machine_2_6_class_options(mc);
smc->use_ohci_by_default = true;
compat_props_add(mc->compat_props, hw_compat_2_5, hw_compat_2_5_len);
compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
}
DEFINE_SPAPR_MACHINE(2_5, "2.5", false);
/*
* pseries-2.4
*/
static void spapr_machine_2_4_class_options(MachineClass *mc)
{
sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
spapr_machine_2_5_class_options(mc);
smc->dr_lmb_enabled = false;
compat_props_add(mc->compat_props, hw_compat_2_4, hw_compat_2_4_len);
}
DEFINE_SPAPR_MACHINE(2_4, "2.4", false);
/*
* pseries-2.3
*/
static void spapr_machine_2_3_class_options(MachineClass *mc)
{
static GlobalProperty compat[] = {
{ "spapr-pci-host-bridge", "dynamic-reconfiguration", "off" },
};
spapr_machine_2_4_class_options(mc);
compat_props_add(mc->compat_props, hw_compat_2_3, hw_compat_2_3_len);
compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
}
DEFINE_SPAPR_MACHINE(2_3, "2.3", false);
/*
* pseries-2.2
*/
static void spapr_machine_2_2_class_options(MachineClass *mc)
{
static GlobalProperty compat[] = {
{ TYPE_SPAPR_PCI_HOST_BRIDGE, "mem_win_size", "0x20000000" },
};
spapr_machine_2_3_class_options(mc);
compat_props_add(mc->compat_props, hw_compat_2_2, hw_compat_2_2_len);
compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
mc->default_machine_opts = "modern-hotplug-events=off,suppress-vmdesc=on";
}
DEFINE_SPAPR_MACHINE(2_2, "2.2", false);
/*
* pseries-2.1
*/
static void spapr_machine_2_1_class_options(MachineClass *mc)
{
spapr_machine_2_2_class_options(mc);
compat_props_add(mc->compat_props, hw_compat_2_1, hw_compat_2_1_len);
}
DEFINE_SPAPR_MACHINE(2_1, "2.1", false);
static void spapr_machine_register_types(void)
{
type_register_static(&spapr_machine_info);
}
type_init(spapr_machine_register_types)