2020-09-01 14:56:39 +02:00
|
|
|
/*
|
|
|
|
* QEMU PowerPC pSeries Logical Partition NUMA associativity handling
|
|
|
|
*
|
|
|
|
* Copyright IBM Corp. 2020
|
|
|
|
*
|
|
|
|
* Authors:
|
|
|
|
* Daniel Henrique Barboza <danielhb413@gmail.com>
|
|
|
|
*
|
|
|
|
* This work is licensed under the terms of the GNU GPL, version 2 or later.
|
|
|
|
* See the COPYING file in the top-level directory.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "qemu/osdep.h"
|
|
|
|
#include "qemu-common.h"
|
|
|
|
#include "hw/ppc/spapr_numa.h"
|
2020-09-04 00:06:36 +02:00
|
|
|
#include "hw/pci-host/spapr.h"
|
2020-09-01 14:56:39 +02:00
|
|
|
#include "hw/ppc/fdt.h"
|
|
|
|
|
2020-09-04 00:06:36 +02:00
|
|
|
/* Moved from hw/ppc/spapr_pci_nvlink2.c */
|
|
|
|
#define SPAPR_GPU_NUMA_ID (cpu_to_be32(1))
|
spapr: introduce SpaprMachineState::numa_assoc_array
The next step to centralize all NUMA/associativity handling in
the spapr machine is to create a 'one stop place' for all
things ibm,associativity.
This patch introduces numa_assoc_array, a 2 dimensional array
that will store all ibm,associativity arrays of all NUMA nodes.
This array is initialized in a new spapr_numa_associativity_init()
function, called in spapr_machine_init(). It is being initialized
with the same values used in other ibm,associativity properties
around spapr files (i.e. all zeros, last value is node_id).
The idea is to remove all hardcoded definitions and FDT writes
of ibm,associativity arrays, doing instead a call to the new
helper spapr_numa_write_associativity_dt() helper, that will
be able to write the DT with the correct values.
We'll start small, handling the trivial cases first. The
remaining instances of ibm,associativity will be handled
next.
Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>
Message-Id: <20200903220639.563090-2-danielhb413@gmail.com>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2020-09-04 00:06:33 +02:00
|
|
|
|
|
|
|
void spapr_numa_associativity_init(SpaprMachineState *spapr,
|
|
|
|
MachineState *machine)
|
|
|
|
{
|
2020-09-04 00:06:36 +02:00
|
|
|
SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
|
spapr: introduce SpaprMachineState::numa_assoc_array
The next step to centralize all NUMA/associativity handling in
the spapr machine is to create a 'one stop place' for all
things ibm,associativity.
This patch introduces numa_assoc_array, a 2 dimensional array
that will store all ibm,associativity arrays of all NUMA nodes.
This array is initialized in a new spapr_numa_associativity_init()
function, called in spapr_machine_init(). It is being initialized
with the same values used in other ibm,associativity properties
around spapr files (i.e. all zeros, last value is node_id).
The idea is to remove all hardcoded definitions and FDT writes
of ibm,associativity arrays, doing instead a call to the new
helper spapr_numa_write_associativity_dt() helper, that will
be able to write the DT with the correct values.
We'll start small, handling the trivial cases first. The
remaining instances of ibm,associativity will be handled
next.
Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>
Message-Id: <20200903220639.563090-2-danielhb413@gmail.com>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2020-09-04 00:06:33 +02:00
|
|
|
int nb_numa_nodes = machine->numa_state->num_nodes;
|
2020-09-04 00:06:36 +02:00
|
|
|
int i, j, max_nodes_with_gpus;
|
spapr: introduce SpaprMachineState::numa_assoc_array
The next step to centralize all NUMA/associativity handling in
the spapr machine is to create a 'one stop place' for all
things ibm,associativity.
This patch introduces numa_assoc_array, a 2 dimensional array
that will store all ibm,associativity arrays of all NUMA nodes.
This array is initialized in a new spapr_numa_associativity_init()
function, called in spapr_machine_init(). It is being initialized
with the same values used in other ibm,associativity properties
around spapr files (i.e. all zeros, last value is node_id).
The idea is to remove all hardcoded definitions and FDT writes
of ibm,associativity arrays, doing instead a call to the new
helper spapr_numa_write_associativity_dt() helper, that will
be able to write the DT with the correct values.
We'll start small, handling the trivial cases first. The
remaining instances of ibm,associativity will be handled
next.
Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>
Message-Id: <20200903220639.563090-2-danielhb413@gmail.com>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2020-09-04 00:06:33 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* For all associativity arrays: first position is the size,
|
|
|
|
* position MAX_DISTANCE_REF_POINTS is always the numa_id,
|
|
|
|
* represented by the index 'i'.
|
|
|
|
*
|
|
|
|
* This will break on sparse NUMA setups, when/if QEMU starts
|
|
|
|
* to support it, because there will be no more guarantee that
|
|
|
|
* 'i' will be a valid node_id set by the user.
|
|
|
|
*/
|
|
|
|
for (i = 0; i < nb_numa_nodes; i++) {
|
|
|
|
spapr->numa_assoc_array[i][0] = cpu_to_be32(MAX_DISTANCE_REF_POINTS);
|
|
|
|
spapr->numa_assoc_array[i][MAX_DISTANCE_REF_POINTS] = cpu_to_be32(i);
|
|
|
|
}
|
2020-09-04 00:06:36 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize NVLink GPU associativity arrays. We know that
|
|
|
|
* the first GPU will take the first available NUMA id, and
|
|
|
|
* we'll have a maximum of NVGPU_MAX_NUM GPUs in the machine.
|
|
|
|
* At this point we're not sure if there are GPUs or not, but
|
|
|
|
* let's initialize the associativity arrays and allow NVLink
|
|
|
|
* GPUs to be handled like regular NUMA nodes later on.
|
|
|
|
*/
|
|
|
|
max_nodes_with_gpus = nb_numa_nodes + NVGPU_MAX_NUM;
|
|
|
|
|
|
|
|
for (i = nb_numa_nodes; i < max_nodes_with_gpus; i++) {
|
|
|
|
spapr->numa_assoc_array[i][0] = cpu_to_be32(MAX_DISTANCE_REF_POINTS);
|
|
|
|
|
|
|
|
for (j = 1; j < MAX_DISTANCE_REF_POINTS; j++) {
|
|
|
|
uint32_t gpu_assoc = smc->pre_5_1_assoc_refpoints ?
|
|
|
|
SPAPR_GPU_NUMA_ID : cpu_to_be32(i);
|
|
|
|
spapr->numa_assoc_array[i][j] = gpu_assoc;
|
|
|
|
}
|
|
|
|
|
|
|
|
spapr->numa_assoc_array[i][MAX_DISTANCE_REF_POINTS] = cpu_to_be32(i);
|
|
|
|
}
|
spapr: introduce SpaprMachineState::numa_assoc_array
The next step to centralize all NUMA/associativity handling in
the spapr machine is to create a 'one stop place' for all
things ibm,associativity.
This patch introduces numa_assoc_array, a 2 dimensional array
that will store all ibm,associativity arrays of all NUMA nodes.
This array is initialized in a new spapr_numa_associativity_init()
function, called in spapr_machine_init(). It is being initialized
with the same values used in other ibm,associativity properties
around spapr files (i.e. all zeros, last value is node_id).
The idea is to remove all hardcoded definitions and FDT writes
of ibm,associativity arrays, doing instead a call to the new
helper spapr_numa_write_associativity_dt() helper, that will
be able to write the DT with the correct values.
We'll start small, handling the trivial cases first. The
remaining instances of ibm,associativity will be handled
next.
Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>
Message-Id: <20200903220639.563090-2-danielhb413@gmail.com>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2020-09-04 00:06:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void spapr_numa_write_associativity_dt(SpaprMachineState *spapr, void *fdt,
|
|
|
|
int offset, int nodeid)
|
|
|
|
{
|
|
|
|
_FDT((fdt_setprop(fdt, offset, "ibm,associativity",
|
|
|
|
spapr->numa_assoc_array[nodeid],
|
|
|
|
sizeof(spapr->numa_assoc_array[nodeid]))));
|
2020-09-04 00:06:34 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
int spapr_numa_fixup_cpu_dt(SpaprMachineState *spapr, void *fdt,
|
|
|
|
int offset, PowerPCCPU *cpu)
|
|
|
|
{
|
|
|
|
int vcpu_assoc_size = NUMA_ASSOC_SIZE + 1;
|
|
|
|
uint32_t vcpu_assoc[vcpu_assoc_size];
|
|
|
|
int index = spapr_get_vcpu_id(cpu);
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* VCPUs have an extra 'cpu_id' value in ibm,associativity
|
|
|
|
* compared to other resources. Increment the size at index
|
|
|
|
* 0, copy all associativity domains already set, then put
|
|
|
|
* cpu_id last.
|
|
|
|
*/
|
|
|
|
vcpu_assoc[0] = cpu_to_be32(MAX_DISTANCE_REF_POINTS + 1);
|
|
|
|
|
|
|
|
for (i = 1; i <= MAX_DISTANCE_REF_POINTS; i++) {
|
|
|
|
vcpu_assoc[i] = spapr->numa_assoc_array[cpu->node_id][i];
|
|
|
|
}
|
|
|
|
|
|
|
|
vcpu_assoc[vcpu_assoc_size - 1] = cpu_to_be32(index);
|
|
|
|
|
|
|
|
/* Advertise NUMA via ibm,associativity */
|
|
|
|
return fdt_setprop(fdt, offset, "ibm,associativity",
|
|
|
|
vcpu_assoc, sizeof(vcpu_assoc));
|
spapr: introduce SpaprMachineState::numa_assoc_array
The next step to centralize all NUMA/associativity handling in
the spapr machine is to create a 'one stop place' for all
things ibm,associativity.
This patch introduces numa_assoc_array, a 2 dimensional array
that will store all ibm,associativity arrays of all NUMA nodes.
This array is initialized in a new spapr_numa_associativity_init()
function, called in spapr_machine_init(). It is being initialized
with the same values used in other ibm,associativity properties
around spapr files (i.e. all zeros, last value is node_id).
The idea is to remove all hardcoded definitions and FDT writes
of ibm,associativity arrays, doing instead a call to the new
helper spapr_numa_write_associativity_dt() helper, that will
be able to write the DT with the correct values.
We'll start small, handling the trivial cases first. The
remaining instances of ibm,associativity will be handled
next.
Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>
Message-Id: <20200903220639.563090-2-danielhb413@gmail.com>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
2020-09-04 00:06:33 +02:00
|
|
|
}
|
|
|
|
|
2020-09-04 00:06:35 +02:00
|
|
|
|
|
|
|
int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState *spapr, void *fdt,
|
|
|
|
int offset)
|
|
|
|
{
|
|
|
|
MachineState *machine = MACHINE(spapr);
|
|
|
|
int nb_numa_nodes = machine->numa_state->num_nodes;
|
|
|
|
int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
|
|
|
|
uint32_t *int_buf, *cur_index, buf_len;
|
|
|
|
int ret, i;
|
|
|
|
|
|
|
|
/* ibm,associativity-lookup-arrays */
|
|
|
|
buf_len = (nr_nodes * MAX_DISTANCE_REF_POINTS + 2) * sizeof(uint32_t);
|
|
|
|
cur_index = int_buf = g_malloc0(buf_len);
|
|
|
|
int_buf[0] = cpu_to_be32(nr_nodes);
|
|
|
|
/* Number of entries per associativity list */
|
|
|
|
int_buf[1] = cpu_to_be32(MAX_DISTANCE_REF_POINTS);
|
|
|
|
cur_index += 2;
|
|
|
|
for (i = 0; i < nr_nodes; i++) {
|
|
|
|
/*
|
|
|
|
* For the lookup-array we use the ibm,associativity array,
|
|
|
|
* from numa_assoc_array. without the first element (size).
|
|
|
|
*/
|
|
|
|
uint32_t *associativity = spapr->numa_assoc_array[i];
|
|
|
|
memcpy(cur_index, ++associativity,
|
|
|
|
sizeof(uint32_t) * MAX_DISTANCE_REF_POINTS);
|
|
|
|
cur_index += MAX_DISTANCE_REF_POINTS;
|
|
|
|
}
|
|
|
|
ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
|
|
|
|
(cur_index - int_buf) * sizeof(uint32_t));
|
|
|
|
g_free(int_buf);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2020-09-01 14:56:39 +02:00
|
|
|
/*
|
|
|
|
* Helper that writes ibm,associativity-reference-points and
|
|
|
|
* max-associativity-domains in the RTAS pointed by @rtas
|
|
|
|
* in the DT @fdt.
|
|
|
|
*/
|
|
|
|
void spapr_numa_write_rtas_dt(SpaprMachineState *spapr, void *fdt, int rtas)
|
|
|
|
{
|
|
|
|
SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
|
|
|
|
uint32_t refpoints[] = {
|
|
|
|
cpu_to_be32(0x4),
|
|
|
|
cpu_to_be32(0x4),
|
|
|
|
cpu_to_be32(0x2),
|
|
|
|
};
|
|
|
|
uint32_t nr_refpoints = ARRAY_SIZE(refpoints);
|
|
|
|
uint32_t maxdomain = cpu_to_be32(spapr->gpu_numa_id > 1 ? 1 : 0);
|
|
|
|
uint32_t maxdomains[] = {
|
|
|
|
cpu_to_be32(4),
|
|
|
|
maxdomain,
|
|
|
|
maxdomain,
|
|
|
|
maxdomain,
|
|
|
|
cpu_to_be32(spapr->gpu_numa_id),
|
|
|
|
};
|
|
|
|
|
|
|
|
if (smc->pre_5_1_assoc_refpoints) {
|
|
|
|
nr_refpoints = 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
_FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points",
|
|
|
|
refpoints, nr_refpoints * sizeof(refpoints[0])));
|
|
|
|
|
|
|
|
_FDT(fdt_setprop(fdt, rtas, "ibm,max-associativity-domains",
|
|
|
|
maxdomains, sizeof(maxdomains)));
|
|
|
|
}
|