2021-07-19 13:21:15 +02:00
|
|
|
/*
|
|
|
|
* SGX common code
|
|
|
|
*
|
|
|
|
* Copyright (C) 2021 Intel Corporation
|
|
|
|
*
|
|
|
|
* Authors:
|
|
|
|
* Yang Zhong<yang.zhong@intel.com>
|
|
|
|
* Sean Christopherson <sean.j.christopherson@intel.com>
|
|
|
|
*
|
|
|
|
* This work is licensed under the terms of the GNU GPL, version 2 or later.
|
|
|
|
* See the COPYING file in the top-level directory.
|
|
|
|
*/
|
|
|
|
#include "qemu/osdep.h"
|
|
|
|
#include "hw/i386/pc.h"
|
|
|
|
#include "hw/i386/sgx-epc.h"
|
|
|
|
#include "hw/mem/memory-device.h"
|
|
|
|
#include "monitor/qdev.h"
|
2021-10-07 19:56:12 +02:00
|
|
|
#include "monitor/monitor.h"
|
|
|
|
#include "monitor/hmp-target.h"
|
2021-07-19 13:21:15 +02:00
|
|
|
#include "qapi/error.h"
|
2021-10-07 19:56:11 +02:00
|
|
|
#include "qapi/qapi-commands-misc-target.h"
|
2021-07-19 13:21:15 +02:00
|
|
|
#include "exec/address-spaces.h"
|
2021-09-10 12:22:57 +02:00
|
|
|
#include "sysemu/hw_accel.h"
|
2021-11-01 17:20:09 +01:00
|
|
|
#include "sysemu/reset.h"
|
|
|
|
#include <sys/ioctl.h>
|
numa: Enable numa for SGX EPC sections
The basic SGX did not enable numa for SGX EPC sections, which
result in all EPC sections located in numa node 0. This patch
enable SGX numa function in the guest and the EPC section can
work with RAM as one numa node.
The Guest kernel related log:
[ 0.009981] ACPI: SRAT: Node 0 PXM 0 [mem 0x180000000-0x183ffffff]
[ 0.009982] ACPI: SRAT: Node 1 PXM 1 [mem 0x184000000-0x185bfffff]
The SRAT table can normally show SGX EPC sections menory info in different
numa nodes.
The SGX EPC numa related command:
......
-m 4G,maxmem=20G \
-smp sockets=2,cores=2 \
-cpu host,+sgx-provisionkey \
-object memory-backend-ram,size=2G,host-nodes=0,policy=bind,id=node0 \
-object memory-backend-epc,id=mem0,size=64M,prealloc=on,host-nodes=0,policy=bind \
-numa node,nodeid=0,cpus=0-1,memdev=node0 \
-object memory-backend-ram,size=2G,host-nodes=1,policy=bind,id=node1 \
-object memory-backend-epc,id=mem1,size=28M,prealloc=on,host-nodes=1,policy=bind \
-numa node,nodeid=1,cpus=2-3,memdev=node1 \
-M sgx-epc.0.memdev=mem0,sgx-epc.0.node=0,sgx-epc.1.memdev=mem1,sgx-epc.1.node=1 \
......
Signed-off-by: Yang Zhong <yang.zhong@intel.com>
Message-Id: <20211101162009.62161-2-yang.zhong@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-11-01 17:20:05 +01:00
|
|
|
#include "hw/acpi/aml-build.h"
|
2021-09-10 12:22:57 +02:00
|
|
|
|
|
|
|
#define SGX_MAX_EPC_SECTIONS 8
|
|
|
|
#define SGX_CPUID_EPC_INVALID 0x0
|
|
|
|
|
|
|
|
/* A valid EPC section. */
|
|
|
|
#define SGX_CPUID_EPC_SECTION 0x1
|
|
|
|
#define SGX_CPUID_EPC_MASK 0xF
|
|
|
|
|
2021-11-01 17:20:09 +01:00
|
|
|
#define SGX_MAGIC 0xA4
|
|
|
|
#define SGX_IOC_VEPC_REMOVE_ALL _IO(SGX_MAGIC, 0x04)
|
|
|
|
|
|
|
|
#define RETRY_NUM 2
|
|
|
|
|
numa: Enable numa for SGX EPC sections
The basic SGX did not enable numa for SGX EPC sections, which
result in all EPC sections located in numa node 0. This patch
enable SGX numa function in the guest and the EPC section can
work with RAM as one numa node.
The Guest kernel related log:
[ 0.009981] ACPI: SRAT: Node 0 PXM 0 [mem 0x180000000-0x183ffffff]
[ 0.009982] ACPI: SRAT: Node 1 PXM 1 [mem 0x184000000-0x185bfffff]
The SRAT table can normally show SGX EPC sections menory info in different
numa nodes.
The SGX EPC numa related command:
......
-m 4G,maxmem=20G \
-smp sockets=2,cores=2 \
-cpu host,+sgx-provisionkey \
-object memory-backend-ram,size=2G,host-nodes=0,policy=bind,id=node0 \
-object memory-backend-epc,id=mem0,size=64M,prealloc=on,host-nodes=0,policy=bind \
-numa node,nodeid=0,cpus=0-1,memdev=node0 \
-object memory-backend-ram,size=2G,host-nodes=1,policy=bind,id=node1 \
-object memory-backend-epc,id=mem1,size=28M,prealloc=on,host-nodes=1,policy=bind \
-numa node,nodeid=1,cpus=2-3,memdev=node1 \
-M sgx-epc.0.memdev=mem0,sgx-epc.0.node=0,sgx-epc.1.memdev=mem1,sgx-epc.1.node=1 \
......
Signed-off-by: Yang Zhong <yang.zhong@intel.com>
Message-Id: <20211101162009.62161-2-yang.zhong@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-11-01 17:20:05 +01:00
|
|
|
static int sgx_epc_device_list(Object *obj, void *opaque)
|
|
|
|
{
|
|
|
|
GSList **list = opaque;
|
|
|
|
|
|
|
|
if (object_dynamic_cast(obj, TYPE_SGX_EPC)) {
|
|
|
|
*list = g_slist_append(*list, DEVICE(obj));
|
|
|
|
}
|
|
|
|
|
|
|
|
object_child_foreach(obj, sgx_epc_device_list, opaque);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static GSList *sgx_epc_get_device_list(void)
|
|
|
|
{
|
|
|
|
GSList *list = NULL;
|
|
|
|
|
|
|
|
object_child_foreach(qdev_get_machine(), sgx_epc_device_list, &list);
|
|
|
|
return list;
|
|
|
|
}
|
|
|
|
|
|
|
|
void sgx_epc_build_srat(GArray *table_data)
|
|
|
|
{
|
|
|
|
GSList *device_list = sgx_epc_get_device_list();
|
|
|
|
|
|
|
|
for (; device_list; device_list = device_list->next) {
|
|
|
|
DeviceState *dev = device_list->data;
|
|
|
|
Object *obj = OBJECT(dev);
|
|
|
|
uint64_t addr, size;
|
|
|
|
int node;
|
|
|
|
|
|
|
|
node = object_property_get_uint(obj, SGX_EPC_NUMA_NODE_PROP,
|
|
|
|
&error_abort);
|
|
|
|
addr = object_property_get_uint(obj, SGX_EPC_ADDR_PROP, &error_abort);
|
|
|
|
size = object_property_get_uint(obj, SGX_EPC_SIZE_PROP, &error_abort);
|
|
|
|
|
|
|
|
build_srat_memory(table_data, addr, size, node, MEM_AFFINITY_ENABLED);
|
|
|
|
}
|
|
|
|
g_slist_free(device_list);
|
|
|
|
}
|
|
|
|
|
2021-09-10 12:22:57 +02:00
|
|
|
static uint64_t sgx_calc_section_metric(uint64_t low, uint64_t high)
|
|
|
|
{
|
|
|
|
return (low & MAKE_64BIT_MASK(12, 20)) +
|
|
|
|
((high & MAKE_64BIT_MASK(0, 20)) << 32);
|
|
|
|
}
|
|
|
|
|
|
|
|
static uint64_t sgx_calc_host_epc_section_size(void)
|
|
|
|
{
|
|
|
|
uint32_t i, type;
|
|
|
|
uint32_t eax, ebx, ecx, edx;
|
|
|
|
uint64_t size = 0;
|
|
|
|
|
|
|
|
for (i = 0; i < SGX_MAX_EPC_SECTIONS; i++) {
|
|
|
|
host_cpuid(0x12, i + 2, &eax, &ebx, &ecx, &edx);
|
|
|
|
|
|
|
|
type = eax & SGX_CPUID_EPC_MASK;
|
|
|
|
if (type == SGX_CPUID_EPC_INVALID) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (type != SGX_CPUID_EPC_SECTION) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
size += sgx_calc_section_metric(ecx, edx);
|
|
|
|
}
|
|
|
|
|
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
2021-11-01 17:20:09 +01:00
|
|
|
static void sgx_epc_reset(void *opaque)
|
|
|
|
{
|
|
|
|
PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
|
|
|
|
HostMemoryBackend *hostmem;
|
|
|
|
SGXEPCDevice *epc;
|
|
|
|
int failures;
|
|
|
|
int fd, i, j, r;
|
|
|
|
static bool warned = false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The second pass is needed to remove SECS pages that could not
|
|
|
|
* be removed during the first.
|
|
|
|
*/
|
|
|
|
for (i = 0; i < RETRY_NUM; i++) {
|
|
|
|
failures = 0;
|
|
|
|
for (j = 0; j < pcms->sgx_epc.nr_sections; j++) {
|
|
|
|
epc = pcms->sgx_epc.sections[j];
|
|
|
|
hostmem = MEMORY_BACKEND(epc->hostmem);
|
|
|
|
fd = memory_region_get_fd(host_memory_backend_get_memory(hostmem));
|
|
|
|
|
|
|
|
r = ioctl(fd, SGX_IOC_VEPC_REMOVE_ALL);
|
|
|
|
if (r == -ENOTTY && !warned) {
|
|
|
|
warned = true;
|
|
|
|
warn_report("kernel does not support SGX_IOC_VEPC_REMOVE_ALL");
|
|
|
|
warn_report("SGX might operate incorrectly in the guest after reset");
|
|
|
|
break;
|
|
|
|
} else if (r > 0) {
|
|
|
|
/* SECS pages remain */
|
|
|
|
failures++;
|
|
|
|
if (i == 1) {
|
|
|
|
error_report("cannot reset vEPC section %d", j);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!failures) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-07 19:56:11 +02:00
|
|
|
SGXInfo *qmp_query_sgx_capabilities(Error **errp)
|
2021-09-10 12:22:57 +02:00
|
|
|
{
|
|
|
|
SGXInfo *info = NULL;
|
|
|
|
uint32_t eax, ebx, ecx, edx;
|
|
|
|
|
|
|
|
int fd = qemu_open_old("/dev/sgx_vepc", O_RDWR);
|
|
|
|
if (fd < 0) {
|
|
|
|
error_setg(errp, "SGX is not enabled in KVM");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
info = g_new0(SGXInfo, 1);
|
|
|
|
host_cpuid(0x7, 0, &eax, &ebx, &ecx, &edx);
|
|
|
|
|
|
|
|
info->sgx = ebx & (1U << 2) ? true : false;
|
|
|
|
info->flc = ecx & (1U << 30) ? true : false;
|
|
|
|
|
|
|
|
host_cpuid(0x12, 0, &eax, &ebx, &ecx, &edx);
|
|
|
|
info->sgx1 = eax & (1U << 0) ? true : false;
|
|
|
|
info->sgx2 = eax & (1U << 1) ? true : false;
|
|
|
|
|
|
|
|
info->section_size = sgx_calc_host_epc_section_size();
|
|
|
|
|
|
|
|
close(fd);
|
|
|
|
|
|
|
|
return info;
|
|
|
|
}
|
2021-09-10 12:22:56 +02:00
|
|
|
|
2021-10-07 19:56:12 +02:00
|
|
|
SGXInfo *qmp_query_sgx(Error **errp)
|
2021-09-10 12:22:56 +02:00
|
|
|
{
|
|
|
|
SGXInfo *info = NULL;
|
|
|
|
X86MachineState *x86ms;
|
|
|
|
PCMachineState *pcms =
|
|
|
|
(PCMachineState *)object_dynamic_cast(qdev_get_machine(),
|
|
|
|
TYPE_PC_MACHINE);
|
|
|
|
if (!pcms) {
|
|
|
|
error_setg(errp, "SGX is only supported on PC machines");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
x86ms = X86_MACHINE(pcms);
|
|
|
|
if (!x86ms->sgx_epc_list) {
|
|
|
|
error_setg(errp, "No EPC regions defined, SGX not available");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
SGXEPCState *sgx_epc = &pcms->sgx_epc;
|
|
|
|
info = g_new0(SGXInfo, 1);
|
|
|
|
|
|
|
|
info->sgx = true;
|
|
|
|
info->sgx1 = true;
|
|
|
|
info->sgx2 = true;
|
|
|
|
info->flc = true;
|
|
|
|
info->section_size = sgx_epc->size;
|
|
|
|
|
|
|
|
return info;
|
|
|
|
}
|
2021-07-19 13:21:15 +02:00
|
|
|
|
2021-10-07 19:56:12 +02:00
|
|
|
void hmp_info_sgx(Monitor *mon, const QDict *qdict)
|
|
|
|
{
|
|
|
|
Error *err = NULL;
|
|
|
|
g_autoptr(SGXInfo) info = qmp_query_sgx(&err);
|
|
|
|
|
|
|
|
if (err) {
|
|
|
|
error_report_err(err);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
monitor_printf(mon, "SGX support: %s\n",
|
|
|
|
info->sgx ? "enabled" : "disabled");
|
|
|
|
monitor_printf(mon, "SGX1 support: %s\n",
|
|
|
|
info->sgx1 ? "enabled" : "disabled");
|
|
|
|
monitor_printf(mon, "SGX2 support: %s\n",
|
|
|
|
info->sgx2 ? "enabled" : "disabled");
|
|
|
|
monitor_printf(mon, "FLC support: %s\n",
|
|
|
|
info->flc ? "enabled" : "disabled");
|
|
|
|
monitor_printf(mon, "size: %" PRIu64 "\n",
|
|
|
|
info->section_size);
|
|
|
|
}
|
|
|
|
|
2021-10-07 19:56:10 +02:00
|
|
|
bool sgx_epc_get_section(int section_nr, uint64_t *addr, uint64_t *size)
|
2021-07-19 13:21:15 +02:00
|
|
|
{
|
|
|
|
PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
|
|
|
|
SGXEPCDevice *epc;
|
|
|
|
|
|
|
|
if (pcms->sgx_epc.size == 0 || pcms->sgx_epc.nr_sections <= section_nr) {
|
2021-10-07 19:56:10 +02:00
|
|
|
return true;
|
2021-07-19 13:21:15 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
epc = pcms->sgx_epc.sections[section_nr];
|
|
|
|
|
|
|
|
*addr = epc->addr;
|
|
|
|
*size = memory_device_get_region_size(MEMORY_DEVICE(epc), &error_fatal);
|
|
|
|
|
2021-10-07 19:56:10 +02:00
|
|
|
return false;
|
2021-07-19 13:21:15 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void pc_machine_init_sgx_epc(PCMachineState *pcms)
|
|
|
|
{
|
|
|
|
SGXEPCState *sgx_epc = &pcms->sgx_epc;
|
|
|
|
X86MachineState *x86ms = X86_MACHINE(pcms);
|
|
|
|
SgxEPCList *list = NULL;
|
|
|
|
Object *obj;
|
|
|
|
|
|
|
|
memset(sgx_epc, 0, sizeof(SGXEPCState));
|
|
|
|
if (!x86ms->sgx_epc_list) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
sgx_epc->base = 0x100000000ULL + x86ms->above_4g_mem_size;
|
|
|
|
|
|
|
|
memory_region_init(&sgx_epc->mr, OBJECT(pcms), "sgx-epc", UINT64_MAX);
|
|
|
|
memory_region_add_subregion(get_system_memory(), sgx_epc->base,
|
|
|
|
&sgx_epc->mr);
|
|
|
|
|
|
|
|
for (list = x86ms->sgx_epc_list; list; list = list->next) {
|
|
|
|
obj = object_new("sgx-epc");
|
|
|
|
|
|
|
|
/* set the memdev link with memory backend */
|
|
|
|
object_property_parse(obj, SGX_EPC_MEMDEV_PROP, list->value->memdev,
|
|
|
|
&error_fatal);
|
numa: Enable numa for SGX EPC sections
The basic SGX did not enable numa for SGX EPC sections, which
result in all EPC sections located in numa node 0. This patch
enable SGX numa function in the guest and the EPC section can
work with RAM as one numa node.
The Guest kernel related log:
[ 0.009981] ACPI: SRAT: Node 0 PXM 0 [mem 0x180000000-0x183ffffff]
[ 0.009982] ACPI: SRAT: Node 1 PXM 1 [mem 0x184000000-0x185bfffff]
The SRAT table can normally show SGX EPC sections menory info in different
numa nodes.
The SGX EPC numa related command:
......
-m 4G,maxmem=20G \
-smp sockets=2,cores=2 \
-cpu host,+sgx-provisionkey \
-object memory-backend-ram,size=2G,host-nodes=0,policy=bind,id=node0 \
-object memory-backend-epc,id=mem0,size=64M,prealloc=on,host-nodes=0,policy=bind \
-numa node,nodeid=0,cpus=0-1,memdev=node0 \
-object memory-backend-ram,size=2G,host-nodes=1,policy=bind,id=node1 \
-object memory-backend-epc,id=mem1,size=28M,prealloc=on,host-nodes=1,policy=bind \
-numa node,nodeid=1,cpus=2-3,memdev=node1 \
-M sgx-epc.0.memdev=mem0,sgx-epc.0.node=0,sgx-epc.1.memdev=mem1,sgx-epc.1.node=1 \
......
Signed-off-by: Yang Zhong <yang.zhong@intel.com>
Message-Id: <20211101162009.62161-2-yang.zhong@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-11-01 17:20:05 +01:00
|
|
|
/* set the numa node property for sgx epc object */
|
|
|
|
object_property_set_uint(obj, SGX_EPC_NUMA_NODE_PROP, list->value->node,
|
|
|
|
&error_fatal);
|
2021-07-19 13:21:15 +02:00
|
|
|
object_property_set_bool(obj, "realized", true, &error_fatal);
|
|
|
|
object_unref(obj);
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((sgx_epc->base + sgx_epc->size) < sgx_epc->base) {
|
|
|
|
error_report("Size of all 'sgx-epc' =0x%"PRIu64" causes EPC to wrap",
|
|
|
|
sgx_epc->size);
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
|
|
|
|
|
|
|
memory_region_set_size(&sgx_epc->mr, sgx_epc->size);
|
2021-11-01 17:20:09 +01:00
|
|
|
|
|
|
|
/* register the reset callback for sgx epc */
|
|
|
|
qemu_register_reset(sgx_epc_reset, NULL);
|
2021-07-19 13:21:15 +02:00
|
|
|
}
|